chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1 @@
|
||||
github: [simonw]
|
||||
@@ -0,0 +1,10 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: pip
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: daily
|
||||
groups:
|
||||
python-packages:
|
||||
patterns:
|
||||
- "*"
|
||||
@@ -0,0 +1,49 @@
|
||||
name: Run Cog
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
run-cog:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.head_ref }}
|
||||
|
||||
- name: Set up Python 3.11
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install . --group dev
|
||||
|
||||
- name: Run cog
|
||||
run: |
|
||||
cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md
|
||||
|
||||
- name: Check for changes
|
||||
id: check-changes
|
||||
run: |
|
||||
if [ -n "$(git diff)" ]; then
|
||||
echo "changes=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "changes=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Commit and push if changed
|
||||
if: steps.check-changes.outputs.changes == 'true'
|
||||
run: |
|
||||
git config --local user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git config --local user.name "github-actions[bot]"
|
||||
git add -A
|
||||
git commit -m "Ran cog"
|
||||
git push
|
||||
@@ -0,0 +1,51 @@
|
||||
name: Publish Python Package
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: pip
|
||||
cache-dependency-path: pyproject.toml
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install . --group dev
|
||||
- name: Run tests
|
||||
run: |
|
||||
pytest
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
permissions:
|
||||
id-token: write
|
||||
needs: [test]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.13'
|
||||
cache: pip
|
||||
cache-dependency-path: pyproject.toml
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install setuptools wheel build
|
||||
- name: Build
|
||||
run: |
|
||||
python -m build
|
||||
- name: Publish
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
@@ -0,0 +1,64 @@
|
||||
name: Update Stable Docs
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
update_stable_docs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0 # We need all commits to find docs/ changes
|
||||
- name: Set up Git user
|
||||
run: |
|
||||
git config user.name "Automated"
|
||||
git config user.email "actions@users.noreply.github.com"
|
||||
- name: Create stable branch if it does not yet exist
|
||||
run: |
|
||||
if ! git ls-remote --heads origin stable | grep stable; then
|
||||
git checkout -b stable
|
||||
# If there are any releases, copy docs/ in from most recent
|
||||
LATEST_RELEASE=$(git tag | sort -Vr | head -n1)
|
||||
if [ -n "$LATEST_RELEASE" ]; then
|
||||
rm -rf docs/
|
||||
git checkout $LATEST_RELEASE -- docs/
|
||||
fi
|
||||
git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes"
|
||||
git push -u origin stable
|
||||
fi
|
||||
- name: Handle Release
|
||||
if: github.event_name == 'release' && !github.event.release.prerelease
|
||||
run: |
|
||||
git fetch --all
|
||||
git checkout stable
|
||||
git reset --hard ${GITHUB_REF#refs/tags/}
|
||||
git push origin stable --force
|
||||
- name: Handle Commit to Main
|
||||
if: contains(github.event.head_commit.message, '!stable-docs')
|
||||
run: |
|
||||
git fetch origin
|
||||
git checkout -b stable origin/stable
|
||||
# Get the list of modified files in docs/ from the current commit
|
||||
FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/)
|
||||
# Check if the list of files is non-empty
|
||||
if [[ -n "$FILES" ]]; then
|
||||
# Checkout those files to the stable branch to over-write with their contents
|
||||
for FILE in $FILES; do
|
||||
git checkout ${{ github.sha }} -- $FILE
|
||||
done
|
||||
git add docs/
|
||||
git commit -m "Doc changes from ${{ github.sha }}"
|
||||
git push origin stable
|
||||
else
|
||||
echo "No changes to docs/ in this commit."
|
||||
exit 0
|
||||
fi
|
||||
@@ -0,0 +1,69 @@
|
||||
name: Test
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
sqlite-utils-version: [""]
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
python-version: "3.14"
|
||||
sqlite-utils-version: "4.0rc4"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: pip
|
||||
cache-dependency-path: pyproject.toml
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install . --group dev
|
||||
- name: Install sqlite-utils pre-release
|
||||
if: matrix.sqlite-utils-version != ''
|
||||
run: |
|
||||
pip install sqlite-utils==${{ matrix.sqlite-utils-version }}
|
||||
- name: Run tests
|
||||
run: |
|
||||
python -m pytest -vv
|
||||
- name: Check if cog needs to be run
|
||||
if: matrix.sqlite-utils-version == '4.0rc4'
|
||||
run: |
|
||||
cog --check \
|
||||
-p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \
|
||||
docs/**/*.md docs/*.md
|
||||
- name: Run Black
|
||||
if: matrix.os != 'windows-latest'
|
||||
run: |
|
||||
black --check .
|
||||
- name: Run mypy
|
||||
if: matrix.os != 'windows-latest'
|
||||
run: |
|
||||
mypy llm
|
||||
- name: Run ruff
|
||||
if: matrix.os != 'windows-latest'
|
||||
run: |
|
||||
ruff check .
|
||||
- name: Check it builds
|
||||
run: |
|
||||
python -m build
|
||||
- name: Run test-llm-load-plugins.sh
|
||||
if: matrix.os != 'windows-latest'
|
||||
run: |
|
||||
llm install llm-cluster llm-mistral
|
||||
./tests/test-llm-load-plugins.sh
|
||||
- name: Upload artifact of builds
|
||||
if: matrix.python-version == '3.13' && matrix.os == 'ubuntu-latest'
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: dist-${{ matrix.os }}-${{ matrix.python-version }}
|
||||
path: dist/*
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
.venv
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
venv
|
||||
.eggs
|
||||
.pytest_cache
|
||||
*.egg-info
|
||||
build/
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
uv.lock
|
||||
*.db
|
||||
@@ -0,0 +1,19 @@
|
||||
version: 2
|
||||
|
||||
build:
|
||||
os: ubuntu-22.04
|
||||
tools:
|
||||
python: "3.11"
|
||||
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
|
||||
formats:
|
||||
- pdf
|
||||
- epub
|
||||
|
||||
python:
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
- method: pip
|
||||
path: .
|
||||
@@ -0,0 +1,24 @@
|
||||
# AGENTS.md
|
||||
|
||||
This project uses a Python environment for development and tests.
|
||||
|
||||
## Setting up a development environment
|
||||
|
||||
1. Install the project with its test dependencies:
|
||||
```bash
|
||||
pip install -e '.[test]'
|
||||
```
|
||||
2. Run the tests:
|
||||
```bash
|
||||
pytest
|
||||
```
|
||||
|
||||
## Building the documentation
|
||||
|
||||
Run the following commands if you want to build the docs locally:
|
||||
|
||||
```bash
|
||||
cd docs
|
||||
pip install -r requirements.txt
|
||||
make html
|
||||
```
|
||||
@@ -0,0 +1,46 @@
|
||||
# Run tests and linters
|
||||
@default: test lint
|
||||
|
||||
# Run pytest with supplied options
|
||||
@test *options:
|
||||
uv run pytest {{options}}
|
||||
|
||||
# Run linters
|
||||
@lint:
|
||||
echo "Linters..."
|
||||
echo " Black"
|
||||
uv run black . --check
|
||||
echo " cog"
|
||||
uv run --with sqlite-utils==4.0rc2 cog --check \
|
||||
-p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" \
|
||||
README.md docs/*.md
|
||||
echo " mypy"
|
||||
uv run mypy llm
|
||||
echo " ruff"
|
||||
uv run ruff check .
|
||||
|
||||
# Run mypy
|
||||
@mypy:
|
||||
uv run mypy llm
|
||||
|
||||
# Rebuild docs with cog
|
||||
@cog:
|
||||
uv run --with sqlite-utils==4.0rc2 cog -r -p "import sys, os; sys._called_from_test=True; os.environ['LLM_USER_PATH'] = '/tmp'" docs/**/*.md docs/*.md README.md
|
||||
|
||||
# Serve live docs on localhost:8000
|
||||
@docs: cog
|
||||
rm -rf docs/_build
|
||||
cd docs && uv run make livehtml
|
||||
|
||||
# Apply Black
|
||||
@black:
|
||||
uv run black .
|
||||
|
||||
# Run automatic fixes
|
||||
@fix: cog
|
||||
uv run ruff check . --fix
|
||||
uv run black .
|
||||
|
||||
# Push commit if tests pass
|
||||
@push: test lint
|
||||
git push
|
||||
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1 @@
|
||||
global-exclude tests/*
|
||||
@@ -0,0 +1,368 @@
|
||||
<!-- [[[cog
|
||||
# README.md is generated from docs/index.md using sphinx_markdown_builder
|
||||
import tempfile
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
readme_markdown = ''
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir)
|
||||
# Run: sphinx-build -M markdown ./docs ./tmpdir
|
||||
subprocess.run([
|
||||
"sphinx-build",
|
||||
"-M", "markdown",
|
||||
"./docs",
|
||||
str(tmp_path)
|
||||
], check=True)
|
||||
index_file = tmp_path / "markdown" / "index.md"
|
||||
readme_markdown = index_file.read_text(encoding="utf-8")
|
||||
|
||||
cog.out(readme_markdown)
|
||||
]]] -->
|
||||
# LLM
|
||||
|
||||
[](https://github.com/simonw/llm)
|
||||
[](https://pypi.org/project/llm/)
|
||||
[](https://llm.datasette.io/en/stable/changelog.html)
|
||||
[](https://github.com/simonw/llm/actions?query=workflow%3ATest)
|
||||
[](https://github.com/simonw/llm/blob/main/LICENSE)
|
||||
[](https://datasette.io/discord-llm)
|
||||
[](https://formulae.brew.sh/formula/llm)
|
||||
|
||||
A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Meta’s Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine.
|
||||
|
||||
Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/).
|
||||
|
||||
With LLM you can:
|
||||
|
||||
- [Run prompts from the command-line](https://llm.datasette.io/en/stable/usage.html#usage-executing-prompts)
|
||||
- [Store prompts and responses in SQLite](https://llm.datasette.io/en/stable/logging.html#logging)
|
||||
- [Generate and store embeddings](https://llm.datasette.io/en/stable/embeddings/index.html#embeddings)
|
||||
- [Extract structured content from text and images](https://llm.datasette.io/en/stable/schemas.html#schemas)
|
||||
- [Grant models the ability to execute tools](https://llm.datasette.io/en/stable/tools.html#tools)
|
||||
- … and much, much more
|
||||
|
||||
## Quick start
|
||||
|
||||
First, install LLM using `pip` or Homebrew or `pipx` or `uv`:
|
||||
|
||||
```bash
|
||||
pip install llm
|
||||
```
|
||||
|
||||
Or with Homebrew (see [warning note](https://llm.datasette.io/en/stable/setup.html#homebrew-warning)):
|
||||
|
||||
```bash
|
||||
brew install llm
|
||||
```
|
||||
|
||||
Or with [pipx](https://pypa.github.io/pipx/):
|
||||
|
||||
```bash
|
||||
pipx install llm
|
||||
```
|
||||
|
||||
Or with [uv](https://docs.astral.sh/uv/guides/tools/)
|
||||
|
||||
```bash
|
||||
uv tool install llm
|
||||
```
|
||||
|
||||
If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this:
|
||||
|
||||
```bash
|
||||
# Paste your OpenAI API key into this
|
||||
llm keys set openai
|
||||
|
||||
# Run a prompt (with the default gpt-4o-mini model)
|
||||
llm "Ten fun names for a pet pelican"
|
||||
|
||||
# Extract text from an image
|
||||
llm "extract text" -a scanned-document.jpg
|
||||
|
||||
# Use a system prompt against a file
|
||||
cat myfile.py | llm -s "Explain this code"
|
||||
```
|
||||
|
||||
Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic](https://console.anthropic.com/) with their respective plugins:
|
||||
|
||||
```bash
|
||||
llm install llm-gemini
|
||||
llm keys set gemini
|
||||
# Paste Gemini API key here
|
||||
llm -m gemini-2.0-flash 'Tell me fun facts about Mountain View'
|
||||
|
||||
llm install llm-anthropic
|
||||
llm keys set anthropic
|
||||
# Paste Anthropic API key here
|
||||
llm -m claude-4-opus 'Impress me with wild facts about turnips'
|
||||
```
|
||||
|
||||
You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/):
|
||||
|
||||
```bash
|
||||
# Install the plugin
|
||||
llm install llm-ollama
|
||||
|
||||
# Download and run a prompt against the Orca Mini 7B model
|
||||
ollama pull llama3.2:latest
|
||||
llm -m llama3.2:latest 'What is the capital of France?'
|
||||
```
|
||||
|
||||
To start [an interactive chat](https://llm.datasette.io/en/stable/usage.html#usage-chat) with a model, use `llm chat`:
|
||||
|
||||
```bash
|
||||
llm chat -m gpt-4.1
|
||||
```
|
||||
|
||||
```default
|
||||
Chatting with gpt-4.1
|
||||
Type 'exit' or 'quit' to exit
|
||||
Type '!multi' to enter multiple lines, then '!end' to finish
|
||||
Type '!edit' to open your default editor and modify the prompt.
|
||||
Type '!fragment <my_fragment> [<another_fragment> ...]' to insert one or more fragments
|
||||
> Tell me a joke about a pelican
|
||||
Why don't pelicans like to tip waiters?
|
||||
|
||||
Because they always have a big bill!
|
||||
```
|
||||
|
||||
More background on this project:
|
||||
|
||||
- [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/)
|
||||
- [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/)
|
||||
- [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/)
|
||||
- [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/)
|
||||
- [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/)
|
||||
- [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/)
|
||||
- [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/)
|
||||
|
||||
See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog.
|
||||
|
||||
## Contents
|
||||
|
||||
* [Setup](https://llm.datasette.io/en/stable/setup.html)
|
||||
* [Installation](https://llm.datasette.io/en/stable/setup.html#installation)
|
||||
* [Upgrading to the latest version](https://llm.datasette.io/en/stable/setup.html#upgrading-to-the-latest-version)
|
||||
* [Using uvx](https://llm.datasette.io/en/stable/setup.html#using-uvx)
|
||||
* [A note about Homebrew and PyTorch](https://llm.datasette.io/en/stable/setup.html#a-note-about-homebrew-and-pytorch)
|
||||
* [Installing plugins](https://llm.datasette.io/en/stable/setup.html#installing-plugins)
|
||||
* [API key management](https://llm.datasette.io/en/stable/setup.html#api-key-management)
|
||||
* [Saving and using stored keys](https://llm.datasette.io/en/stable/setup.html#saving-and-using-stored-keys)
|
||||
* [Passing keys using the –key option](https://llm.datasette.io/en/stable/setup.html#passing-keys-using-the-key-option)
|
||||
* [Keys in environment variables](https://llm.datasette.io/en/stable/setup.html#keys-in-environment-variables)
|
||||
* [Configuration](https://llm.datasette.io/en/stable/setup.html#configuration)
|
||||
* [Setting a custom default model](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-default-model)
|
||||
* [Setting a custom directory location](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-directory-location)
|
||||
* [Turning SQLite logging on and off](https://llm.datasette.io/en/stable/setup.html#turning-sqlite-logging-on-and-off)
|
||||
* [Usage](https://llm.datasette.io/en/stable/usage.html)
|
||||
* [Executing a prompt](https://llm.datasette.io/en/stable/usage.html#executing-a-prompt)
|
||||
* [Model options](https://llm.datasette.io/en/stable/usage.html#model-options)
|
||||
* [Attachments](https://llm.datasette.io/en/stable/usage.html#attachments)
|
||||
* [System prompts](https://llm.datasette.io/en/stable/usage.html#system-prompts)
|
||||
* [Tools](https://llm.datasette.io/en/stable/usage.html#tools)
|
||||
* [Extracting fenced code blocks](https://llm.datasette.io/en/stable/usage.html#extracting-fenced-code-blocks)
|
||||
* [Schemas](https://llm.datasette.io/en/stable/usage.html#schemas)
|
||||
* [Fragments](https://llm.datasette.io/en/stable/usage.html#fragments)
|
||||
* [Continuing a conversation](https://llm.datasette.io/en/stable/usage.html#continuing-a-conversation)
|
||||
* [Tips for using LLM with Bash or Zsh](https://llm.datasette.io/en/stable/usage.html#tips-for-using-llm-with-bash-or-zsh)
|
||||
* [Completion prompts](https://llm.datasette.io/en/stable/usage.html#completion-prompts)
|
||||
* [Starting an interactive chat](https://llm.datasette.io/en/stable/usage.html#starting-an-interactive-chat)
|
||||
* [Listing available models](https://llm.datasette.io/en/stable/usage.html#listing-available-models)
|
||||
* [Setting default options for models](https://llm.datasette.io/en/stable/usage.html#setting-default-options-for-models)
|
||||
* [OpenAI models](https://llm.datasette.io/en/stable/openai-models.html)
|
||||
* [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration)
|
||||
* [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models)
|
||||
* [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features)
|
||||
* [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models)
|
||||
* [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models)
|
||||
* [Adding more OpenAI models](https://llm.datasette.io/en/stable/openai-models.html#adding-more-openai-models)
|
||||
* [Other models](https://llm.datasette.io/en/stable/other-models.html)
|
||||
* [Installing and using a local model](https://llm.datasette.io/en/stable/other-models.html#installing-and-using-a-local-model)
|
||||
* [OpenAI-compatible models](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models)
|
||||
* [Extra HTTP headers](https://llm.datasette.io/en/stable/other-models.html#extra-http-headers)
|
||||
* [Tools](https://llm.datasette.io/en/stable/tools.html)
|
||||
* [How tools work](https://llm.datasette.io/en/stable/tools.html#how-tools-work)
|
||||
* [Trying out tools](https://llm.datasette.io/en/stable/tools.html#trying-out-tools)
|
||||
* [LLM’s implementation of tools](https://llm.datasette.io/en/stable/tools.html#llm-s-implementation-of-tools)
|
||||
* [Default tools](https://llm.datasette.io/en/stable/tools.html#default-tools)
|
||||
* [Tips for implementing tools](https://llm.datasette.io/en/stable/tools.html#tips-for-implementing-tools)
|
||||
* [Schemas](https://llm.datasette.io/en/stable/schemas.html)
|
||||
* [Schemas tutorial](https://llm.datasette.io/en/stable/schemas.html#schemas-tutorial)
|
||||
* [Getting started with dogs](https://llm.datasette.io/en/stable/schemas.html#getting-started-with-dogs)
|
||||
* [Extracting people from a news articles](https://llm.datasette.io/en/stable/schemas.html#extracting-people-from-a-news-articles)
|
||||
* [Using JSON schemas](https://llm.datasette.io/en/stable/schemas.html#using-json-schemas)
|
||||
* [Ways to specify a schema](https://llm.datasette.io/en/stable/schemas.html#ways-to-specify-a-schema)
|
||||
* [Concise LLM schema syntax](https://llm.datasette.io/en/stable/schemas.html#concise-llm-schema-syntax)
|
||||
* [Saving reusable schemas in templates](https://llm.datasette.io/en/stable/schemas.html#saving-reusable-schemas-in-templates)
|
||||
* [Browsing logged JSON objects created using schemas](https://llm.datasette.io/en/stable/schemas.html#browsing-logged-json-objects-created-using-schemas)
|
||||
* [Templates](https://llm.datasette.io/en/stable/templates.html)
|
||||
* [Getting started with <code>–save</code>](https://llm.datasette.io/en/stable/templates.html#getting-started-with-save)
|
||||
* [Using a template](https://llm.datasette.io/en/stable/templates.html#using-a-template)
|
||||
* [Listing available templates](https://llm.datasette.io/en/stable/templates.html#listing-available-templates)
|
||||
* [Templates as YAML files](https://llm.datasette.io/en/stable/templates.html#templates-as-yaml-files)
|
||||
* [System prompts](https://llm.datasette.io/en/stable/templates.html#system-prompts)
|
||||
* [Fragments](https://llm.datasette.io/en/stable/templates.html#fragments)
|
||||
* [Options](https://llm.datasette.io/en/stable/templates.html#options)
|
||||
* [Tools](https://llm.datasette.io/en/stable/templates.html#tools)
|
||||
* [Schemas](https://llm.datasette.io/en/stable/templates.html#schemas)
|
||||
* [Additional template variables](https://llm.datasette.io/en/stable/templates.html#additional-template-variables)
|
||||
* [Specifying default parameters](https://llm.datasette.io/en/stable/templates.html#specifying-default-parameters)
|
||||
* [Configuring code extraction](https://llm.datasette.io/en/stable/templates.html#configuring-code-extraction)
|
||||
* [Setting a default model for a template](https://llm.datasette.io/en/stable/templates.html#setting-a-default-model-for-a-template)
|
||||
* [Template loaders from plugins](https://llm.datasette.io/en/stable/templates.html#template-loaders-from-plugins)
|
||||
* [Fragments](https://llm.datasette.io/en/stable/fragments.html)
|
||||
* [Using fragments in a prompt](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-a-prompt)
|
||||
* [Using fragments in chat](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-chat)
|
||||
* [Browsing fragments](https://llm.datasette.io/en/stable/fragments.html#browsing-fragments)
|
||||
* [Setting aliases for fragments](https://llm.datasette.io/en/stable/fragments.html#setting-aliases-for-fragments)
|
||||
* [Viewing fragments in your logs](https://llm.datasette.io/en/stable/fragments.html#viewing-fragments-in-your-logs)
|
||||
* [Using fragments from plugins](https://llm.datasette.io/en/stable/fragments.html#using-fragments-from-plugins)
|
||||
* [Listing available fragment prefixes](https://llm.datasette.io/en/stable/fragments.html#listing-available-fragment-prefixes)
|
||||
* [Model aliases](https://llm.datasette.io/en/stable/aliases.html)
|
||||
* [Listing aliases](https://llm.datasette.io/en/stable/aliases.html#listing-aliases)
|
||||
* [Adding a new alias](https://llm.datasette.io/en/stable/aliases.html#adding-a-new-alias)
|
||||
* [Removing an alias](https://llm.datasette.io/en/stable/aliases.html#removing-an-alias)
|
||||
* [Viewing the aliases file](https://llm.datasette.io/en/stable/aliases.html#viewing-the-aliases-file)
|
||||
* [Embeddings](https://llm.datasette.io/en/stable/embeddings/index.html)
|
||||
* [Embedding with the CLI](https://llm.datasette.io/en/stable/embeddings/cli.html)
|
||||
* [llm embed](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed)
|
||||
* [llm embed-multi](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-multi)
|
||||
* [llm similar](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-similar)
|
||||
* [llm embed-models](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-models)
|
||||
* [llm collections list](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-list)
|
||||
* [llm collections delete](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-delete)
|
||||
* [Using embeddings from Python](https://llm.datasette.io/en/stable/embeddings/python-api.html)
|
||||
* [Working with collections](https://llm.datasette.io/en/stable/embeddings/python-api.html#working-with-collections)
|
||||
* [Retrieving similar items](https://llm.datasette.io/en/stable/embeddings/python-api.html#retrieving-similar-items)
|
||||
* [SQL schema](https://llm.datasette.io/en/stable/embeddings/python-api.html#sql-schema)
|
||||
* [Writing plugins to add new embedding models](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html)
|
||||
* [`EmbeddingModel`](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#llm.EmbeddingModel)
|
||||
* [Embedding binary content](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#embedding-binary-content)
|
||||
* [Embedding storage format](https://llm.datasette.io/en/stable/embeddings/storage.html)
|
||||
* [Plugins](https://llm.datasette.io/en/stable/plugins/index.html)
|
||||
* [Installing plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html)
|
||||
* [Listing installed plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#listing-installed-plugins)
|
||||
* [Running with a subset of plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#running-with-a-subset-of-plugins)
|
||||
* [Plugin directory](https://llm.datasette.io/en/stable/plugins/directory.html)
|
||||
* [Local models](https://llm.datasette.io/en/stable/plugins/directory.html#local-models)
|
||||
* [Remote APIs](https://llm.datasette.io/en/stable/plugins/directory.html#remote-apis)
|
||||
* [Tools](https://llm.datasette.io/en/stable/plugins/directory.html#tools)
|
||||
* [Fragments and template loaders](https://llm.datasette.io/en/stable/plugins/directory.html#fragments-and-template-loaders)
|
||||
* [Embedding models](https://llm.datasette.io/en/stable/plugins/directory.html#embedding-models)
|
||||
* [Extra commands](https://llm.datasette.io/en/stable/plugins/directory.html#extra-commands)
|
||||
* [Just for fun](https://llm.datasette.io/en/stable/plugins/directory.html#just-for-fun)
|
||||
* [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html)
|
||||
* [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli)
|
||||
* [register_models(register, model_aliases)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register-model-aliases)
|
||||
* [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register)
|
||||
* [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register)
|
||||
* [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register)
|
||||
* [register_fragment_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-fragment-loaders-register)
|
||||
* [Developing a model plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html)
|
||||
* [The initial structure of the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#the-initial-structure-of-the-plugin)
|
||||
* [Installing your plugin to try it out](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#installing-your-plugin-to-try-it-out)
|
||||
* [Building the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#building-the-markov-chain)
|
||||
* [Executing the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#executing-the-markov-chain)
|
||||
* [Adding that to the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-that-to-the-plugin)
|
||||
* [Understanding execute()](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#understanding-execute)
|
||||
* [Prompts and responses are logged to the database](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#prompts-and-responses-are-logged-to-the-database)
|
||||
* [Adding options](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-options)
|
||||
* [Distributing your plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#distributing-your-plugin)
|
||||
* [GitHub repositories](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#github-repositories)
|
||||
* [Publishing plugins to PyPI](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#publishing-plugins-to-pypi)
|
||||
* [Adding metadata](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-metadata)
|
||||
* [What to do if it breaks](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#what-to-do-if-it-breaks)
|
||||
* [Advanced model plugins](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html)
|
||||
* [Tip: lazily load expensive dependencies](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tip-lazily-load-expensive-dependencies)
|
||||
* [Models that accept API keys](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#models-that-accept-api-keys)
|
||||
* [Async models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#async-models)
|
||||
* [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas)
|
||||
* [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools)
|
||||
* [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models)
|
||||
* [Structured messages and streaming events](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#structured-messages-and-streaming-events)
|
||||
* [Consuming prompt.messages in build_messages](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#consuming-prompt-messages-in-build-messages)
|
||||
* [Restoring opaque metadata on subsequent requests](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#restoring-opaque-metadata-on-subsequent-requests)
|
||||
* [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage)
|
||||
* [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names)
|
||||
* [LLM_RAISE_ERRORS](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#llm-raise-errors)
|
||||
* [Utility functions for plugins](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html)
|
||||
* [llm.get_key()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-get-key)
|
||||
* [llm.user_dir()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-user-dir)
|
||||
* [llm.ModelError](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-modelerror)
|
||||
* [Response.fake()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#response-fake)
|
||||
* [Python API](https://llm.datasette.io/en/stable/python-api.html)
|
||||
* [Basic prompt execution](https://llm.datasette.io/en/stable/python-api.html#basic-prompt-execution)
|
||||
* [System prompts](https://llm.datasette.io/en/stable/python-api.html#system-prompts)
|
||||
* [Attachments](https://llm.datasette.io/en/stable/python-api.html#attachments)
|
||||
* [Tools](https://llm.datasette.io/en/stable/python-api.html#tools)
|
||||
* [Schemas](https://llm.datasette.io/en/stable/python-api.html#schemas)
|
||||
* [Fragments](https://llm.datasette.io/en/stable/python-api.html#fragments)
|
||||
* [Model options](https://llm.datasette.io/en/stable/python-api.html#model-options)
|
||||
* [Passing an API key](https://llm.datasette.io/en/stable/python-api.html#passing-an-api-key)
|
||||
* [Models from plugins](https://llm.datasette.io/en/stable/python-api.html#models-from-plugins)
|
||||
* [Accessing the underlying JSON](https://llm.datasette.io/en/stable/python-api.html#accessing-the-underlying-json)
|
||||
* [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage)
|
||||
* [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses)
|
||||
* [Structured messages and streaming events](https://llm.datasette.io/en/stable/python-api.html#structured-messages-and-streaming-events)
|
||||
* [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models)
|
||||
* [`AsyncResponse`](https://llm.datasette.io/en/stable/python-api.html#llm.AsyncResponse)
|
||||
* [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async)
|
||||
* [Tool use for async models](https://llm.datasette.io/en/stable/python-api.html#tool-use-for-async-models)
|
||||
* [Conversations](https://llm.datasette.io/en/stable/python-api.html#conversations)
|
||||
* [Conversations using tools](https://llm.datasette.io/en/stable/python-api.html#conversations-using-tools)
|
||||
* [Listing models](https://llm.datasette.io/en/stable/python-api.html#listing-models)
|
||||
* [Running code when a response has completed](https://llm.datasette.io/en/stable/python-api.html#running-code-when-a-response-has-completed)
|
||||
* [Other functions](https://llm.datasette.io/en/stable/python-api.html#other-functions)
|
||||
* [set_alias(alias, model_id)](https://llm.datasette.io/en/stable/python-api.html#set-alias-alias-model-id)
|
||||
* [remove_alias(alias)](https://llm.datasette.io/en/stable/python-api.html#remove-alias-alias)
|
||||
* [set_default_model(alias)](https://llm.datasette.io/en/stable/python-api.html#set-default-model-alias)
|
||||
* [get_default_model()](https://llm.datasette.io/en/stable/python-api.html#get-default-model)
|
||||
* [set_default_embedding_model(alias) and get_default_embedding_model()](https://llm.datasette.io/en/stable/python-api.html#set-default-embedding-model-alias-and-get-default-embedding-model)
|
||||
* [Logging to SQLite](https://llm.datasette.io/en/stable/logging.html)
|
||||
* [Viewing the logs](https://llm.datasette.io/en/stable/logging.html#viewing-the-logs)
|
||||
* [-s/–short mode](https://llm.datasette.io/en/stable/logging.html#s-short-mode)
|
||||
* [Logs for a conversation](https://llm.datasette.io/en/stable/logging.html#logs-for-a-conversation)
|
||||
* [Searching the logs](https://llm.datasette.io/en/stable/logging.html#searching-the-logs)
|
||||
* [Filtering past a specific ID](https://llm.datasette.io/en/stable/logging.html#filtering-past-a-specific-id)
|
||||
* [Filtering by model](https://llm.datasette.io/en/stable/logging.html#filtering-by-model)
|
||||
* [Filtering by prompts that used specific fragments](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-fragments)
|
||||
* [Filtering by prompts that used specific tools](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-tools)
|
||||
* [Browsing data collected using schemas](https://llm.datasette.io/en/stable/logging.html#browsing-data-collected-using-schemas)
|
||||
* [Browsing logs using Datasette](https://llm.datasette.io/en/stable/logging.html#browsing-logs-using-datasette)
|
||||
* [Backing up your database](https://llm.datasette.io/en/stable/logging.html#backing-up-your-database)
|
||||
* [SQL schema](https://llm.datasette.io/en/stable/logging.html#sql-schema)
|
||||
* [Related tools](https://llm.datasette.io/en/stable/related-tools.html)
|
||||
* [strip-tags](https://llm.datasette.io/en/stable/related-tools.html#strip-tags)
|
||||
* [ttok](https://llm.datasette.io/en/stable/related-tools.html#ttok)
|
||||
* [Symbex](https://llm.datasette.io/en/stable/related-tools.html#symbex)
|
||||
* [CLI reference](https://llm.datasette.io/en/stable/help.html)
|
||||
* [llm –help](https://llm.datasette.io/en/stable/help.html#llm-help)
|
||||
* [llm prompt –help](https://llm.datasette.io/en/stable/help.html#llm-prompt-help)
|
||||
* [llm chat –help](https://llm.datasette.io/en/stable/help.html#llm-chat-help)
|
||||
* [llm keys –help](https://llm.datasette.io/en/stable/help.html#llm-keys-help)
|
||||
* [llm logs –help](https://llm.datasette.io/en/stable/help.html#llm-logs-help)
|
||||
* [llm models –help](https://llm.datasette.io/en/stable/help.html#llm-models-help)
|
||||
* [llm templates –help](https://llm.datasette.io/en/stable/help.html#llm-templates-help)
|
||||
* [llm schemas –help](https://llm.datasette.io/en/stable/help.html#llm-schemas-help)
|
||||
* [llm tools –help](https://llm.datasette.io/en/stable/help.html#llm-tools-help)
|
||||
* [llm aliases –help](https://llm.datasette.io/en/stable/help.html#llm-aliases-help)
|
||||
* [llm fragments –help](https://llm.datasette.io/en/stable/help.html#llm-fragments-help)
|
||||
* [llm plugins –help](https://llm.datasette.io/en/stable/help.html#llm-plugins-help)
|
||||
* [llm install –help](https://llm.datasette.io/en/stable/help.html#llm-install-help)
|
||||
* [llm uninstall –help](https://llm.datasette.io/en/stable/help.html#llm-uninstall-help)
|
||||
* [llm embed –help](https://llm.datasette.io/en/stable/help.html#llm-embed-help)
|
||||
* [llm embed-multi –help](https://llm.datasette.io/en/stable/help.html#llm-embed-multi-help)
|
||||
* [llm similar –help](https://llm.datasette.io/en/stable/help.html#llm-similar-help)
|
||||
* [llm embed-models –help](https://llm.datasette.io/en/stable/help.html#llm-embed-models-help)
|
||||
* [llm collections –help](https://llm.datasette.io/en/stable/help.html#llm-collections-help)
|
||||
* [llm openai –help](https://llm.datasette.io/en/stable/help.html#llm-openai-help)
|
||||
* [Contributing](https://llm.datasette.io/en/stable/contributing.html)
|
||||
* [Updating recorded HTTP API interactions and associated snapshots](https://llm.datasette.io/en/stable/contributing.html#updating-recorded-http-api-interactions-and-associated-snapshots)
|
||||
* [Debugging tricks](https://llm.datasette.io/en/stable/contributing.html#debugging-tricks)
|
||||
* [Documentation](https://llm.datasette.io/en/stable/contributing.html#documentation)
|
||||
* [Release process](https://llm.datasette.io/en/stable/contributing.html#release-process)
|
||||
|
||||
* [Changelog](https://llm.datasette.io/en/stable/changelog.html)
|
||||
<!-- [[[end]]] -->
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`simonw/llm`
|
||||
- 原始仓库：https://github.com/simonw/llm
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
@@ -0,0 +1 @@
|
||||
_build
|
||||
@@ -0,0 +1,23 @@
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line.
|
||||
SPHINXOPTS =
|
||||
SPHINXBUILD = sphinx-build
|
||||
SPHINXPROJ = sqlite-utils
|
||||
SOURCEDIR = .
|
||||
BUILDDIR = _build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
livehtml:
|
||||
sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0)
|
||||
Vendored
+16
@@ -0,0 +1,16 @@
|
||||
{%- extends "!base.html" %}
|
||||
|
||||
{%- block htmltitle -%}
|
||||
{% if not docstitle %}
|
||||
<title>{{ title|striptags|e }}</title>
|
||||
{% elif pagename == master_doc %}
|
||||
<title>LLM: A CLI utility and Python library for interacting with Large Language Models</title>
|
||||
{% else %}
|
||||
<title>{{ title|striptags|e }} - {{ docstitle|striptags|e }}</title>
|
||||
{% endif %}
|
||||
{%- endblock -%}
|
||||
|
||||
{% block site_meta %}
|
||||
{{ super() }}
|
||||
<script defer data-domain="llm.datasette.io" src="https://plausible.io/js/plausible.js"></script>
|
||||
{% endblock %}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
(aliases)=
|
||||
# Model aliases
|
||||
|
||||
LLM supports model aliases, which allow you to refer to a model by a short name instead of its full ID.
|
||||
|
||||
## Listing aliases
|
||||
|
||||
To list current aliases, run this:
|
||||
|
||||
```bash
|
||||
llm aliases
|
||||
```
|
||||
Example output:
|
||||
|
||||
<!-- [[[cog
|
||||
from click.testing import CliRunner
|
||||
from llm.cli import cli
|
||||
result = CliRunner().invoke(cli, ["aliases", "list"])
|
||||
cog.out("```\n{}```".format(result.output))
|
||||
]]] -->
|
||||
```
|
||||
4o : gpt-4o
|
||||
chatgpt-4o : chatgpt-4o-latest
|
||||
4o-mini : gpt-4o-mini
|
||||
4.1 : gpt-4.1
|
||||
4.1-mini : gpt-4.1-mini
|
||||
4.1-nano : gpt-4.1-nano
|
||||
3.5 : gpt-3.5-turbo
|
||||
chatgpt : gpt-3.5-turbo
|
||||
chatgpt-16k : gpt-3.5-turbo-16k
|
||||
3.5-16k : gpt-3.5-turbo-16k
|
||||
4 : gpt-4
|
||||
gpt4 : gpt-4
|
||||
4-32k : gpt-4-32k
|
||||
gpt-4-turbo-preview : gpt-4-turbo
|
||||
4-turbo : gpt-4-turbo
|
||||
4t : gpt-4-turbo
|
||||
gpt-4.5 : gpt-4.5-preview
|
||||
3.5-instruct : gpt-3.5-turbo-instruct
|
||||
chatgpt-instruct : gpt-3.5-turbo-instruct
|
||||
ada : text-embedding-ada-002 (embedding)
|
||||
ada-002 : text-embedding-ada-002 (embedding)
|
||||
3-small : text-embedding-3-small (embedding)
|
||||
3-large : text-embedding-3-large (embedding)
|
||||
3-small-512 : text-embedding-3-small-512 (embedding)
|
||||
3-large-256 : text-embedding-3-large-256 (embedding)
|
||||
3-large-1024 : text-embedding-3-large-1024 (embedding)
|
||||
```
|
||||
<!-- [[[end]]] -->
|
||||
|
||||
Add `--json` to get that list back as JSON:
|
||||
|
||||
```bash
|
||||
llm aliases list --json
|
||||
```
|
||||
Example output:
|
||||
```json
|
||||
{
|
||||
"3.5": "gpt-3.5-turbo",
|
||||
"chatgpt": "gpt-3.5-turbo",
|
||||
"4": "gpt-4",
|
||||
"gpt4": "gpt-4",
|
||||
"ada": "ada-002"
|
||||
}
|
||||
```
|
||||
|
||||
## Adding a new alias
|
||||
|
||||
The `llm aliases set <alias> <model-id>` command can be used to add a new alias:
|
||||
|
||||
```bash
|
||||
llm aliases set mini gpt-4o-mini
|
||||
```
|
||||
You can also pass one or more `-q search` options to set an alias on the first model matching those search terms:
|
||||
```bash
|
||||
llm aliases set mini -q 4o -q mini
|
||||
```
|
||||
Now you can run the `gpt-4o-mini` model using the `mini` alias like this:
|
||||
```bash
|
||||
llm -m mini 'An epic Greek-style saga about a cheesecake that builds a SQL database from scratch'
|
||||
```
|
||||
Aliases can be set for both regular models and {ref}`embedding models <embeddings>` using the same command. To set an alias of `oai` for the OpenAI `ada-002` embedding model use this:
|
||||
```bash
|
||||
llm aliases set oai ada-002
|
||||
```
|
||||
Now you can embed a string using that model like so:
|
||||
```bash
|
||||
llm embed -c 'hello world' -m oai
|
||||
```
|
||||
Output:
|
||||
```
|
||||
[-0.014945968054234982, 0.0014304015785455704, ...]
|
||||
```
|
||||
|
||||
## Removing an alias
|
||||
|
||||
The `llm aliases remove <alias>` command will remove the specified alias:
|
||||
|
||||
```bash
|
||||
llm aliases remove mini
|
||||
```
|
||||
|
||||
## Viewing the aliases file
|
||||
|
||||
Aliases are stored in an `aliases.json` file in the LLM configuration directory.
|
||||
|
||||
To see the path to that file, run this:
|
||||
|
||||
```bash
|
||||
llm aliases path
|
||||
```
|
||||
To view the content of that file, run this:
|
||||
|
||||
```bash
|
||||
cat "$(llm aliases path)"
|
||||
```
|
||||
+1029
File diff suppressed because it is too large
Load Diff
+182
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from subprocess import PIPE, Popen
|
||||
|
||||
# This file is execfile()d with the current directory set to its
|
||||
# containing dir.
|
||||
#
|
||||
# Note that not all possible configuration values are present in this
|
||||
# autogenerated file.
|
||||
#
|
||||
# All configuration values have a default; values that are commented out
|
||||
# serve to show the default.
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#
|
||||
# import os
|
||||
# import sys
|
||||
# sys.path.insert(0, os.path.abspath('.'))
|
||||
|
||||
|
||||
# -- General configuration ------------------------------------------------
|
||||
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
#
|
||||
# needs_sphinx = '1.0'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = [
|
||||
"myst_parser",
|
||||
"sphinx_copybutton",
|
||||
"sphinx_markdown_builder",
|
||||
"sphinx.ext.autodoc",
|
||||
]
|
||||
myst_enable_extensions = ["colon_fence"]
|
||||
|
||||
markdown_http_base = "https://llm.datasette.io/en/stable"
|
||||
markdown_uri_doc_suffix = ".html"
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ["_templates"]
|
||||
|
||||
# The suffix(es) of source filenames.
|
||||
# You can specify multiple suffix as a list of string:
|
||||
#
|
||||
# source_suffix = ['.rst', '.md']
|
||||
source_suffix = ".rst"
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = "index"
|
||||
|
||||
# General information about the project.
|
||||
project = "LLM"
|
||||
copyright = "2025, Simon Willison"
|
||||
author = "Simon Willison"
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True)
|
||||
git_version = pipe.stdout.read().decode("utf8")
|
||||
|
||||
if git_version:
|
||||
version = git_version.rsplit("-", 1)[0]
|
||||
release = git_version
|
||||
else:
|
||||
version = ""
|
||||
release = ""
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#
|
||||
# This is also used if you do content translation via gettext catalogs.
|
||||
# Usually you set "language" from the command line for these cases.
|
||||
language = "en"
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This patterns also effect to html_static_path and html_extra_path
|
||||
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = "sphinx"
|
||||
|
||||
# If true, `todo` and `todoList` produce output, else they produce nothing.
|
||||
todo_include_todos = False
|
||||
|
||||
|
||||
# -- Options for HTML output ----------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#
|
||||
html_theme = "furo"
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
|
||||
html_theme_options = {}
|
||||
html_title = "LLM"
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = []
|
||||
|
||||
|
||||
# -- Options for HTMLHelp output ------------------------------------------
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = "llm-doc"
|
||||
|
||||
|
||||
# -- Options for LaTeX output ---------------------------------------------
|
||||
|
||||
latex_elements = {
|
||||
# The paper size ('letterpaper' or 'a4paper').
|
||||
#
|
||||
# 'papersize': 'letterpaper',
|
||||
# The font size ('10pt', '11pt' or '12pt').
|
||||
#
|
||||
# 'pointsize': '10pt',
|
||||
# Additional stuff for the LaTeX preamble.
|
||||
#
|
||||
# 'preamble': '',
|
||||
# Latex figure (float) alignment
|
||||
#
|
||||
# 'figure_align': 'htbp',
|
||||
}
|
||||
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title,
|
||||
# author, documentclass [howto, manual, or own class]).
|
||||
latex_documents = [
|
||||
(
|
||||
master_doc,
|
||||
"llm.tex",
|
||||
"LLM documentation",
|
||||
"Simon Willison",
|
||||
"manual",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
# -- Options for manual page output ---------------------------------------
|
||||
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
(
|
||||
master_doc,
|
||||
"llm",
|
||||
"LLM documentation",
|
||||
[author],
|
||||
1,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
# -- Options for Texinfo output -------------------------------------------
|
||||
|
||||
# Grouping the document tree into Texinfo files. List of tuples
|
||||
# (source start file, target name, title, author,
|
||||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
(
|
||||
master_doc,
|
||||
"llm",
|
||||
"LLM documentation",
|
||||
author,
|
||||
"llm",
|
||||
" Access large language models from the command-line ",
|
||||
"Miscellaneous",
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,66 @@
|
||||
(contributing)=
|
||||
# Contributing
|
||||
|
||||
To contribute to this tool, first checkout the code. Then run the tests with `uv run`:
|
||||
```bash
|
||||
cd llm
|
||||
uv run pytest
|
||||
```
|
||||
You can run your development copy of `llm` using `uv run` as well:
|
||||
```bash
|
||||
uv run llm --help
|
||||
```
|
||||
|
||||
## Updating recorded HTTP API interactions and associated snapshots
|
||||
|
||||
This project uses [pytest-recording](https://github.com/kiwicom/pytest-recording) to record OpenAI API responses for some of the tests, and [syrupy](https://github.com/syrupy-project/syrupy) to capture snapshots of their results.
|
||||
|
||||
If you add a new test that calls the API you can capture the API response and snapshot like this:
|
||||
```bash
|
||||
PYTEST_OPENAI_API_KEY="$(llm keys get openai)" uv run pytest --record-mode once --snapshot-update
|
||||
```
|
||||
Then review the new snapshots in `tests/__snapshots__/` to make sure they look correct.
|
||||
|
||||
## Debugging tricks
|
||||
|
||||
The default OpenAI plugin has a debugging mechanism for showing the exact requests and responses that were sent to the OpenAI API.
|
||||
|
||||
Set the `LLM_OPENAI_SHOW_RESPONSES` environment variable like this:
|
||||
```bash
|
||||
LLM_OPENAI_SHOW_RESPONSES=1 uv run llm -m chatgpt 'three word slogan for an otter-run bakery'
|
||||
```
|
||||
This will output details of the API requests and responses to the console.
|
||||
|
||||
Use `--no-stream` to see a more readable version of the body that avoids streaming the response:
|
||||
|
||||
```bash
|
||||
LLM_OPENAI_SHOW_RESPONSES=1 uv run llm -m chatgpt --no-stream \
|
||||
'three word slogan for an otter-run bakery'
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
Documentation for this project uses [MyST](https://myst-parser.readthedocs.io/) - it is written in Markdown and rendered using Sphinx.
|
||||
|
||||
To build the documentation locally, run the following:
|
||||
```bash
|
||||
just docs
|
||||
```
|
||||
This will start a live preview server, using [sphinx-autobuild](https://pypi.org/project/sphinx-autobuild/).
|
||||
|
||||
The CLI `--help` examples in the documentation are managed using [Cog](https://github.com/nedbat/cog). Update those files like this:
|
||||
```bash
|
||||
just cog
|
||||
```
|
||||
You'll need [Just](https://github.com/casey/just) installed to run these commands.
|
||||
|
||||
## Release process
|
||||
|
||||
To release a new version:
|
||||
|
||||
1. Update `docs/changelog.md` with the new changes.
|
||||
2. Update the version number in `pyproject.toml`
|
||||
3. Run `just cog` to update `docs/fragments.md` with the new version number.
|
||||
4. [Create a GitHub release](https://github.com/simonw/llm/releases/new) for the new version.
|
||||
5. Wait for the package to push to PyPI and then...
|
||||
6. Run the [regenerate.yaml](https://github.com/simonw/homebrew-llm/actions/workflows/regenerate.yaml) workflow to update the Homebrew tap to the latest version.
|
||||
@@ -0,0 +1,443 @@
|
||||
(embeddings-cli)=
|
||||
# Embedding with the CLI
|
||||
|
||||
LLM provides command-line utilities for calculating and storing embeddings for pieces of content.
|
||||
|
||||
(embeddings-cli-embed)=
|
||||
## llm embed
|
||||
|
||||
The `llm embed` command can be used to calculate embedding vectors for a string of content. These can be returned directly to the terminal, stored in a SQLite database, or both.
|
||||
|
||||
### Returning embeddings to the terminal
|
||||
|
||||
The simplest way to use this command is to pass content to it using the `-c/--content` option, like this:
|
||||
|
||||
```bash
|
||||
llm embed -c 'This is some content' -m 3-small
|
||||
```
|
||||
`-m 3-small` specifies the OpenAI `text-embedding-3-small` model. You will need to have set an OpenAI API key using `llm keys set openai` for this to work.
|
||||
|
||||
You can install plugins to access other models. The [llm-sentence-transformers](https://github.com/simonw/llm-sentence-transformers) plugin can be used to run models on your own laptop, such as the [MiniLM-L6](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) model:
|
||||
|
||||
```bash
|
||||
llm install llm-sentence-transformers
|
||||
llm embed -c 'This is some content' -m sentence-transformers/all-MiniLM-L6-v2
|
||||
```
|
||||
|
||||
The `llm embed` command returns a JSON array of floating point numbers directly to the terminal:
|
||||
|
||||
```json
|
||||
[0.123, 0.456, 0.789...]
|
||||
```
|
||||
You can omit the `-m/--model` option if you set a {ref}`default embedding model <embeddings-cli-embed-models-default>`.
|
||||
|
||||
You can also set the `LLM_EMBEDDING_MODEL` environment variable to set a default model for all `llm embed` commands in the current shell session:
|
||||
|
||||
```bash
|
||||
export LLM_EMBEDDING_MODEL=3-small
|
||||
llm embed -c 'This is some content'
|
||||
```
|
||||
|
||||
LLM also offers a binary storage format for embeddings, described in {ref}`embeddings storage format <embeddings-storage>`.
|
||||
|
||||
You can output embeddings using that format as raw bytes using `--format blob`, or in hexadecimal using `--format hex`, or in Base64 using `--format base64`:
|
||||
|
||||
```bash
|
||||
llm embed -c 'This is some content' -m 3-small --format base64
|
||||
```
|
||||
This outputs:
|
||||
```
|
||||
8NGzPFtdgTqHcZw7aUT6u+++WrwwpZo8XbSxv...
|
||||
```
|
||||
Some models such as [llm-clip](https://github.com/simonw/llm-clip) can run against binary data. You can pass in binary data using the `-i` and `--binary` options:
|
||||
|
||||
```bash
|
||||
llm embed --binary -m clip -i image.jpg
|
||||
```
|
||||
Or from standard input like this:
|
||||
```bash
|
||||
cat image.jpg | llm embed --binary -m clip -i -
|
||||
```
|
||||
|
||||
(embeddings-collections)=
|
||||
### Storing embeddings in SQLite
|
||||
|
||||
Embeddings are much more useful if you store them somewhere, so you can calculate similarity scores between different embeddings later on.
|
||||
|
||||
LLM includes the concept of a **collection** of embeddings. A collection groups together a set of stored embeddings created using the same model, each with a unique ID within that collection.
|
||||
|
||||
Embeddings also store a hash of the content that was embedded. This hash is later used to avoid calculating duplicate embeddings for the same content.
|
||||
|
||||
First, we'll set a default model so we don't have to keep repeating it:
|
||||
```bash
|
||||
llm embed-models default 3-small
|
||||
```
|
||||
|
||||
The `llm embed` command can store results directly in a named collection like this:
|
||||
|
||||
```bash
|
||||
llm embed quotations philkarlton-1 -c \
|
||||
'There are only two hard things in Computer Science: cache invalidation and naming things'
|
||||
```
|
||||
This stores the given text in the `quotations` collection under the key `philkarlton-1`.
|
||||
|
||||
You can also pipe content to standard input, like this:
|
||||
```bash
|
||||
cat one.txt | llm embed files one
|
||||
```
|
||||
This will store the embedding for the contents of `one.txt` in the `files` collection under the key `one`.
|
||||
|
||||
A collection will be created the first time you mention it.
|
||||
|
||||
Collections have a fixed embedding model, which is the model that was used for the first embedding stored in that collection.
|
||||
|
||||
In the above example this would have been the default embedding model at the time that the command was run.
|
||||
|
||||
The following example stores the embedding for the string "my happy hound" in a collection called `phrases` under the key `hound` and using the model `3-small`:
|
||||
|
||||
```bash
|
||||
llm embed phrases hound -m 3-small -c 'my happy hound'
|
||||
```
|
||||
By default, the SQLite database used to store embeddings is the `embeddings.db` in the user content directory managed by LLM.
|
||||
|
||||
You can see the path to this directory by running `llm collections path`.
|
||||
|
||||
You can store embeddings in a different SQLite database by passing a path to it using the `-d/--database` option to `llm embed`. If this file does not exist yet the command will create it:
|
||||
|
||||
```bash
|
||||
llm embed phrases hound -d my-embeddings.db -c 'my happy hound'
|
||||
```
|
||||
This creates a database file called `my-embeddings.db` in the current directory.
|
||||
|
||||
(embeddings-collections-content-metadata)=
|
||||
#### Storing content and metadata
|
||||
|
||||
By default, only the entry ID and the embedding vector are stored in the database table.
|
||||
|
||||
You can store a copy of the original text in the `content` column by passing the `--store` option:
|
||||
|
||||
```bash
|
||||
llm embed phrases hound -c 'my happy hound' --store
|
||||
```
|
||||
You can also store a JSON object containing arbitrary metadata in the `metadata` column by passing the `--metadata` option. This example uses both `--store` and `--metadata` options:
|
||||
|
||||
```bash
|
||||
llm embed phrases hound \
|
||||
-m 3-small \
|
||||
-c 'my happy hound' \
|
||||
--metadata '{"name": "Hound"}' \
|
||||
--store
|
||||
```
|
||||
Data stored in this way will be returned by calls to `llm similar`, for example:
|
||||
```bash
|
||||
llm similar phrases -c 'hound'
|
||||
```
|
||||
```
|
||||
{"id": "hound", "score": 0.8484683588631485, "content": "my happy hound", "metadata": {"name": "Hound"}}
|
||||
```
|
||||
|
||||
(embeddings-cli-embed-multi)=
|
||||
## llm embed-multi
|
||||
|
||||
The `llm embed` command embeds a single string at a time.
|
||||
|
||||
`llm embed-multi` can be used to embed multiple strings at once, taking advantage of any efficiencies that the embedding model may provide when processing multiple strings.
|
||||
|
||||
This command can be called in one of three ways:
|
||||
|
||||
1. With a CSV, TSV, JSON or newline-delimited JSON file
|
||||
2. With a SQLite database and a SQL query
|
||||
3. With one or more paths to directories, each accompanied by a glob pattern
|
||||
|
||||
All three mechanisms support these options:
|
||||
|
||||
- `-m model_id` to specify the embedding model to use
|
||||
- `-d database.db` to specify a different database file to store the embeddings in
|
||||
- `--store` to store the original content in the embeddings table in addition to the embedding vector
|
||||
- `--prefix` to prepend a prefix to the stored ID of each item
|
||||
- `--prepend` to prepend a string to the content before embedding
|
||||
- `--batch-size SIZE` to process embeddings in batches of the specified size
|
||||
|
||||
The `--prepend` option is useful for embedding models that require you to prepend a special token to the content before embedding it. [nomic-embed-text-v2-moe](https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe) for example requires documents to be prepended `'search_document: '` and search queries to be prepended `'search_query: '`.
|
||||
|
||||
(embeddings-cli-embed-multi-csv-etc)=
|
||||
### Embedding data from a CSV, TSV or JSON file
|
||||
|
||||
You can embed data from a CSV, TSV or JSON file by passing that file to the command as the second option, after the collection name.
|
||||
|
||||
Your file must contain at least two columns. The first one is expected to contain the ID of the item, and any subsequent columns will be treated as containing content to be embedded.
|
||||
|
||||
An example CSV file might look like this:
|
||||
|
||||
```
|
||||
id,content
|
||||
one,This is the first item
|
||||
two,This is the second item
|
||||
```
|
||||
TSV would use tabs instead of commas.
|
||||
|
||||
JSON files can be structured like this:
|
||||
|
||||
```json
|
||||
[
|
||||
{"id": "one", "content": "This is the first item"},
|
||||
{"id": "two", "content": "This is the second item"}
|
||||
]
|
||||
```
|
||||
Or as newline-delimited JSON like this:
|
||||
```json
|
||||
{"id": "one", "content": "This is the first item"}
|
||||
{"id": "two", "content": "This is the second item"}
|
||||
```
|
||||
In each of these cases the file can be passed to `llm embed-multi` like this:
|
||||
```bash
|
||||
llm embed-multi items mydata.csv
|
||||
```
|
||||
The first argument is the name of the collection, the second is the filename.
|
||||
|
||||
You can also pipe content to standard input of the tool using `-`:
|
||||
|
||||
```bash
|
||||
cat mydata.json | llm embed-multi items -
|
||||
```
|
||||
LLM will attempt to detect the format of your data automatically. If this doesn't work you can specify the format using the `--format` option. This is required if you are piping newline-delimited JSON to standard input.
|
||||
|
||||
```bash
|
||||
cat mydata.json | llm embed-multi items - --format nl
|
||||
```
|
||||
Other supported `--format` options are `csv`, `tsv` and `json`.
|
||||
|
||||
This example embeds the data from a JSON file in a collection called `items` in database called `docs.db` using the `3-small` model and stores the original content in the `embeddings` table as well, adding a prefix of `my-items/` to each ID:
|
||||
|
||||
```bash
|
||||
llm embed-multi items mydata.json \
|
||||
-d docs.db \
|
||||
-m 3-small \
|
||||
--prefix my-items/ \
|
||||
--store
|
||||
```
|
||||
|
||||
(embeddings-cli-embed-multi-sqlite)=
|
||||
### Embedding data from a SQLite database
|
||||
|
||||
You can embed data from a SQLite database using `--sql`, optionally combined with `--attach` to attach an additional database.
|
||||
|
||||
If you are storing embeddings in the same database as the source data, you can do this:
|
||||
|
||||
```bash
|
||||
llm embed-multi docs \
|
||||
-d docs.db \
|
||||
--sql 'select id, title, content from documents' \
|
||||
-m 3-small
|
||||
```
|
||||
The `docs.db` database here contains a `documents` table, and we want to embed the `title` and `content` columns from that table and store the results back in the same database.
|
||||
|
||||
To load content from a database other than the one you are using to store embeddings, attach it with the `--attach` option and use `alias.table` in your SQLite query:
|
||||
|
||||
```bash
|
||||
llm embed-multi docs \
|
||||
-d embeddings.db \
|
||||
--attach other other.db \
|
||||
--sql 'select id, title, content from other.documents' \
|
||||
-m 3-small
|
||||
```
|
||||
|
||||
(embeddings-cli-embed-multi-directories)=
|
||||
### Embedding data from files in directories
|
||||
|
||||
LLM can embed the content of every text file in a specified directory, using the file's path and name as the ID.
|
||||
|
||||
Consider a directory structure like this:
|
||||
```
|
||||
docs/aliases.md
|
||||
docs/contributing.md
|
||||
docs/embeddings/binary.md
|
||||
docs/embeddings/cli.md
|
||||
docs/embeddings/index.md
|
||||
docs/index.md
|
||||
docs/logging.md
|
||||
docs/plugins/directory.md
|
||||
docs/plugins/index.md
|
||||
```
|
||||
To embed all of those documents, you can run the following:
|
||||
|
||||
```bash
|
||||
llm embed-multi documentation \
|
||||
-m 3-small \
|
||||
--files docs '**/*.md' \
|
||||
-d documentation.db \
|
||||
--store
|
||||
```
|
||||
Here `--files docs '**/*.md'` specifies that the `docs` directory should be scanned for files matching the `**/*.md` glob pattern - which will match Markdown files in any nested directory.
|
||||
|
||||
The result of the above command is a `embeddings` table with the following IDs:
|
||||
|
||||
```
|
||||
aliases.md
|
||||
contributing.md
|
||||
embeddings/binary.md
|
||||
embeddings/cli.md
|
||||
embeddings/index.md
|
||||
index.md
|
||||
logging.md
|
||||
plugins/directory.md
|
||||
plugins/index.md
|
||||
```
|
||||
Each corresponding to embedded content for the file in question.
|
||||
|
||||
The `--prefix` option can be used to add a prefix to each ID:
|
||||
|
||||
```bash
|
||||
llm embed-multi documentation \
|
||||
-m 3-small \
|
||||
--files docs '**/*.md' \
|
||||
-d documentation.db \
|
||||
--store \
|
||||
--prefix llm-docs/
|
||||
```
|
||||
This will result in the following IDs instead:
|
||||
|
||||
```
|
||||
llm-docs/aliases.md
|
||||
llm-docs/contributing.md
|
||||
llm-docs/embeddings/binary.md
|
||||
llm-docs/embeddings/cli.md
|
||||
llm-docs/embeddings/index.md
|
||||
llm-docs/index.md
|
||||
llm-docs/logging.md
|
||||
llm-docs/plugins/directory.md
|
||||
llm-docs/plugins/index.md
|
||||
```
|
||||
Files are assumed to be `utf-8`, but LLM will fall back to `latin-1` if it encounters an encoding error. You can specify a different set of encodings using the `--encoding` option.
|
||||
|
||||
This example will try `utf-16` first and then `mac_roman` before falling back to `latin-1`:
|
||||
```
|
||||
llm embed-multi documentation \
|
||||
-m 3-small \
|
||||
--files docs '**/*.md' \
|
||||
-d documentation.db \
|
||||
--encoding utf-16 \
|
||||
--encoding mac_roman \
|
||||
--encoding latin-1
|
||||
```
|
||||
If a file cannot be read it will be logged to standard error but the script will keep on running.
|
||||
|
||||
If you are embedding binary content such as images for use with CLIP, add the `--binary` option:
|
||||
```
|
||||
llm embed-multi photos \
|
||||
-m clip \
|
||||
--files photos/ '*.jpeg' --binary
|
||||
```
|
||||
|
||||
(embeddings-cli-similar)=
|
||||
## llm similar
|
||||
|
||||
The `llm similar` command searches a collection of embeddings for the items that are most similar to a given or item ID, based on [cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity).
|
||||
|
||||
This currently uses a slow brute-force approach which does not scale well to large collections. See [issue 216](https://github.com/simonw/llm/issues/216) for plans to add a more scalable approach via vector indexes provided by plugins.
|
||||
|
||||
To search the `quotations` collection for items that are semantically similar to `'computer science'`:
|
||||
|
||||
```bash
|
||||
llm similar quotations -c 'computer science'
|
||||
```
|
||||
This embeds the provided string and returns a newline-delimited list of JSON objects like this:
|
||||
```json
|
||||
{"id": "philkarlton-1", "score": 0.8323904531677017, "content": null, "metadata": null}
|
||||
```
|
||||
Use `-p/--plain` to get back results in plain text instead of JSON:
|
||||
```bash
|
||||
llm similar quotations -c 'computer science' -p
|
||||
```
|
||||
Example output:
|
||||
```
|
||||
philkarlton-1 (0.8323904531677017)
|
||||
```
|
||||
You can compare against text stored in a file using `-i filename`:
|
||||
```bash
|
||||
llm similar quotations -i one.txt
|
||||
```
|
||||
Or feed text to standard input using `-i -`:
|
||||
```bash
|
||||
echo 'computer science' | llm similar quotations -i -
|
||||
```
|
||||
When using a model like CLIP, you can find images similar to an input image using `-i filename` with `--binary`:
|
||||
```bash
|
||||
llm similar photos -i image.jpg --binary
|
||||
```
|
||||
|
||||
You can filter results to only show IDs that begin with a specific prefix using --prefix:
|
||||
|
||||
```bash
|
||||
llm similar quotations --prefix 'movies/' -c 'star wars'
|
||||
```
|
||||
|
||||
(embeddings-cli-embed-models)=
|
||||
## llm embed-models
|
||||
|
||||
To list all available embedding models, including those provided by plugins, run this command:
|
||||
|
||||
```bash
|
||||
llm embed-models
|
||||
```
|
||||
The output should look something like this:
|
||||
```
|
||||
OpenAIEmbeddingModel: text-embedding-ada-002 (aliases: ada, ada-002)
|
||||
OpenAIEmbeddingModel: text-embedding-3-small (aliases: 3-small)
|
||||
OpenAIEmbeddingModel: text-embedding-3-large (aliases: 3-large)
|
||||
...
|
||||
```
|
||||
Add `-q` one or more times to search for models matching those terms:
|
||||
```bash
|
||||
llm embed-models -q 3-small
|
||||
```
|
||||
|
||||
(embeddings-cli-embed-models-default)=
|
||||
### llm embed-models default
|
||||
|
||||
This command can be used to get and set the default embedding model.
|
||||
|
||||
This will return the name of the current default model:
|
||||
```bash
|
||||
llm embed-models default
|
||||
```
|
||||
You can set a different default like this:
|
||||
```bash
|
||||
llm embed-models default 3-small
|
||||
```
|
||||
This will set the default model to OpenAI's `3-small` model.
|
||||
|
||||
Any of the supported aliases for a model can be passed to this command.
|
||||
|
||||
You can unset the default model using `--remove-default`:
|
||||
|
||||
```bash
|
||||
llm embed-models default --remove-default
|
||||
```
|
||||
When no default model is set, the `llm embed` and `llm embed-multi` commands will require that a model is specified using `-m/--model`.
|
||||
|
||||
## llm collections list
|
||||
|
||||
To list all of the collections in the embeddings database, run this command:
|
||||
|
||||
```bash
|
||||
llm collections list
|
||||
```
|
||||
Add `--json` for JSON output:
|
||||
```bash
|
||||
llm collections list --json
|
||||
```
|
||||
Add `-d/--database` to specify a different database file:
|
||||
```bash
|
||||
llm collections list -d my-embeddings.db
|
||||
```
|
||||
## llm collections delete
|
||||
|
||||
To delete a collection from the database, run this:
|
||||
```bash
|
||||
llm collections delete collection-name
|
||||
```
|
||||
Pass `-d` to specify a different database file:
|
||||
```bash
|
||||
llm collections delete collection-name -d my-embeddings.db
|
||||
```
|
||||
@@ -0,0 +1,26 @@
|
||||
(embeddings)=
|
||||
# Embeddings
|
||||
|
||||
Embedding models allow you to take a piece of text - a word, sentence, paragraph or even a whole article, and convert that into an array of floating point numbers.
|
||||
|
||||
This floating point array is called an "embedding vector", and works as a numerical representation of the semantic meaning of the content in a many-multi-dimensional space.
|
||||
|
||||
By calculating the distance between embedding vectors, we can identify which content is semantically "nearest" to other content.
|
||||
|
||||
This can be used to build features like related article lookups. It can also be used to build semantic search, where a user can search for a phrase and get back results that are semantically similar to that phrase even if they do not share any exact keywords.
|
||||
|
||||
Some embedding models like [CLIP](https://github.com/simonw/llm-clip) can even work against binary files such as images. These can be used to search for images that are similar to other images, or to search for images that are semantically similar to a piece of text.
|
||||
|
||||
LLM supports multiple embedding models through {ref}`plugins <plugins>`. Once installed, an embedding model can be used on the command-line or via the Python API to calculate and store embeddings for content, and then to perform similarity searches against those embeddings.
|
||||
|
||||
See [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) for an extended explanation of embeddings, why they are useful and what you can do with them.
|
||||
|
||||
```{toctree}
|
||||
---
|
||||
maxdepth: 3
|
||||
---
|
||||
cli
|
||||
python-api
|
||||
writing-plugins
|
||||
storage
|
||||
```
|
||||
@@ -0,0 +1,211 @@
|
||||
(embeddings-python-api)=
|
||||
# Using embeddings from Python
|
||||
|
||||
You can load an embedding model using its model ID or alias like this:
|
||||
```python
|
||||
import llm
|
||||
|
||||
embedding_model = llm.get_embedding_model("3-small")
|
||||
```
|
||||
To embed a string, returning a Python list of floating point numbers, use the `.embed()` method:
|
||||
```python
|
||||
vector = embedding_model.embed("my happy hound")
|
||||
```
|
||||
If the embedding model can handle binary input, you can call `.embed()` with a byte string instead. You can check the `supports_binary` property to see if this is supported:
|
||||
```python
|
||||
if embedding_model.supports_binary:
|
||||
vector = embedding_model.embed(open("my-image.jpg", "rb").read())
|
||||
```
|
||||
The `embedding_model.supports_text` property indicates if the model supports text input.
|
||||
|
||||
Many embeddings models are more efficient when you embed multiple strings or binary strings at once. To embed multiple strings at once, use the `.embed_multi()` method:
|
||||
```python
|
||||
vectors = list(embedding_model.embed_multi(["my happy hound", "my dissatisfied cat"]))
|
||||
```
|
||||
This returns a generator that yields one embedding vector per string.
|
||||
|
||||
Embeddings are calculated in batches. By default all items will be processed in a single batch, unless the underlying embedding model has defined its own preferred batch size. You can pass a custom batch size using `batch_size=N`, for example:
|
||||
|
||||
```python
|
||||
vectors = list(embedding_model.embed_multi(lines_from_file, batch_size=20))
|
||||
```
|
||||
|
||||
(embeddings-python-collections)=
|
||||
## Working with collections
|
||||
|
||||
The `llm.Collection` class can be used to work with **collections** of embeddings from Python code.
|
||||
|
||||
A collection is a named group of embedding vectors, each stored along with their IDs in a SQLite database table.
|
||||
|
||||
To work with embeddings in this way you will need an instance of a [sqlite-utils Database](https://sqlite-utils.datasette.io/en/stable/python-api.html#connecting-to-or-creating-a-database) object. You can then pass that to the `llm.Collection` constructor along with the unique string name of the collection and the ID of the embedding model you will be using with that collection:
|
||||
|
||||
```python
|
||||
import sqlite_utils
|
||||
import llm
|
||||
|
||||
# This collection will use an in-memory database that will be
|
||||
# discarded when the Python process exits
|
||||
collection = llm.Collection("entries", model_id="3-small")
|
||||
|
||||
# Or you can persist the database to disk like this:
|
||||
db = sqlite_utils.Database("my-embeddings.db")
|
||||
collection = llm.Collection("entries", db, model_id="3-small")
|
||||
|
||||
# You can pass a model directly using model= instead of model_id=
|
||||
embedding_model = llm.get_embedding_model("3-small")
|
||||
collection = llm.Collection("entries", db, model=embedding_model)
|
||||
```
|
||||
If the collection already exists in the database you can omit the `model` or `model_id` argument - the model ID will be read from the `collections` table.
|
||||
|
||||
To embed a single string and store it in the collection, use the `embed()` method:
|
||||
|
||||
```python
|
||||
collection.embed("hound", "my happy hound")
|
||||
```
|
||||
This stores the embedding for the string "my happy hound" in the `entries` collection under the key `hound`.
|
||||
|
||||
Add `store=True` to store the text content itself in the database table along with the embedding vector.
|
||||
|
||||
To attach additional metadata to an item, pass a JSON-compatible dictionary as the `metadata=` argument:
|
||||
|
||||
```python
|
||||
collection.embed("hound", "my happy hound", metadata={"name": "Hound"}, store=True)
|
||||
```
|
||||
This additional metadata will be stored as JSON in the `metadata` column of the embeddings database table.
|
||||
|
||||
(embeddings-python-bulk)=
|
||||
### Storing embeddings in bulk
|
||||
|
||||
The `collection.embed_multi()` method can be used to store embeddings for multiple items at once. This can be more efficient for some embedding models.
|
||||
|
||||
```python
|
||||
collection.embed_multi(
|
||||
[
|
||||
("hound", "my happy hound"),
|
||||
("cat", "my dissatisfied cat"),
|
||||
],
|
||||
# Add this to store the strings in the content column:
|
||||
store=True,
|
||||
)
|
||||
```
|
||||
To include metadata to be stored with each item, call `embed_multi_with_metadata()`:
|
||||
|
||||
```python
|
||||
collection.embed_multi_with_metadata(
|
||||
[
|
||||
("hound", "my happy hound", {"name": "Hound"}),
|
||||
("cat", "my dissatisfied cat", {"name": "Cat"}),
|
||||
],
|
||||
# This can also take the store=True argument:
|
||||
store=True,
|
||||
)
|
||||
```
|
||||
The `batch_size=` argument defaults to 100, and will be used unless the embedding model itself defines a lower batch size. You can adjust this if you are having trouble with memory while embedding large collections:
|
||||
|
||||
```python
|
||||
collection.embed_multi(
|
||||
(
|
||||
(i, line)
|
||||
for i, line in enumerate(lines_in_file)
|
||||
),
|
||||
batch_size=10
|
||||
)
|
||||
```
|
||||
|
||||
(embeddings-python-collection-class)=
|
||||
### Collection class reference
|
||||
|
||||
A collection instance has the following properties and methods:
|
||||
|
||||
- `id` - the integer ID of the collection in the database
|
||||
- `name` - the string name of the collection (unique in the database)
|
||||
- `model_id` - the string ID of the embedding model used for this collection
|
||||
- `model()` - returns the `EmbeddingModel` instance, based on that `model_id`
|
||||
- `count()` - returns the integer number of items in the collection
|
||||
- `embed(id: str, text: str, metadata: dict=None, store: bool=False)` - embeds the given string and stores it in the collection under the given ID. Can optionally include metadata (stored as JSON) and store the text content itself in the database table.
|
||||
- `embed_multi(entries: Iterable, store: bool=False, batch_size: int=100)` - see above
|
||||
- `embed_multi_with_metadata(entries: Iterable, store: bool=False, batch_size: int=100)` - see above
|
||||
- `similar(query: str, number: int=10)` - returns a list of entries that are most similar to the embedding of the given query string
|
||||
- `similar_by_id(id: str, number: int=10)` - returns a list of entries that are most similar to the embedding of the item with the given ID
|
||||
- `similar_by_vector(vector: List[float], number: int=10, skip_id: str=None)` - returns a list of entries that are most similar to the given embedding vector, optionally skipping the entry with the given ID
|
||||
- `delete()` - deletes the collection and its embeddings from the database
|
||||
|
||||
There is also a `Collection.exists(db, name)` class method which returns a boolean value and can be used to determine if a collection exists or not in a database:
|
||||
|
||||
```python
|
||||
if Collection.exists(db, "entries"):
|
||||
print("The entries collection exists")
|
||||
```
|
||||
|
||||
(embeddings-python-similar)=
|
||||
## Retrieving similar items
|
||||
|
||||
Once you have populated a collection of embeddings you can retrieve the entries that are most similar to a given string using the `similar()` method.
|
||||
|
||||
This method uses a brute force approach, calculating distance scores against every document. This is fine for small collections, but will not scale to large collections. See [issue 216](https://github.com/simonw/llm/issues/216) for plans to add a more scalable approach via vector indexes provided by plugins.
|
||||
|
||||
```python
|
||||
for entry in collection.similar("hound"):
|
||||
print(entry.id, entry.score)
|
||||
```
|
||||
The string will first by embedded using the model for the collection.
|
||||
|
||||
The `entry` object returned is an object with the following properties:
|
||||
|
||||
- `id` - the string ID of the item
|
||||
- `score` - the floating point similarity score between the item and the query string
|
||||
- `content` - the string text content of the item, if it was stored - or `None`
|
||||
- `metadata` - the dictionary (from JSON) metadata for the item, if it was stored - or `None`
|
||||
|
||||
This defaults to returning the 10 most similar items. You can change this by passing a different `number=` argument:
|
||||
```python
|
||||
for entry in collection.similar("hound", number=5):
|
||||
print(entry.id, entry.score)
|
||||
```
|
||||
The `similar_by_id()` method takes the ID of another item in the collection and returns the most similar items to that one, based on the embedding that has already been stored for it:
|
||||
|
||||
```python
|
||||
for entry in collection.similar_by_id("cat"):
|
||||
print(entry.id, entry.score)
|
||||
```
|
||||
The item itself is excluded from the results.
|
||||
|
||||
(embeddings-sql-schema)=
|
||||
## SQL schema
|
||||
|
||||
Here's the SQL schema used by the embeddings database:
|
||||
|
||||
<!-- [[[cog
|
||||
import cog
|
||||
from llm.embeddings_migrations import embeddings_migrations
|
||||
import sqlite_utils
|
||||
import re
|
||||
db = sqlite_utils.Database(memory=True)
|
||||
embeddings_migrations.apply(db)
|
||||
|
||||
cog.out("```sql\n")
|
||||
for table in ("collections", "embeddings"):
|
||||
schema = db[table].schema
|
||||
cog.out(format(schema))
|
||||
cog.out("\n")
|
||||
cog.out("```\n")
|
||||
]]] -->
|
||||
```sql
|
||||
CREATE TABLE "collections" (
|
||||
"id" INTEGER PRIMARY KEY,
|
||||
"name" TEXT,
|
||||
"model" TEXT
|
||||
)
|
||||
CREATE TABLE "embeddings" (
|
||||
"collection_id" INTEGER REFERENCES "collections"("id"),
|
||||
"id" TEXT,
|
||||
"embedding" BLOB,
|
||||
"content" TEXT,
|
||||
"content_blob" BLOB,
|
||||
"content_hash" BLOB,
|
||||
"metadata" TEXT,
|
||||
"updated" INTEGER,
|
||||
PRIMARY KEY ("collection_id", "id")
|
||||
)
|
||||
```
|
||||
<!-- [[[end]]] -->
|
||||
@@ -0,0 +1,31 @@
|
||||
(embeddings-storage)=
|
||||
# Embedding storage format
|
||||
|
||||
The default output format of the `llm embed` command is a JSON array of floating point numbers.
|
||||
|
||||
LLM stores embeddings in space-efficient format: a little-endian binary sequences of 32-bit floating point numbers, each represented using 4 bytes.
|
||||
|
||||
These are stored in a `BLOB` column in a SQLite database.
|
||||
|
||||
The following Python functions can be used to convert between this format and an array of floating point numbers:
|
||||
|
||||
```python
|
||||
import struct
|
||||
|
||||
def encode(values):
|
||||
return struct.pack("<" + "f" * len(values), *values)
|
||||
|
||||
def decode(binary):
|
||||
return struct.unpack("<" + "f" * (len(binary) // 4), binary)
|
||||
```
|
||||
|
||||
These functions are available as `llm.encode()` and `llm.decode()`.
|
||||
|
||||
If you are using [NumPy](https://numpy.org/) you can decode one of these binary values like this:
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
|
||||
numpy_array = np.frombuffer(value, "<f4")
|
||||
```
|
||||
The `<f4` format string here ensures NumPy will treat the data as a little-endian sequence of 32-bit floats.
|
||||
@@ -0,0 +1,73 @@
|
||||
(embeddings-writing-plugins)=
|
||||
# Writing plugins to add new embedding models
|
||||
|
||||
Read the {ref}`plugin tutorial <tutorial-model-plugin>` for details on how to develop and package a plugin.
|
||||
|
||||
This page shows an example plugin that implements and registers a new embedding model.
|
||||
|
||||
There are two components to an embedding model plugin:
|
||||
|
||||
1. An implementation of the `register_embedding_models()` hook, which takes a `register` callback function and calls it to register the new model with the LLM plugin system.
|
||||
2. A class that extends the `llm.EmbeddingModel` abstract base class.
|
||||
|
||||
The only required method on this class is `embed_batch(texts)`, which takes an iterable of strings and returns an iterator over lists of floating point numbers.
|
||||
|
||||
The following example uses the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) package to provide access to the [MiniLM-L6](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) embedding model.
|
||||
|
||||
```{eval-rst}
|
||||
.. autoclass:: llm.EmbeddingModel
|
||||
:members: embed, embed_multi, embed_batch
|
||||
```
|
||||
|
||||
```python
|
||||
import llm
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
|
||||
@llm.hookimpl
|
||||
def register_embedding_models(register):
|
||||
model_id = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
register(SentenceTransformerModel(model_id, model_id), aliases=("all-MiniLM-L6-v2",))
|
||||
|
||||
|
||||
class SentenceTransformerModel(llm.EmbeddingModel):
|
||||
def __init__(self, model_id, model_name):
|
||||
self.model_id = model_id
|
||||
self.model_name = model_name
|
||||
self._model = None
|
||||
|
||||
def embed_batch(self, texts):
|
||||
if self._model is None:
|
||||
self._model = SentenceTransformer(self.model_name)
|
||||
results = self._model.encode(texts)
|
||||
return (list(map(float, result)) for result in results)
|
||||
```
|
||||
Once installed, the model provided by this plugin can be used with the {ref}`llm embed <embeddings-cli-embed>` command like this:
|
||||
|
||||
```bash
|
||||
cat file.txt | llm embed -m sentence-transformers/all-MiniLM-L6-v2
|
||||
```
|
||||
Or via its registered alias like this:
|
||||
```bash
|
||||
cat file.txt | llm embed -m all-MiniLM-L6-v2
|
||||
```
|
||||
[llm-sentence-transformers](https://github.com/simonw/llm-sentence-transformers) is a complete example of a plugin that provides an embedding model.
|
||||
|
||||
[Execute Jina embeddings with a CLI using llm-embed-jina](https://simonwillison.net/2023/Oct/26/llm-embed-jina/#how-i-built-the-plugin) talks through a similar process to add support for the [Jina embeddings models](https://jina.ai/news/jina-ai-launches-worlds-first-open-source-8k-text-embedding-rivaling-openai/).
|
||||
|
||||
## Embedding binary content
|
||||
|
||||
If your model can embed binary content, use the `supports_binary` property to indicate that:
|
||||
|
||||
```python
|
||||
class ClipEmbeddingModel(llm.EmbeddingModel):
|
||||
model_id = "clip"
|
||||
supports_binary = True
|
||||
supports_text= True
|
||||
```
|
||||
|
||||
`supports_text` defaults to `True` and so is not necessary here. You can set it to `False` if your model only supports binary data.
|
||||
|
||||
If your model accepts binary, your `.embed_batch()` model may be called with a list of Python bytestrings. These may be mixed with regular strings if the model accepts both types of input.
|
||||
|
||||
[llm-clip](https://github.com/simonw/llm-clip) is an example of a model that can embed both binary and text content.
|
||||
@@ -0,0 +1,217 @@
|
||||
(fragments)=
|
||||
# Fragments
|
||||
|
||||
LLM prompts can optionally be composed out of **fragments** - reusable pieces of text that are logged just once to the database and can then be attached to multiple prompts.
|
||||
|
||||
These are particularly useful when you are working with long context models, which support feeding large amounts of text in as part of your prompt.
|
||||
|
||||
Fragments primarily exist to save space in the database, but may be used to support other features such as vendor prompt caching as well.
|
||||
|
||||
Fragments can be specified using several different mechanisms:
|
||||
|
||||
- URLs to text files online
|
||||
- Paths to text files on disk
|
||||
- Aliases that have been attached to a specific fragment
|
||||
- Hash IDs of stored fragments, where the ID is the SHA256 hash of the fragment content
|
||||
- Fragments that are provided by custom plugins - these look like `plugin-name:argument`
|
||||
|
||||
(fragments-usage)=
|
||||
## Using fragments in a prompt
|
||||
|
||||
Use the `-f/--fragment` option to specify one or more fragments to be used as part of your prompt:
|
||||
|
||||
```bash
|
||||
llm -f https://llm.datasette.io/robots.txt "Explain this robots.txt file in detail"
|
||||
```
|
||||
Here we are specifying a fragment using a URL. The contents of that URL will be included in the prompt that is sent to the model, prepended prior to the prompt text.
|
||||
|
||||
<!--[[[cog
|
||||
from importlib.metadata import version
|
||||
llm_version = version("llm")
|
||||
cog.out(f'The URL will be fetched with the user-agent `llm/{llm_version} (https://llm.datasette.io/)`.')
|
||||
]]]-->
|
||||
The URL will be fetched with the user-agent `llm/0.32a3 (https://llm.datasette.io/)`.
|
||||
<!--[[[end]]]-->
|
||||
|
||||
The `-f` option can be used multiple times to combine together multiple fragments.
|
||||
|
||||
Fragments can also be files on disk, for example:
|
||||
```bash
|
||||
llm -f setup.py 'extract the metadata'
|
||||
```
|
||||
Use `-` to specify a fragment that is read from standard input:
|
||||
```bash
|
||||
llm -f - 'extract the metadata' < setup.py
|
||||
```
|
||||
This will read the contents of `setup.py` from standard input and use it as a fragment.
|
||||
|
||||
Fragments can also be used as part of your system prompt. Use `--sf value` or `--system-fragment value` instead of `-f`.
|
||||
|
||||
## Using fragments in chat
|
||||
|
||||
The `chat` command also supports the `-f` and `--sf` arguments to start a chat with fragments.
|
||||
|
||||
```bash
|
||||
llm chat -f my_doc.txt
|
||||
Chatting with gpt-4
|
||||
Type 'exit' or 'quit' to exit
|
||||
Type '!multi' to enter multiple lines, then '!end' to finish
|
||||
Type '!edit' to open your default editor and modify the prompt.
|
||||
Type '!fragment <my_fragment> [<another_fragment> ...]' to insert one or more fragments
|
||||
> Explain this document to me
|
||||
```
|
||||
|
||||
Fragments can also be added *during* a chat conversation using the `!fragment <my_fragment>` command.
|
||||
|
||||
```bash
|
||||
Chatting with gpt-4
|
||||
Type 'exit' or 'quit' to exit
|
||||
Type '!multi' to enter multiple lines, then '!end' to finish
|
||||
Type '!edit' to open your default editor and modify the prompt.
|
||||
Type '!fragment <my_fragment> [<another_fragment> ...]' to insert one or more fragments
|
||||
> !fragment https://llm.datasette.io/en/stable/fragments.html
|
||||
```
|
||||
|
||||
This can be combined with `!multi`:
|
||||
|
||||
```bash
|
||||
> !multi
|
||||
Explain the difference between fragments and templates to me
|
||||
!fragment https://llm.datasette.io/en/stable/fragments.html https://llm.datasette.io/en/stable/templates.html
|
||||
!end
|
||||
```
|
||||
|
||||
Any `!fragment` lines found in a prompt created with `!edit` will not be parsed.
|
||||
|
||||
(fragments-browsing)=
|
||||
## Browsing fragments
|
||||
|
||||
You can view a truncated version of the fragments you have previously stored in your database with the `llm fragments` command:
|
||||
|
||||
```bash
|
||||
llm fragments
|
||||
```
|
||||
The output from that command looks like this:
|
||||
|
||||
```yaml
|
||||
- hash: 0d6e368f9bc21f8db78c01e192ecf925841a957d8b991f5bf9f6239aa4d81815
|
||||
aliases: []
|
||||
datetime_utc: '2025-04-06 07:36:53'
|
||||
source: https://raw.githubusercontent.com/simonw/llm-docs/refs/heads/main/llm/0.22.txt
|
||||
content: |-
|
||||
<documents>
|
||||
<document index="1">
|
||||
<source>docs/aliases.md</source>
|
||||
<document_content>
|
||||
(aliases)=
|
||||
#...
|
||||
- hash: 16b686067375182573e2aa16b5bfc1e64d48350232535d06444537e51f1fd60c
|
||||
aliases: []
|
||||
datetime_utc: '2025-04-06 23:03:47'
|
||||
source: simonw/files-to-prompt/pyproject.toml
|
||||
content: |-
|
||||
[project]
|
||||
name = "files-to-prompt"
|
||||
version = "0.6"
|
||||
description = "Concatenate a directory full of...
|
||||
```
|
||||
Those long `hash` values are IDs that can be used to reference a fragment in the future:
|
||||
```bash
|
||||
llm -f 16b686067375182573e2aa16b5bfc1e64d48350232535d06444537e51f1fd60c 'Extract metadata'
|
||||
```
|
||||
Use `-q searchterm` one or more times to search for fragments that match a specific set of search terms.
|
||||
|
||||
To view the full content of a fragment use `llm fragments show`:
|
||||
```bash
|
||||
llm fragments show 0d6e368f9bc21f8db78c01e192ecf925841a957d8b991f5bf9f6239aa4d81815
|
||||
```
|
||||
|
||||
(fragments-aliases)=
|
||||
## Setting aliases for fragments
|
||||
|
||||
You can assign aliases to fragments that you use often using the `llm fragments set` command:
|
||||
```bash
|
||||
llm fragments set mydocs ./docs.md
|
||||
```
|
||||
To remove an alias, use `llm fragments remove`:
|
||||
```bash
|
||||
llm fragments remove mydocs
|
||||
```
|
||||
You can then use that alias in place of the fragment hash ID:
|
||||
```bash
|
||||
llm -f mydocs 'How do I access metadata?'
|
||||
```
|
||||
Use `llm fragments --aliases` to see a full list of fragments that have been assigned aliases:
|
||||
```bash
|
||||
llm fragments --aliases
|
||||
```
|
||||
|
||||
(fragments-logs)=
|
||||
## Viewing fragments in your logs
|
||||
|
||||
The `llm logs` command lists the fragments that were used for a prompt. By default these are listed as fragment hash IDs, but you can use the `--expand` option to show the full content of each fragment.
|
||||
|
||||
This command will show the expanded fragments for your most recent conversation:
|
||||
|
||||
```bash
|
||||
llm logs -c --expand
|
||||
```
|
||||
You can filter for logs that used a specific fragment using the `-f/--fragment` option:
|
||||
```bash
|
||||
llm logs -c -f 0d6e368f9bc21f8db78c01e192ecf925841a957d8b991f5bf9f6239aa4d81815
|
||||
```
|
||||
This accepts URLs, file paths, aliases, and hash IDs.
|
||||
|
||||
Multiple `-f` options will return responses that used **all** of the specified fragments.
|
||||
|
||||
Fragments are returned by `llm logs --json` as well. By default these are truncated but you can add the `-e/--expand` option to show the full content of each fragment.
|
||||
|
||||
```bash
|
||||
llm logs -c --json --expand
|
||||
```
|
||||
|
||||
(fragments-plugins)=
|
||||
## Using fragments from plugins
|
||||
|
||||
LLM plugins can provide custom fragment loaders which do useful things.
|
||||
|
||||
One example is the [llm-fragments-github plugin](https://github.com/simonw/llm-fragments-github). This can convert the files from a public GitHub repository into a list of fragments, allowing you to ask questions about the full repository.
|
||||
|
||||
Here's how to try that out:
|
||||
|
||||
```bash
|
||||
llm install llm-fragments-github
|
||||
llm -f github:simonw/s3-credentials 'Suggest new features for this tool'
|
||||
```
|
||||
This plugin turns a single call to `-f github:simonw/s3-credentials` into multiple fragments, one for every text file in the [simonw/s3-credentials](https://github.com/simonw/s3-credentials) GitHub repository.
|
||||
|
||||
Running `llm logs -c` will show that this prompt incorporated 26 fragments, one for each file.
|
||||
|
||||
Running `llm logs -c --usage --expand` (shortcut: `llm logs -cue`) includes token usage information and turns each fragment ID into a full copy of that file. [Here's the output of that command](https://gist.github.com/simonw/c9bbbc5f6560b01f4b7882ac0194fb25).
|
||||
|
||||
Fragment plugins can return {ref}`attachments <usage-attachments>` (such as images) as well.
|
||||
|
||||
See the {ref}`register_fragment_loaders() plugin hook <plugin-hooks-register-fragment-loaders>` documentation for details on writing your own custom fragment plugin.
|
||||
|
||||
(fragments-loaders)=
|
||||
## Listing available fragment prefixes
|
||||
|
||||
The `llm fragments loaders` command shows all prefixes that have been installed by plugins, along with their documentation:
|
||||
|
||||
```bash
|
||||
llm install llm-fragments-github
|
||||
llm fragments loaders
|
||||
```
|
||||
Example output:
|
||||
```
|
||||
github:
|
||||
Load files from a GitHub repository as fragments
|
||||
|
||||
Argument is a GitHub repository URL or username/repository
|
||||
|
||||
issue:
|
||||
Fetch GitHub issue and comments as Markdown
|
||||
|
||||
Argument is either "owner/repo/NUMBER"
|
||||
or "https://github.com/owner/repo/issues/NUMBER"
|
||||
```
|
||||
+1088
File diff suppressed because it is too large
Load Diff
+133
@@ -0,0 +1,133 @@
|
||||
# LLM
|
||||
|
||||
[](https://github.com/simonw/llm)
|
||||
[](https://pypi.org/project/llm/)
|
||||
[](https://llm.datasette.io/en/stable/changelog.html)
|
||||
[](https://github.com/simonw/llm/actions?query=workflow%3ATest)
|
||||
[](https://github.com/simonw/llm/blob/main/LICENSE)
|
||||
[](https://datasette.io/discord-llm)
|
||||
[](https://formulae.brew.sh/formula/llm)
|
||||
|
||||
A CLI tool and Python library for interacting with **OpenAI**, **Anthropic's Claude**, **Google's Gemini**, **Meta's Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine.
|
||||
|
||||
Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/).
|
||||
|
||||
With LLM you can:
|
||||
- {ref}`Run prompts from the command-line <usage-executing-prompts>`
|
||||
- {ref}`Store prompts and responses in SQLite <logging>`
|
||||
- {ref}`Generate and store embeddings <embeddings>`
|
||||
- {ref}`Extract structured content from text and images <schemas>`
|
||||
- {ref}`Grant models the ability to execute tools <tools>`
|
||||
- ... and much, much more
|
||||
|
||||
## Quick start
|
||||
|
||||
First, install LLM using `pip` or Homebrew or `pipx` or `uv`:
|
||||
|
||||
```bash
|
||||
pip install llm
|
||||
```
|
||||
Or with Homebrew (see {ref}`warning note <homebrew-warning>`):
|
||||
```bash
|
||||
brew install llm
|
||||
```
|
||||
Or with [pipx](https://pypa.github.io/pipx/):
|
||||
```bash
|
||||
pipx install llm
|
||||
```
|
||||
Or with [uv](https://docs.astral.sh/uv/guides/tools/)
|
||||
```bash
|
||||
uv tool install llm
|
||||
```
|
||||
If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this:
|
||||
```bash
|
||||
# Paste your OpenAI API key into this
|
||||
llm keys set openai
|
||||
|
||||
# Run a prompt (with the default gpt-4o-mini model)
|
||||
llm "Ten fun names for a pet pelican"
|
||||
|
||||
# Extract text from an image
|
||||
llm "extract text" -a scanned-document.jpg
|
||||
|
||||
# Use a system prompt against a file
|
||||
cat myfile.py | llm -s "Explain this code"
|
||||
```
|
||||
Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic](https://console.anthropic.com/) with their respective plugins:
|
||||
```bash
|
||||
llm install llm-gemini
|
||||
llm keys set gemini
|
||||
# Paste Gemini API key here
|
||||
llm -m gemini-2.0-flash 'Tell me fun facts about Mountain View'
|
||||
|
||||
llm install llm-anthropic
|
||||
llm keys set anthropic
|
||||
# Paste Anthropic API key here
|
||||
llm -m claude-4-opus 'Impress me with wild facts about turnips'
|
||||
```
|
||||
You can also {ref}`install a plugin <installing-plugins>` to access models that can run on your local device. If you use [Ollama](https://ollama.com/):
|
||||
```bash
|
||||
# Install the plugin
|
||||
llm install llm-ollama
|
||||
|
||||
# Download and run a prompt against the Orca Mini 7B model
|
||||
ollama pull llama3.2:latest
|
||||
llm -m llama3.2:latest 'What is the capital of France?'
|
||||
```
|
||||
To start {ref}`an interactive chat <usage-chat>` with a model, use `llm chat`:
|
||||
```bash
|
||||
llm chat -m gpt-4.1
|
||||
```
|
||||
```
|
||||
Chatting with gpt-4.1
|
||||
Type 'exit' or 'quit' to exit
|
||||
Type '!multi' to enter multiple lines, then '!end' to finish
|
||||
Type '!edit' to open your default editor and modify the prompt.
|
||||
Type '!fragment <my_fragment> [<another_fragment> ...]' to insert one or more fragments
|
||||
> Tell me a joke about a pelican
|
||||
Why don't pelicans like to tip waiters?
|
||||
|
||||
Because they always have a big bill!
|
||||
```
|
||||
|
||||
More background on this project:
|
||||
|
||||
- [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/)
|
||||
- [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/)
|
||||
- [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/)
|
||||
- [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/)
|
||||
- [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/)
|
||||
- [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/)
|
||||
- [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/)
|
||||
|
||||
See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog.
|
||||
|
||||
## Contents
|
||||
|
||||
```{toctree}
|
||||
---
|
||||
maxdepth: 3
|
||||
---
|
||||
setup
|
||||
usage
|
||||
openai-models
|
||||
other-models
|
||||
tools
|
||||
schemas
|
||||
templates
|
||||
fragments
|
||||
aliases
|
||||
embeddings/index
|
||||
plugins/index
|
||||
python-api
|
||||
logging
|
||||
related-tools
|
||||
help
|
||||
contributing
|
||||
```
|
||||
```{toctree}
|
||||
---
|
||||
maxdepth: 1
|
||||
---
|
||||
changelog
|
||||
```
|
||||
+425
@@ -0,0 +1,425 @@
|
||||
(logging)=
|
||||
# Logging to SQLite
|
||||
|
||||
`llm` defaults to logging all prompts and responses to a SQLite database.
|
||||
|
||||
You can find the location of that database using the `llm logs path` command:
|
||||
|
||||
```bash
|
||||
llm logs path
|
||||
```
|
||||
On my Mac that outputs:
|
||||
```
|
||||
/Users/simon/Library/Application Support/io.datasette.llm/logs.db
|
||||
```
|
||||
This will differ for other operating systems.
|
||||
|
||||
To avoid logging an individual prompt, pass `--no-log` or `-n` to the command:
|
||||
```bash
|
||||
llm 'Ten names for cheesecakes' -n
|
||||
```
|
||||
|
||||
To turn logging by default off:
|
||||
|
||||
```bash
|
||||
llm logs off
|
||||
```
|
||||
If you've turned off logging you can still log an individual prompt and response by adding `--log`:
|
||||
```bash
|
||||
llm 'Five ambitious names for a pet pterodactyl' --log
|
||||
```
|
||||
To turn logging by default back on again:
|
||||
|
||||
```bash
|
||||
llm logs on
|
||||
```
|
||||
To see the status of the logs database, run this:
|
||||
```bash
|
||||
llm logs status
|
||||
```
|
||||
Example output:
|
||||
```
|
||||
Logging is ON for all prompts
|
||||
Found log database at /Users/simon/Library/Application Support/io.datasette.llm/logs.db
|
||||
Number of conversations logged: 33
|
||||
Number of responses logged: 48
|
||||
Database file size: 19.96MB
|
||||
```
|
||||
|
||||
(logging-view)=
|
||||
|
||||
## Viewing the logs
|
||||
|
||||
You can view the logs using the `llm logs` command:
|
||||
```bash
|
||||
llm logs
|
||||
```
|
||||
This will output the three most recent logged items in Markdown format, showing both the prompt and the response formatted using Markdown.
|
||||
|
||||
To get back just the most recent prompt response as plain text, add `-r/--response`:
|
||||
|
||||
```bash
|
||||
llm logs -r
|
||||
```
|
||||
Use `-x/--extract` to extract and return the first fenced code block from the selected log entries:
|
||||
|
||||
```bash
|
||||
llm logs --extract
|
||||
```
|
||||
Or `--xl/--extract-last` for the last fenced code block:
|
||||
```bash
|
||||
llm logs --extract-last
|
||||
```
|
||||
|
||||
Add `--json` to get the log messages in JSON instead:
|
||||
|
||||
```bash
|
||||
llm logs --json
|
||||
```
|
||||
|
||||
Add `-n 10` to see the ten most recent items:
|
||||
```bash
|
||||
llm logs -n 10
|
||||
```
|
||||
Or `-n 0` to see everything that has ever been logged:
|
||||
```bash
|
||||
llm logs -n 0
|
||||
```
|
||||
You can truncate the display of the prompts and responses using the `-t/--truncate` option. This can help make the JSON output more readable - though the `--short` option is usually better.
|
||||
```bash
|
||||
llm logs -n 1 -t --json
|
||||
```
|
||||
Example output:
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": "01jm8ec74wxsdatyn5pq1fp0s5",
|
||||
"model": "anthropic/claude-3-haiku-20240307",
|
||||
"prompt": "hi",
|
||||
"system": null,
|
||||
"prompt_json": null,
|
||||
"response": "Hello! How can I assist you today?",
|
||||
"conversation_id": "01jm8ec74taftdgj2t4zra9z0j",
|
||||
"duration_ms": 560,
|
||||
"datetime_utc": "2025-02-16T22:34:30.374882+00:00",
|
||||
"input_tokens": 8,
|
||||
"output_tokens": 12,
|
||||
"token_details": null,
|
||||
"conversation_name": "hi",
|
||||
"conversation_model": "anthropic/claude-3-haiku-20240307",
|
||||
"attachments": []
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
(logging-short)=
|
||||
|
||||
### -s/--short mode
|
||||
|
||||
Use `-s/--short` to see a shortened YAML log with truncated prompts and no responses:
|
||||
```bash
|
||||
llm logs -n 2 --short
|
||||
```
|
||||
Example output:
|
||||
```yaml
|
||||
- model: deepseek-reasoner
|
||||
datetime: '2025-02-02T06:39:53'
|
||||
conversation: 01jk2pk05xq3d0vgk0202zrsg1
|
||||
prompt: H01 There are five huts. H02 The Scotsman lives in the purple hut. H03 The Welshman owns the parrot. H04 Kombucha is...
|
||||
- model: o3-mini
|
||||
datetime: '2025-02-02T19:03:05'
|
||||
conversation: 01jk40qkxetedzpf1zd8k9bgww
|
||||
system: Formatting re-enabled. Write a detailed README with extensive usage examples.
|
||||
prompt: <documents> <document index="1"> <source>./Cargo.toml</source> <document_content> [package] name = "py-limbo" version...
|
||||
```
|
||||
Include `-u/--usage` to include token usage information:
|
||||
|
||||
```bash
|
||||
llm logs -n 1 --short --usage
|
||||
```
|
||||
Example output:
|
||||
```yaml
|
||||
- model: o3-mini
|
||||
datetime: '2025-02-16T23:00:56'
|
||||
conversation: 01jm8fxxnef92n1663c6ays8xt
|
||||
system: Produce Python code that demonstrates every possible usage of yaml.dump
|
||||
with all of the arguments it can take, especi...
|
||||
prompt: <documents> <document index="1"> <source>./setup.py</source> <document_content>
|
||||
NAME = 'PyYAML' VERSION = '7.0.0.dev0...
|
||||
usage:
|
||||
input: 74793
|
||||
output: 3550
|
||||
details:
|
||||
completion_tokens_details:
|
||||
reasoning_tokens: 2240
|
||||
```
|
||||
|
||||
(logging-conversation)=
|
||||
|
||||
### Logs for a conversation
|
||||
|
||||
To view the logs for the most recent {ref}`conversation <usage-conversation>` you have had with a model, use `-c`:
|
||||
|
||||
```bash
|
||||
llm logs -c
|
||||
```
|
||||
To see logs for a specific conversation based on its ID, use `--cid ID` or `--conversation ID`:
|
||||
|
||||
```bash
|
||||
llm logs --cid 01h82n0q9crqtnzmf13gkyxawg
|
||||
```
|
||||
|
||||
(logging-search)=
|
||||
|
||||
### Searching the logs
|
||||
|
||||
You can search the logs for a search term in the `prompt` or the `response` columns.
|
||||
```bash
|
||||
llm logs -q 'cheesecake'
|
||||
```
|
||||
The most relevant results will be shown first.
|
||||
|
||||
To switch to sorting with most recent first, add `-l/--latest`. This can be combined with `-n` to limit the number of results shown:
|
||||
```bash
|
||||
llm logs -q 'cheesecake' -l -n 3
|
||||
```
|
||||
|
||||
(logging-filter-id)=
|
||||
|
||||
### Filtering past a specific ID
|
||||
|
||||
If you want to retrieve all of the logs that were recorded since a specific response ID you can do so using these options:
|
||||
|
||||
- `--id-gt $ID` - every record with an ID greater than $ID
|
||||
- `--id-gte $ID` - every record with an ID greater than or equal to $ID
|
||||
|
||||
IDs are always issued in ascending order by time, so this provides a useful way to see everything that has happened since a particular record.
|
||||
|
||||
This can be particularly useful when {ref}`working with schema data <schemas-logs>`, where you might want to access every record that you have created using a specific `--schema` but exclude records you have previously processed.
|
||||
|
||||
(logging-filter-model)=
|
||||
|
||||
### Filtering by model
|
||||
|
||||
You can filter to logs just for a specific model (or model alias) using `-m/--model`:
|
||||
```bash
|
||||
llm logs -m chatgpt
|
||||
```
|
||||
|
||||
(logging-filter-fragments)=
|
||||
|
||||
### Filtering by prompts that used specific fragments
|
||||
|
||||
The `-f/--fragment X` option will filter for just responses that were created using the specified {ref}`fragment <usage-fragments>` hash or alias or URL or filename.
|
||||
|
||||
Fragments are displayed in the logs as their hash ID. Add `-e/--expand` to display fragments as their full content - this option works for both the default Markdown and the `--json` mode:
|
||||
|
||||
```bash
|
||||
llm logs -f https://llm.datasette.io/robots.txt --expand
|
||||
```
|
||||
You can display just the content for a specific fragment hash ID (or alias) using the `llm fragments show` command:
|
||||
|
||||
```bash
|
||||
llm fragments show 993fd38d898d2b59fd2d16c811da5bdac658faa34f0f4d411edde7c17ebb0680
|
||||
```
|
||||
If you provide multiple fragments you will get back responses that used _all_ of those fragments.
|
||||
|
||||
(logging-filter-tools)=
|
||||
|
||||
### Filtering by prompts that used specific tools
|
||||
|
||||
You can filter for responses that used tools from specific fragments with the `--tool/-T` option:
|
||||
|
||||
```bash
|
||||
llm logs -T simple_eval
|
||||
```
|
||||
This will match responses that involved a _result_ from that tool. If the tool was not executed it will not be included in the filtered responses.
|
||||
|
||||
Pass `--tool/-T` multiple times for responses that used all of the specified tools.
|
||||
|
||||
Use the `llm logs --tools` flag to see _all_ responses that involved at least one tool result, including from `--functions`:
|
||||
|
||||
```bash
|
||||
llm logs --tools
|
||||
```
|
||||
|
||||
(logging-filter-schemas)=
|
||||
|
||||
### Browsing data collected using schemas
|
||||
|
||||
The `--schema X` option can be used to view responses that used the specified schema, using any of the {ref}`ways to specify a schema <schemas-specify>`:
|
||||
|
||||
```bash
|
||||
llm logs --schema 'name, age int, bio'
|
||||
```
|
||||
|
||||
This can be combined with `--data` and `--data-array` and `--data-key` to extract just the returned JSON data - consult the {ref}`schemas documentation <schemas-logs>` for details.
|
||||
|
||||
(logging-datasette)=
|
||||
|
||||
## Browsing logs using Datasette
|
||||
|
||||
You can also use [Datasette](https://datasette.io/) to browse your logs like this:
|
||||
|
||||
```bash
|
||||
datasette "$(llm logs path)"
|
||||
```
|
||||
|
||||
(logging-backup)=
|
||||
|
||||
## Backing up your database
|
||||
|
||||
You can backup your logs to another file using the `llm logs backup` command:
|
||||
|
||||
```bash
|
||||
llm logs backup /tmp/backup.db
|
||||
```
|
||||
This uses SQLite [VACUUM INTO](https://sqlite.org/lang_vacuum.html#vacuum_with_an_into_clause) under the hood.
|
||||
|
||||
(logging-sql-schema)=
|
||||
|
||||
## SQL schema
|
||||
|
||||
Here's the SQL schema used by the `logs.db` database:
|
||||
|
||||
<!-- [[[cog
|
||||
import cog
|
||||
from llm.migrations import migrate
|
||||
import sqlite_utils
|
||||
import re
|
||||
db = sqlite_utils.Database(memory=True)
|
||||
migrate(db)
|
||||
|
||||
def cleanup_sql(sql):
|
||||
first_line = sql.split('(')[0]
|
||||
inner = re.search(r'\((.*)\)', sql, re.DOTALL).group(1)
|
||||
columns = [l.strip() for l in inner.split(',')]
|
||||
return first_line + '(\n ' + ',\n '.join(columns) + '\n);'
|
||||
|
||||
cog.out("```sql\n")
|
||||
for table in (
|
||||
"conversations", "schemas", "responses", "responses_fts", "attachments", "prompt_attachments",
|
||||
"fragments", "fragment_aliases", "prompt_fragments", "system_fragments", "tools",
|
||||
"tool_responses", "tool_calls", "tool_results", "tool_instances"
|
||||
):
|
||||
schema = db[table].schema
|
||||
cog.out(format(cleanup_sql(schema)))
|
||||
cog.out("\n")
|
||||
cog.out("```\n")
|
||||
]]] -->
|
||||
```sql
|
||||
CREATE TABLE "conversations" (
|
||||
"id" TEXT PRIMARY KEY,
|
||||
"name" TEXT,
|
||||
"model" TEXT
|
||||
);
|
||||
CREATE TABLE "schemas" (
|
||||
"id" TEXT PRIMARY KEY,
|
||||
"content" TEXT
|
||||
);
|
||||
CREATE TABLE "responses" (
|
||||
"id" TEXT PRIMARY KEY,
|
||||
"model" TEXT,
|
||||
"prompt" TEXT,
|
||||
"system" TEXT,
|
||||
"prompt_json" TEXT,
|
||||
"options_json" TEXT,
|
||||
"response" TEXT,
|
||||
"response_json" TEXT,
|
||||
"conversation_id" TEXT REFERENCES "conversations"("id"),
|
||||
"duration_ms" INTEGER,
|
||||
"datetime_utc" TEXT,
|
||||
"input_tokens" INTEGER,
|
||||
"output_tokens" INTEGER,
|
||||
"token_details" TEXT,
|
||||
"schema_id" TEXT REFERENCES "schemas"("id"),
|
||||
"resolved_model" TEXT,
|
||||
"reasoning" TEXT
|
||||
);
|
||||
CREATE VIRTUAL TABLE "responses_fts" USING FTS5 (
|
||||
"prompt",
|
||||
"response",
|
||||
content="responses"
|
||||
);
|
||||
CREATE TABLE "attachments" (
|
||||
"id" TEXT PRIMARY KEY,
|
||||
"type" TEXT,
|
||||
"path" TEXT,
|
||||
"url" TEXT,
|
||||
"content" BLOB
|
||||
);
|
||||
CREATE TABLE "prompt_attachments" (
|
||||
"response_id" TEXT REFERENCES "responses"("id"),
|
||||
"attachment_id" TEXT REFERENCES "attachments"("id"),
|
||||
"order" INTEGER,
|
||||
PRIMARY KEY ("response_id",
|
||||
"attachment_id")
|
||||
);
|
||||
CREATE TABLE "fragments" (
|
||||
"id" INTEGER PRIMARY KEY,
|
||||
"hash" TEXT,
|
||||
"content" TEXT,
|
||||
"datetime_utc" TEXT,
|
||||
"source" TEXT
|
||||
);
|
||||
CREATE TABLE "fragment_aliases" (
|
||||
"alias" TEXT PRIMARY KEY,
|
||||
"fragment_id" INTEGER REFERENCES "fragments"("id")
|
||||
);
|
||||
CREATE TABLE "prompt_fragments" (
|
||||
"response_id" TEXT REFERENCES "responses"("id"),
|
||||
"fragment_id" INTEGER REFERENCES "fragments"("id"),
|
||||
"order" INTEGER,
|
||||
PRIMARY KEY ("response_id",
|
||||
"fragment_id",
|
||||
"order")
|
||||
);
|
||||
CREATE TABLE "system_fragments" (
|
||||
"response_id" TEXT REFERENCES "responses"("id"),
|
||||
"fragment_id" INTEGER REFERENCES "fragments"("id"),
|
||||
"order" INTEGER,
|
||||
PRIMARY KEY ("response_id",
|
||||
"fragment_id",
|
||||
"order")
|
||||
);
|
||||
CREATE TABLE "tools" (
|
||||
"id" INTEGER PRIMARY KEY,
|
||||
"hash" TEXT,
|
||||
"name" TEXT,
|
||||
"description" TEXT,
|
||||
"input_schema" TEXT,
|
||||
"plugin" TEXT
|
||||
);
|
||||
CREATE TABLE "tool_responses" (
|
||||
"tool_id" INTEGER REFERENCES "tools"("id"),
|
||||
"response_id" TEXT REFERENCES "responses"("id"),
|
||||
PRIMARY KEY ("tool_id",
|
||||
"response_id")
|
||||
);
|
||||
CREATE TABLE "tool_calls" (
|
||||
"id" INTEGER PRIMARY KEY,
|
||||
"response_id" TEXT REFERENCES "responses"("id"),
|
||||
"tool_id" INTEGER REFERENCES "tools"("id"),
|
||||
"name" TEXT,
|
||||
"arguments" TEXT,
|
||||
"tool_call_id" TEXT
|
||||
);
|
||||
CREATE TABLE "tool_results" (
|
||||
"id" INTEGER PRIMARY KEY,
|
||||
"response_id" TEXT REFERENCES "responses"("id"),
|
||||
"tool_id" INTEGER REFERENCES "tools"("id"),
|
||||
"name" TEXT,
|
||||
"output" TEXT,
|
||||
"tool_call_id" TEXT,
|
||||
"instance_id" INTEGER REFERENCES "tool_instances"("id"),
|
||||
"exception" TEXT
|
||||
);
|
||||
CREATE TABLE "tool_instances" (
|
||||
"id" INTEGER PRIMARY KEY,
|
||||
"plugin" TEXT,
|
||||
"name" TEXT,
|
||||
"arguments" TEXT
|
||||
);
|
||||
```
|
||||
<!-- [[[end]]] -->
|
||||
`responses_fts` configures [SQLite full-text search](https://www.sqlite.org/fts5.html) against the `prompt` and `response` columns in the `responses` table.
|
||||
@@ -0,0 +1,188 @@
|
||||
(openai-models)=
|
||||
|
||||
# OpenAI models
|
||||
|
||||
LLM ships with a default plugin for talking to OpenAI's API. OpenAI offer both language models and embedding models, and LLM can access both types.
|
||||
|
||||
(openai-models-configuration)=
|
||||
|
||||
## Configuration
|
||||
|
||||
All OpenAI models are accessed using an API key. You can obtain one from [the API keys page](https://platform.openai.com/api-keys) on their site.
|
||||
|
||||
Once you have created a key, configure LLM to use it by running:
|
||||
|
||||
```bash
|
||||
llm keys set openai
|
||||
```
|
||||
Then paste in the API key.
|
||||
|
||||
(openai-models-language)=
|
||||
|
||||
## OpenAI language models
|
||||
|
||||
Run `llm models` for a full list of available models. The OpenAI models supported by LLM are:
|
||||
|
||||
<!-- [[[cog
|
||||
from click.testing import CliRunner
|
||||
from llm.cli import cli
|
||||
result = CliRunner().invoke(cli, ["models", "list"])
|
||||
models = [line for line in result.output.split("\n") if line.startswith("OpenAI ")]
|
||||
cog.out("```\n{}\n```".format("\n".join(models)))
|
||||
]]] -->
|
||||
```
|
||||
OpenAI Chat: gpt-4o (aliases: 4o)
|
||||
OpenAI Chat: chatgpt-4o-latest (aliases: chatgpt-4o)
|
||||
OpenAI Chat: gpt-4o-mini (aliases: 4o-mini)
|
||||
OpenAI Chat: gpt-4o-audio-preview
|
||||
OpenAI Chat: gpt-4o-audio-preview-2024-12-17
|
||||
OpenAI Chat: gpt-4o-audio-preview-2024-10-01
|
||||
OpenAI Chat: gpt-4o-mini-audio-preview
|
||||
OpenAI Chat: gpt-4o-mini-audio-preview-2024-12-17
|
||||
OpenAI Chat: gpt-4.1 (aliases: 4.1)
|
||||
OpenAI Chat: gpt-4.1-mini (aliases: 4.1-mini)
|
||||
OpenAI Chat: gpt-4.1-nano (aliases: 4.1-nano)
|
||||
OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt)
|
||||
OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k)
|
||||
OpenAI Chat: gpt-4 (aliases: 4, gpt4)
|
||||
OpenAI Chat: gpt-4-32k (aliases: 4-32k)
|
||||
OpenAI Chat: gpt-4-1106-preview
|
||||
OpenAI Chat: gpt-4-0125-preview
|
||||
OpenAI Chat: gpt-4-turbo-2024-04-09
|
||||
OpenAI Chat: gpt-4-turbo (aliases: gpt-4-turbo-preview, 4-turbo, 4t)
|
||||
OpenAI Chat: gpt-4.5-preview-2025-02-27
|
||||
OpenAI Chat: gpt-4.5-preview (aliases: gpt-4.5)
|
||||
OpenAI Responses: o1
|
||||
OpenAI Responses: o1-2024-12-17
|
||||
OpenAI Chat: o1-preview
|
||||
OpenAI Chat: o1-mini
|
||||
OpenAI Responses: o3-mini
|
||||
OpenAI Responses: o3
|
||||
OpenAI Responses: o4-mini
|
||||
OpenAI Responses: gpt-5
|
||||
OpenAI Responses: gpt-5-mini
|
||||
OpenAI Responses: gpt-5-nano
|
||||
OpenAI Responses: gpt-5-2025-08-07
|
||||
OpenAI Responses: gpt-5-mini-2025-08-07
|
||||
OpenAI Responses: gpt-5-nano-2025-08-07
|
||||
OpenAI Responses: gpt-5.1
|
||||
OpenAI Responses: gpt-5.1-chat-latest
|
||||
OpenAI Responses: gpt-5.2
|
||||
OpenAI Responses: gpt-5.2-chat-latest
|
||||
OpenAI Responses: gpt-5.4
|
||||
OpenAI Responses: gpt-5.4-2026-03-05
|
||||
OpenAI Responses: gpt-5.4-mini
|
||||
OpenAI Responses: gpt-5.4-mini-2026-03-17
|
||||
OpenAI Responses: gpt-5.4-nano
|
||||
OpenAI Responses: gpt-5.4-nano-2026-03-17
|
||||
OpenAI Responses: gpt-5.5
|
||||
OpenAI Responses: gpt-5.5-2026-04-23
|
||||
OpenAI Completion: gpt-3.5-turbo-instruct (aliases: 3.5-instruct, chatgpt-instruct)
|
||||
```
|
||||
<!-- [[[end]]] -->
|
||||
|
||||
See [the OpenAI models documentation](https://platform.openai.com/docs/models) for details of each of these.
|
||||
|
||||
`gpt-4o-mini` (aliased to `4o-mini`) is the least expensive model, and is the default for if you don't specify a model at all. Consult [OpenAI's model documentation](https://platform.openai.com/docs/models) for details of the other models.
|
||||
|
||||
[o1-pro](https://platform.openai.com/docs/models/o1-pro) is not available through the Chat Completions API used by LLM's default OpenAI plugin. You can install the new [llm-openai-plugin](https://github.com/simonw/llm-openai-plugin) plugin to access that model.
|
||||
|
||||
## Model features
|
||||
|
||||
The following features work with OpenAI models:
|
||||
|
||||
- {ref}`System prompts <usage-system-prompts>` can be used to provide instructions that have a higher weight than the prompt itself.
|
||||
- {ref}`Attachments <usage-attachments>`. Many OpenAI models support image inputs - check which ones using `llm models --options`. Any model that accepts images can also accept PDFs.
|
||||
- {ref}`Schemas <usage-schemas>` can be used to influence the JSON structure of the model output.
|
||||
- {ref}`Model options <usage-model-options>` can be used to set parameters like `temperature`. Use `llm models --options` for a full list of supported options.
|
||||
|
||||
(openai-models-embedding)=
|
||||
|
||||
## OpenAI embedding models
|
||||
|
||||
Run `llm embed-models` for a list of {ref}`embedding models <embeddings>`. The following OpenAI embedding models are supported by LLM:
|
||||
|
||||
```
|
||||
ada-002 (aliases: ada, oai)
|
||||
3-small
|
||||
3-large
|
||||
3-small-512
|
||||
3-large-256
|
||||
3-large-1024
|
||||
```
|
||||
|
||||
The `3-small` model is currently the most inexpensive. `3-large` costs more but is more capable - see [New embedding models and API updates](https://openai.com/blog/new-embedding-models-and-api-updates) on the OpenAI blog for details and benchmarks.
|
||||
|
||||
An important characteristic of any embedding model is the size of the vector it returns. Smaller vectors cost less to store and query, but may be less accurate.
|
||||
|
||||
OpenAI `3-small` and `3-large` vectors can be safely truncated to lower dimensions without losing too much accuracy. The `-int` models provided by LLM are pre-configured to do this, so `3-large-256` is the `3-large` model truncated to 256 dimensions.
|
||||
|
||||
The vector size of the supported OpenAI embedding models are as follows:
|
||||
|
||||
| Model | Size |
|
||||
| --- | --- |
|
||||
| ada-002 | 1536 |
|
||||
| 3-small | 1536 |
|
||||
| 3-large | 3072 |
|
||||
| 3-small-512 | 512 |
|
||||
| 3-large-256 | 256 |
|
||||
| 3-large-1024 | 1024 |
|
||||
|
||||
(openai-completion-models)=
|
||||
|
||||
## OpenAI completion models
|
||||
|
||||
The `gpt-3.5-turbo-instruct` model is a little different - it is a completion model rather than a chat model, described in [the OpenAI completions documentation](https://platform.openai.com/docs/api-reference/completions/create).
|
||||
|
||||
Completion models can be called with the `-o logprobs 3` option (not supported by chat models) which will cause LLM to store 3 log probabilities for each returned token in the SQLite database. Consult [this issue](https://github.com/simonw/llm/issues/284#issuecomment-1724772704) for details on how to read these values.
|
||||
|
||||
(openai-extra-models)=
|
||||
|
||||
## Adding more OpenAI models
|
||||
|
||||
OpenAI occasionally release new models with new names. LLM aims to ship new releases to support these, but you can also configure them directly, by adding them to a `extra-openai-models.yaml` configuration file.
|
||||
|
||||
Run this command to find the directory in which this file should be created:
|
||||
|
||||
```bash
|
||||
dirname "$(llm logs path)"
|
||||
```
|
||||
On my Mac laptop I get this:
|
||||
```
|
||||
~/Library/Application Support/io.datasette.llm
|
||||
```
|
||||
Create a file in that directory called `extra-openai-models.yaml`.
|
||||
|
||||
Let's say OpenAI have just released the `gpt-3.5-turbo-0613` model and you want to use it, despite LLM not yet shipping support. You could configure that by adding this to the file:
|
||||
|
||||
```yaml
|
||||
- model_id: gpt-3.5-turbo-0613
|
||||
model_name: gpt-3.5-turbo-0613
|
||||
aliases: ["0613"]
|
||||
```
|
||||
The `model_id` is the identifier that will be recorded in the LLM logs. You can use this to specify the model, or you can optionally include a list of aliases for that model. The `model_name` is the actual model identifier that will be passed to the API, which must match exactly what the API expects.
|
||||
|
||||
If the model is a completion model (such as `gpt-3.5-turbo-instruct`) add `completion: true` to the configuration.
|
||||
|
||||
If the model supports structured extraction using json_schema, add `supports_schema: true` to the configuration.
|
||||
|
||||
For reasoning models like `o1` or `o3-mini` add `reasoning: true`.
|
||||
|
||||
With this configuration in place, the following command should run a prompt against the new model:
|
||||
|
||||
```bash
|
||||
llm -m 0613 'What is the capital of France?'
|
||||
```
|
||||
Run `llm models` to confirm that the new model is now available:
|
||||
```bash
|
||||
llm models
|
||||
```
|
||||
Example output:
|
||||
```
|
||||
OpenAI Chat: gpt-3.5-turbo (aliases: 3.5, chatgpt)
|
||||
OpenAI Chat: gpt-3.5-turbo-16k (aliases: chatgpt-16k, 3.5-16k)
|
||||
OpenAI Chat: gpt-4 (aliases: 4, gpt4)
|
||||
OpenAI Chat: gpt-4-32k (aliases: 4-32k)
|
||||
OpenAI Chat: gpt-3.5-turbo-0613 (aliases: 0613)
|
||||
```
|
||||
Running `llm logs -n 1` should confirm that the prompt and response has been correctly logged to the database.
|
||||
@@ -0,0 +1,77 @@
|
||||
(other-models)=
|
||||
# Other models
|
||||
|
||||
LLM supports OpenAI models by default. You can install {ref}`plugins <plugins>` to add support for other models. You can also add additional OpenAI-API-compatible models {ref}`using a configuration file <openai-extra-models>`.
|
||||
|
||||
## Installing and using a local model
|
||||
|
||||
{ref}`LLM plugins <plugins>` can provide local models that run on your machine.
|
||||
|
||||
To install **[llm-gpt4all](https://github.com/simonw/llm-gpt4all)**, providing 17 models from the [GPT4All](https://gpt4all.io/) project, run this:
|
||||
|
||||
```bash
|
||||
llm install llm-gpt4all
|
||||
```
|
||||
Run `llm models` to see the expanded list of available models.
|
||||
|
||||
To run a prompt through one of the models from GPT4All specify it using `-m/--model`:
|
||||
```bash
|
||||
llm -m orca-mini-3b-gguf2-q4_0 'What is the capital of France?'
|
||||
```
|
||||
The model will be downloaded and cached the first time you use it.
|
||||
|
||||
Check the {ref}`plugin directory <plugin-directory>` for the latest list of available plugins for other models.
|
||||
|
||||
(openai-compatible-models)=
|
||||
|
||||
## OpenAI-compatible models
|
||||
|
||||
Projects such as [LocalAI](https://localai.io/) offer a REST API that imitates the OpenAI API but can be used to run other models, including models that can be installed on your own machine. These can be added using the same configuration mechanism.
|
||||
|
||||
The `model_id` is the name LLM will use for the model. The `model_name` is the name which needs to be passed to the API - this might differ from the `model_id`, especially if the `model_id` could potentially clash with other installed models.
|
||||
|
||||
The `api_base` key can be used to point the OpenAI client library at a different API endpoint.
|
||||
|
||||
To add the `orca-mini-3b` model hosted by a local installation of [LocalAI](https://localai.io/), add this to your `extra-openai-models.yaml` file:
|
||||
|
||||
```yaml
|
||||
- model_id: orca-openai-compat
|
||||
model_name: orca-mini-3b.ggmlv3
|
||||
api_base: "http://localhost:8080"
|
||||
```
|
||||
If the `api_base` is set, the existing configured `openai` API key will not be sent by default.
|
||||
|
||||
You can set `api_key_name` to the name of a key stored using the {ref}`api-keys` feature.
|
||||
|
||||
Other keys you can use here:
|
||||
|
||||
- `completion: true` for completion models that should use the `/completion` endpoint as opposed to `/completion/chat`
|
||||
- `supports_tools: true` for models that support tool calling
|
||||
- `can_stream: false` to disable streaming mode for models that cannot stream
|
||||
- `supports_schema: true` for models that support JSON structured schema output
|
||||
- `vision: true` for models that can accept images as input
|
||||
- `audio: true` for models that accept audio attachments
|
||||
|
||||
Having configured the model like this, run `llm models --options -m MODEL_ID` to check that it installed correctly. You can then run prompts against it like so:
|
||||
|
||||
```bash
|
||||
llm -m orca-openai-compat 'What is the capital of France?'
|
||||
```
|
||||
And confirm they were logged correctly with:
|
||||
```bash
|
||||
llm logs -n 1
|
||||
```
|
||||
|
||||
### Extra HTTP headers
|
||||
|
||||
Some providers such as [openrouter.ai](https://openrouter.ai/docs) may require the setting of additional HTTP headers. You can set those using the `headers:` key like this:
|
||||
|
||||
```yaml
|
||||
- model_id: claude
|
||||
model_name: anthropic/claude-2
|
||||
api_base: "https://openrouter.ai/api/v1"
|
||||
api_key_name: openrouter
|
||||
headers:
|
||||
HTTP-Referer: "https://llm.datasette.io/"
|
||||
X-Title: LLM
|
||||
```
|
||||
@@ -0,0 +1,614 @@
|
||||
(advanced-model-plugins)=
|
||||
# Advanced model plugins
|
||||
|
||||
The {ref}`model plugin tutorial <tutorial-model-plugin>` covers the basics of developing a plugin that adds support for a new model. This document covers more advanced topics.
|
||||
|
||||
Features to consider for your model plugin include:
|
||||
|
||||
- {ref}`Accepting API keys <advanced-model-plugins-api-keys>` using the standard mechanism that incorporates `llm keys set`, environment variables and support for passing an explicit key to the model.
|
||||
- Including support for {ref}`Async models <advanced-model-plugins-async>` that can be used with Python's `asyncio` library.
|
||||
- Support for {ref}`structured output <advanced-model-plugins-schemas>` using JSON schemas.
|
||||
- Support for {ref}`tools <advanced-model-plugins-tools>`.
|
||||
- Handling {ref}`attachments <advanced-model-plugins-attachments>` (images, audio and more) for multi-modal models.
|
||||
- Tracking {ref}`token usage <advanced-model-plugins-usage>` for models that charge by the token.
|
||||
|
||||
(advanced-model-plugins-lazy)=
|
||||
|
||||
## Tip: lazily load expensive dependencies
|
||||
|
||||
If your plugin depends on an expensive library such as [PyTorch](https://pytorch.org/) you should avoid importing that dependency (or a dependency that uses that dependency) at the top level of your module. Expensive imports in plugins mean that even simple commands like `llm --help` can take a long time to run.
|
||||
|
||||
Instead, move those imports to inside the methods that need them. Here's an example [change to llm-sentence-transformers](https://github.com/simonw/llm-sentence-transformers/commit/f87df71e8a652a8cb05ad3836a79b815bcbfa64b) that shaved 1.8 seconds off the time it took to run `llm --help`!
|
||||
|
||||
(advanced-model-plugins-api-keys)=
|
||||
|
||||
## Models that accept API keys
|
||||
|
||||
Models that call out to API providers such as OpenAI, Anthropic or Google Gemini usually require an API key.
|
||||
|
||||
LLM's API key management mechanism {ref}`is described here <api-keys>`.
|
||||
|
||||
If your plugin requires an API key you should subclass the `llm.KeyModel` class instead of the `llm.Model` class. Start your model definition like this:
|
||||
|
||||
```python
|
||||
import llm
|
||||
|
||||
class HostedModel(llm.KeyModel):
|
||||
needs_key = "hosted" # Required
|
||||
key_env_var = "HOSTED_API_KEY" # Optional
|
||||
```
|
||||
This tells LLM that your model requires an API key, which may be saved in the key registry under the key name `hosted` or might also be provided as the `HOSTED_API_KEY` environment variable.
|
||||
|
||||
Then when you define your `execute()` method it should take an extra `key=` parameter like this:
|
||||
|
||||
```python
|
||||
def execute(self, prompt, stream, response, conversation, key=None):
|
||||
# key= here will be the API key to use
|
||||
```
|
||||
LLM will pass in the key from the environment variable, key registry or that has been passed to LLM as the `--key` command-line option or the `model.prompt(..., key=)` parameter.
|
||||
|
||||
(advanced-model-plugins-async)=
|
||||
|
||||
## Async models
|
||||
|
||||
Plugins can optionally provide an asynchronous version of their model, suitable for use with Python [asyncio](https://docs.python.org/3/library/asyncio.html). This is particularly useful for remote models accessible by an HTTP API.
|
||||
|
||||
The async version of a model subclasses `llm.AsyncModel` instead of `llm.Model`. It must implement an `async def execute()` async generator method instead of `def execute()`.
|
||||
|
||||
This example shows a subset of the OpenAI default plugin illustrating how this method might work:
|
||||
|
||||
```python
|
||||
from typing import AsyncGenerator
|
||||
import llm
|
||||
|
||||
class MyAsyncModel(llm.AsyncModel):
|
||||
# This can duplicate the model_id of the sync model:
|
||||
model_id = "my-model-id"
|
||||
|
||||
async def execute(
|
||||
self, prompt, stream, response, conversation=None
|
||||
) -> AsyncGenerator[str, None]:
|
||||
if stream:
|
||||
completion = await client.chat.completions.create(
|
||||
model=self.model_id,
|
||||
messages=messages,
|
||||
stream=True,
|
||||
)
|
||||
async for chunk in completion:
|
||||
yield chunk.choices[0].delta.content
|
||||
else:
|
||||
completion = await client.chat.completions.create(
|
||||
model=self.model_name or self.model_id,
|
||||
messages=messages,
|
||||
stream=False,
|
||||
)
|
||||
if completion.choices[0].message.content is not None:
|
||||
yield completion.choices[0].message.content
|
||||
```
|
||||
If your model takes an API key you should instead subclass `llm.AsyncKeyModel` and have a `key=` parameter on your `.execute()` method:
|
||||
|
||||
```python
|
||||
class MyAsyncModel(llm.AsyncKeyModel):
|
||||
...
|
||||
async def execute(
|
||||
self, prompt, stream, response, conversation=None, key=None
|
||||
) -> AsyncGenerator[str, None]:
|
||||
```
|
||||
|
||||
This async model instance should then be passed to the `register()` method in the `register_models()` plugin hook:
|
||||
|
||||
```python
|
||||
@hookimpl
|
||||
def register_models(register):
|
||||
register(
|
||||
MyModel(), MyAsyncModel(), aliases=("my-model-aliases",)
|
||||
)
|
||||
```
|
||||
|
||||
The `prompt` object passed to your `execute()` method is an instance of {class}`~llm.Prompt`:
|
||||
|
||||
```{eval-rst}
|
||||
.. autoclass:: llm.Prompt
|
||||
:members: prompt, system
|
||||
:exclude-members: model, options
|
||||
```
|
||||
|
||||
(advanced-model-plugins-schemas)=
|
||||
|
||||
## Supporting schemas
|
||||
|
||||
If your model supports {ref}`structured output <schemas>` against a defined JSON schema you can implement support by first adding `supports_schema = True` to the class:
|
||||
|
||||
```python
|
||||
class MyModel(llm.KeyModel):
|
||||
...
|
||||
support_schema = True
|
||||
```
|
||||
And then adding code to your `.execute()` method that checks for `prompt.schema` and, if it is present, uses that to prompt the model.
|
||||
|
||||
`prompt.schema` will always be a Python dictionary representing a JSON schema, even if the user passed in a Pydantic model class.
|
||||
|
||||
Check the [llm-gemini](https://github.com/simonw/llm-gemini) and [llm-anthropic](https://github.com/simonw/llm-anthropic) plugins for example of this pattern in action.
|
||||
|
||||
(advanced-model-plugins-tools)=
|
||||
|
||||
## Supporting tools
|
||||
|
||||
Adding {ref}`tools support <tools>` involves several steps:
|
||||
|
||||
1. Add `supports_tools = True` to your model class.
|
||||
2. If `prompt.tools` is populated, turn that list of `llm.Tool` objects into the correct format for your model.
|
||||
3. Look out for requests to call tools in the responses from your model. Call `response.add_tool_call(llm.ToolCall(...))` for each of those. This should work for streaming and non-streaming and async and non-async cases. Pass the provider's tool call ID as `tool_call_id=` if there is one; if you omit it LLM synthesizes a unique `tc_`-prefixed id, since consumers rely on every tool call having one.
|
||||
4. If your prompt has a `prompt.tool_results` list, pass the information from those `llm.ToolResult` objects to your model.
|
||||
5. Include `prompt.tools` and `prompt.tool_results` and tool calls from `response.tool_calls_or_raise()` in the conversation history constructed by your plugin.
|
||||
6. Make sure your code is OK with prompts that do not have `prompt.prompt` set to a value, since they may be carrying exclusively the results of a tool call.
|
||||
|
||||
This [commit to llm-gemini](https://github.com/simonw/llm-gemini/commit/a7f1096cfbb733018eb41c29028a8cc6160be298) implementing tools helps demonstrate what this looks like for a real plugin.
|
||||
|
||||
Here are the relevant dataclasses:
|
||||
|
||||
```{eval-rst}
|
||||
.. autoclass:: llm.Tool
|
||||
|
||||
.. autoclass:: llm.ToolCall
|
||||
|
||||
.. autoclass:: llm.ToolResult
|
||||
```
|
||||
|
||||
|
||||
(advanced-model-plugins-attachments)=
|
||||
|
||||
## Attachments for multi-modal models
|
||||
|
||||
Models such as GPT-4o, Claude 3.5 Sonnet and Google's Gemini 1.5 are multi-modal: they accept input in the form of images and maybe even audio, video and other formats.
|
||||
|
||||
LLM calls these **attachments**. Models can specify the types of attachments they accept and then implement special code in the `.execute()` method to handle them.
|
||||
|
||||
See {ref}`the Python attachments documentation <python-api-attachments>` for details on using attachments in the Python API.
|
||||
|
||||
### Specifying attachment types
|
||||
|
||||
A `Model` subclass can list the types of attachments it accepts by defining a `attachment_types` class attribute:
|
||||
|
||||
```python
|
||||
class NewModel(llm.Model):
|
||||
model_id = "new-model"
|
||||
attachment_types = {
|
||||
"image/png",
|
||||
"image/jpeg",
|
||||
"image/webp",
|
||||
"image/gif",
|
||||
}
|
||||
```
|
||||
These content types are detected when an attachment is passed to LLM using `llm -a filename`, or can be specified by the user using the `--attachment-type filename image/png` option.
|
||||
|
||||
**Note:** MP3 files will have their attachment type detected as `audio/mpeg`, not `audio/mp3`.
|
||||
|
||||
LLM will use the `attachment_types` attribute to validate that provided attachments should be accepted before passing them to the model.
|
||||
|
||||
### Handling attachments
|
||||
|
||||
The `prompt` object passed to the `execute()` method will have an `attachments` attribute containing a list of `Attachment` objects provided by the user.
|
||||
|
||||
An `Attachment` instance has the following properties:
|
||||
|
||||
- `url (str)`: The URL of the attachment, if it was provided as a URL
|
||||
- `path (str)`: The resolved file path of the attachment, if it was provided as a file
|
||||
- `type (str)`: The content type of the attachment, if it was provided
|
||||
- `content (bytes)`: The binary content of the attachment, if it was provided
|
||||
|
||||
Generally only one of `url`, `path` or `content` will be set.
|
||||
|
||||
You should usually access the type and the content through one of these methods:
|
||||
|
||||
- `attachment.resolve_type() -> str`: Returns the `type` if it is available, otherwise attempts to guess the type by looking at the first few bytes of content
|
||||
- `attachment.content_bytes() -> bytes`: Returns the binary content, which it may need to read from a file or fetch from a URL
|
||||
- `attachment.base64_content() -> str`: Returns that content as a base64-encoded string
|
||||
|
||||
A `id()` method returns a database ID for this content, which is either a SHA256 hash of the binary content or, in the case of attachments hosted at an external URL, a hash of `{"url": url}` instead. This is an implementation detail which you should not need to access directly.
|
||||
|
||||
Note that it's possible for a prompt with an attachments to not include a text prompt at all, in which case `prompt.prompt` will be `None`.
|
||||
|
||||
Here's how the OpenAI plugin handles attachments, including the case where no `prompt.prompt` was provided:
|
||||
|
||||
```python
|
||||
if not prompt.attachments:
|
||||
messages.append({"role": "user", "content": prompt.prompt})
|
||||
else:
|
||||
attachment_message = []
|
||||
if prompt.prompt:
|
||||
attachment_message.append({"type": "text", "text": prompt.prompt})
|
||||
for attachment in prompt.attachments:
|
||||
attachment_message.append(_attachment(attachment))
|
||||
messages.append({"role": "user", "content": attachment_message})
|
||||
|
||||
|
||||
# And the code for creating the attachment message
|
||||
def _attachment(attachment):
|
||||
url = attachment.url
|
||||
base64_content = ""
|
||||
if not url or attachment.resolve_type().startswith("audio/"):
|
||||
base64_content = attachment.base64_content()
|
||||
url = f"data:{attachment.resolve_type()};base64,{base64_content}"
|
||||
if attachment.resolve_type().startswith("image/"):
|
||||
return {"type": "image_url", "image_url": {"url": url}}
|
||||
else:
|
||||
format_ = "wav" if attachment.resolve_type() == "audio/wav" else "mp3"
|
||||
return {
|
||||
"type": "input_audio",
|
||||
"input_audio": {
|
||||
"data": base64_content,
|
||||
"format": format_,
|
||||
},
|
||||
}
|
||||
```
|
||||
As you can see, it uses `attachment.url` if that is available and otherwise falls back to using the `base64_content()` method to embed the image directly in the JSON sent to the API. For the OpenAI API audio attachments are always included as base64-encoded strings.
|
||||
|
||||
### Attachments from previous conversations
|
||||
|
||||
Conversation history — including attachments from prior turns — is available on the canonical `prompt.messages` list. See the [next section](#structured-messages-streaming) for how that works.
|
||||
|
||||
(structured-messages-streaming)=
|
||||
|
||||
## Structured messages and streaming events
|
||||
|
||||
The 0.32 alpha introduced a richer contract for plugins than "yield strings":
|
||||
|
||||
1. **`execute()` yields `StreamEvent` objects** (or plain `str`, still supported) so text, reasoning (thinking tokens), tool calls, and server-side tool results each surface as their own event type. The framework assembles these into typed `Part` objects.
|
||||
2. **`build_messages` (or equivalent) reads `prompt.messages`** — a `list[llm.Message]` that is the complete input chain for this turn.
|
||||
3. **Opaque provider tokens round-trip via `provider_metadata`** — Anthropic thinking signatures, Gemini thought signatures, OpenAI Responses API encrypted reasoning blobs. Plugins stash whatever the API returns, then echo it back on the next request.
|
||||
|
||||
**Older plugins still work.** A plugin that still yields plain `str` from `execute()` works unchanged — each string is wrapped as a `StreamEvent(type="text", chunk=...)` internally.
|
||||
|
||||
### Yielding StreamEvent from execute()
|
||||
|
||||
```python
|
||||
from llm.parts import StreamEvent
|
||||
|
||||
def execute(self, prompt, stream, response, conversation, key=None):
|
||||
messages = self.build_messages(prompt, conversation)
|
||||
...
|
||||
|
||||
for chunk in provider_sdk.stream(...):
|
||||
if chunk.type == "text":
|
||||
yield StreamEvent(type="text", chunk=chunk.text)
|
||||
elif chunk.type == "thinking":
|
||||
yield StreamEvent(type="reasoning", chunk=chunk.text)
|
||||
```
|
||||
|
||||
A `StreamEvent` has four frequently-used fields:
|
||||
|
||||
- **`type`** — one of `"text"`, `"reasoning"`, `"tool_call_name"`, `"tool_call_args"`, `"tool_result"`.
|
||||
- **`chunk`** — the text fragment. For tool calls this is the tool name (for `tool_call_name`) or a partial JSON string (for `tool_call_args`).
|
||||
- **`tool_call_id`** — the provider's id for the tool call, set on `tool_call_name` / `tool_call_args` / `tool_result` events. Also the signal the framework uses to group tool-call events into one `ToolCallPart`.
|
||||
- **`provider_metadata`** — an optional `dict[str, dict]` namespaced by provider name. Carries opaque data (signatures, encrypted blobs) that must be echoed back on future requests.
|
||||
|
||||
Three additional fields exist for special cases:
|
||||
|
||||
- **`server_executed: bool`** — set `True` for server-side tool calls (for example, Anthropic web search) and their results. This means the model ran the tool internally as part of responding to the prompt.
|
||||
- **`tool_name`** — set on `tool_result` events to identify which tool this result came from.
|
||||
- **`part_index: int | None`** — defaults to `None`, which means "let the framework decide which Part this event belongs to." Pass an explicit integer only when you need to override the default grouping (see [below](#part-index-overrides)).
|
||||
|
||||
### How events group into Parts
|
||||
|
||||
When you leave `part_index` as `None` (the default), the framework groups events using these rules:
|
||||
|
||||
- **Consecutive same-family events concatenate.** Two `text` events in a row become one `TextPart`. Two `reasoning` events in a row become one `ReasoningPart`. A family transition (text → reasoning, or reasoning → text) starts a new Part.
|
||||
- **Tool calls group by `tool_call_id`.** A `tool_call_name` and any number of `tool_call_args` events sharing a `tool_call_id` combine into one `ToolCallPart` — even if they're interleaved with other events (parallel tool calls).
|
||||
- **`tool_result` is always its own Part**, paired to the originating call by `tool_call_id`.
|
||||
|
||||
| Stream | Resulting Parts |
|
||||
|-------------------------------------------|----------------------------------------------------------|
|
||||
| `text` × N | one `TextPart` |
|
||||
| `reasoning` × N, then `text` × N | `ReasoningPart`, `TextPart` |
|
||||
| `text`, `tool_call_name`+`args`, `text` | `TextPart`, `ToolCallPart`, `TextPart` |
|
||||
| Parallel tool calls (interleaved by id) | one `ToolCallPart` per distinct `tool_call_id` |
|
||||
| `reasoning`, tool call, `reasoning` | `ReasoningPart`, `ToolCallPart`, `ReasoningPart` |
|
||||
|
||||
(part-index-overrides)=
|
||||
### Setting `part_index` explicitly
|
||||
|
||||
In rare cases you'll want to override the default grouping:
|
||||
|
||||
- **Forcing a single TextPart across non-adjacent text bursts.** If your provider interleaves text deltas with tool calls but you want all the text concatenated into one `TextPart`, pass `part_index=0` on every text event. (The default behavior produces separate `TextPart`s on each side of the tool calls — usually what you want, but not always.)
|
||||
- **Tool-call args arriving before the id.** If your provider streams args before the `tool_call_id` is known, assign your own index per logical tool call and pass it on each event of that call.
|
||||
|
||||
You can mix explicit indices with `None` in the same stream — the framework reserves your explicit values and decides the rest.
|
||||
|
||||
(advanced-model-plugins-reasoning-tokens)=
|
||||
### Reasoning tokens
|
||||
|
||||
For streamed reasoning text:
|
||||
|
||||
```python
|
||||
yield StreamEvent(type="reasoning", chunk=text_chunk)
|
||||
```
|
||||
|
||||
Reasoning events that appear before/after text events become distinct `ReasoningPart` and `TextPart` entries in `response.messages` automatically. If your provider emits two thinking blocks separated by a tool call, you'll get two `ReasoningPart`s.
|
||||
|
||||
Plugins should respect `prompt.hide_reasoning`. This is set when the caller passes `hide_reasoning=True` to `model.prompt()`, `conversation.prompt()`, `model.chain()`, `conversation.chain()`, or their async counterparts. It is also set by the CLI `-R/--hide-reasoning` option.
|
||||
|
||||
`prompt.hide_reasoning` means "hide visible reasoning output", not "disable model reasoning". If your provider requires an explicit request for visible reasoning summaries, do not request those summaries when `prompt.hide_reasoning` is true:
|
||||
|
||||
```python
|
||||
kwargs = {}
|
||||
if not prompt.hide_reasoning:
|
||||
kwargs["reasoning"] = {"summary": "auto"}
|
||||
```
|
||||
|
||||
If your provider emits reasoning blocks regardless of request parameters, keep yielding those reasoning events as usual:
|
||||
|
||||
```python
|
||||
if chunk.type == "thinking":
|
||||
yield StreamEvent(type="reasoning", chunk=chunk.text)
|
||||
```
|
||||
|
||||
LLM's display layers use `prompt.hide_reasoning` to avoid showing those events to the user, while still allowing the framework to persist `ReasoningPart` objects and provider metadata for logs, serialization, and future turns.
|
||||
|
||||
### Tool calls
|
||||
|
||||
Each tool call emits two event types sharing a `tool_call_id`:
|
||||
|
||||
```python
|
||||
yield StreamEvent(
|
||||
type="tool_call_name",
|
||||
chunk=tool_name,
|
||||
tool_call_id=tool_call_id,
|
||||
)
|
||||
# then, as the provider streams JSON args:
|
||||
yield StreamEvent(
|
||||
type="tool_call_args",
|
||||
chunk=partial_json_fragment,
|
||||
tool_call_id=tool_call_id,
|
||||
)
|
||||
```
|
||||
|
||||
The framework groups them by `tool_call_id` — so parallel tool calls (where args for tool A and tool B interleave on the wire) work without any per-call index tracking. Some providers (Gemini) emit the complete tool call in one chunk — it's OK to emit both events back-to-back with the full name and full JSON.
|
||||
|
||||
For client-side tool calls — tools that LLM should execute locally in a chain — **also call `response.add_tool_call()`**. The chain-execution path (`response.tool_calls()` → `execute_tool_calls()`) reads from the explicitly-added list, not from the StreamEvent buffer.
|
||||
|
||||
```python
|
||||
response.add_tool_call(
|
||||
llm.ToolCall(
|
||||
tool_call_id=tool_id,
|
||||
name=tool_name,
|
||||
arguments=parsed_args,
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
### Server-side tool calls
|
||||
|
||||
For tools the API executes internally, set `server_executed=True` on the events. Anthropic web search is an example: the API returns a `server_tool_use` block for the search request, followed by a `web_search_tool_result` block containing the result payload.
|
||||
|
||||
```python
|
||||
yield StreamEvent(
|
||||
type="tool_call_name",
|
||||
chunk="web_search",
|
||||
tool_call_id=tool_id,
|
||||
server_executed=True,
|
||||
)
|
||||
yield StreamEvent(
|
||||
type="tool_call_args",
|
||||
chunk=json.dumps(query_args),
|
||||
tool_call_id=tool_id,
|
||||
server_executed=True,
|
||||
)
|
||||
```
|
||||
|
||||
The tool *result* (for example, the search hits) is also emitted as an event:
|
||||
|
||||
```python
|
||||
yield StreamEvent(
|
||||
type="tool_result",
|
||||
chunk=human_readable_summary,
|
||||
tool_call_id=tool_id,
|
||||
server_executed=True,
|
||||
tool_name="web_search",
|
||||
provider_metadata={"myprovider": {"raw_content": full_payload}},
|
||||
)
|
||||
```
|
||||
|
||||
For providers that don't stream server-tool-result contents (Anthropic's `web_search_tool_result` blocks only arrive in the final message), emit those results as a post-stream step. After the main iteration loop completes, inspect the final message and emit tool_result events for any server-side results.
|
||||
|
||||
Do **not** call `response.add_tool_call()` for server-side tool calls. This method should only be used for tool calls that need to be executed locally by the framework.
|
||||
|
||||
### Opaque provider metadata
|
||||
|
||||
Some providers require you to echo back opaque fields on the next request for multi-turn continuity to work:
|
||||
|
||||
- **Anthropic** — `signature` on each thinking block; `encrypted_content` inside web_search_tool_result items.
|
||||
- **Google Gemini** — `thoughtSignature` on `functionCall` parts when thinking is active.
|
||||
- **OpenAI Responses API** — `encrypted_content` on reasoning items in stateless mode.
|
||||
|
||||
These values are attached to a `StreamEvent` via its `provider_metadata` field. The framework merges metadata across events that group into the same Part (last non-None wins per top-level key) and persists it on the finalized Part.
|
||||
|
||||
Namespace under your provider's name so transcripts that mix providers don't collide:
|
||||
|
||||
```python
|
||||
# Anthropic signature arrives at the end of a thinking block.
|
||||
yield StreamEvent(
|
||||
type="reasoning",
|
||||
chunk="",
|
||||
provider_metadata={"anthropic": {"signature": sig}},
|
||||
)
|
||||
```
|
||||
|
||||
```python
|
||||
# Gemini attaches thoughtSignature to a functionCall part.
|
||||
yield StreamEvent(
|
||||
type="tool_call_name",
|
||||
chunk=name,
|
||||
tool_call_id=tc_id,
|
||||
provider_metadata={"gemini": {"thoughtSignature": sig}},
|
||||
)
|
||||
```
|
||||
|
||||
The framework round-trips the value verbatim via JSON, so use JSON-safe primitives (string, int, bool, dict, list) for provider metadata - use base64 encoding if you need to store binary data.
|
||||
|
||||
### Non-streaming path
|
||||
|
||||
When `stream=False` (or the provider returns a complete message at once), emit one event per content block.
|
||||
|
||||
```python
|
||||
else:
|
||||
completion = client.messages.create(**kwargs)
|
||||
response.response_json = completion.model_dump()
|
||||
for block in completion.content:
|
||||
if block.type == "thinking":
|
||||
yield StreamEvent(
|
||||
type="reasoning",
|
||||
chunk=block.thinking,
|
||||
provider_metadata={"anthropic": {"signature": block.signature}},
|
||||
)
|
||||
elif block.type == "text":
|
||||
yield StreamEvent(type="text", chunk=block.text)
|
||||
elif block.type == "tool_use":
|
||||
yield StreamEvent(
|
||||
type="tool_call_name",
|
||||
chunk=block.name,
|
||||
tool_call_id=block.id,
|
||||
)
|
||||
yield StreamEvent(
|
||||
type="tool_call_args",
|
||||
chunk=json.dumps(block.input),
|
||||
tool_call_id=block.id,
|
||||
)
|
||||
```
|
||||
|
||||
## Consuming prompt.messages in build_messages
|
||||
|
||||
`prompt.messages` is an `list[llm.Message]` that is always **the complete input chain for this turn** — whether the caller supplied it explicitly via `model.prompt(messages=[...])`, or it was synthesized from kwargs (`prompt=`, `system=`, `attachments=`, `tool_results=`), or it was pre-built by a `Conversation` or by `response.reply()`.
|
||||
|
||||
**Do not also walk `conversation.responses`.** History is already baked into `prompt.messages`; walking the conversation would double-emit.
|
||||
|
||||
A plugin's `build_messages` (or equivalent) iterates `prompt.messages` and dispatches per `Part` subtype:
|
||||
|
||||
```python
|
||||
from llm.parts import (
|
||||
TextPart,
|
||||
ReasoningPart,
|
||||
ToolCallPart,
|
||||
ToolResultPart,
|
||||
AttachmentPart,
|
||||
)
|
||||
|
||||
def build_messages(self, prompt, conversation):
|
||||
messages = []
|
||||
for msg in prompt.messages:
|
||||
if msg.role == "system":
|
||||
# Some APIs put system on a separate kwarg (Anthropic, Gemini).
|
||||
# OpenAI-style APIs emit it as a message; handle accordingly.
|
||||
continue
|
||||
self._append_message(messages, msg)
|
||||
return messages
|
||||
|
||||
def _append_message(self, out, msg):
|
||||
# Map llm's role to the provider's (assistant→model for Gemini,
|
||||
# tool→user for Anthropic/Gemini tool_result convention, etc.)
|
||||
role = self._provider_role(msg.role)
|
||||
parts = []
|
||||
for part in msg.parts:
|
||||
if isinstance(part, TextPart):
|
||||
parts.append({"type": "text", "text": part.text})
|
||||
elif isinstance(part, ReasoningPart):
|
||||
# Skip redacted reasoning (no content to echo back).
|
||||
if part.redacted or not part.text:
|
||||
continue
|
||||
block = {"type": "thinking", "thinking": part.text}
|
||||
# Restore the signature from provider_metadata.
|
||||
sig = (part.provider_metadata or {}).get("anthropic", {}).get("signature")
|
||||
if sig:
|
||||
block["signature"] = sig
|
||||
parts.append(block)
|
||||
elif isinstance(part, ToolCallPart):
|
||||
parts.append({
|
||||
"type": "tool_use",
|
||||
"id": part.tool_call_id,
|
||||
"name": part.name,
|
||||
"input": part.arguments,
|
||||
})
|
||||
elif isinstance(part, ToolResultPart):
|
||||
parts.append({
|
||||
"type": "tool_result",
|
||||
"tool_use_id": part.tool_call_id,
|
||||
"content": part.output,
|
||||
})
|
||||
elif isinstance(part, AttachmentPart) and part.attachment:
|
||||
parts.append(self._attachment_block(part.attachment))
|
||||
# Merge with the previous message if roles match (some providers
|
||||
# require strict alternation between user and assistant).
|
||||
if out and out[-1]["role"] == role:
|
||||
out[-1]["content"].extend(parts)
|
||||
else:
|
||||
out.append({"role": role, "content": parts})
|
||||
```
|
||||
|
||||
## Restoring opaque metadata on subsequent requests
|
||||
|
||||
When a conversation continues, your `build_messages` walks prior-turn Parts via `prompt.messages`. Each Part's `provider_metadata` is a `dict[str, dict]` keyed by provider name — extract your namespace and fold the fields back into the outgoing request body:
|
||||
|
||||
```python
|
||||
if isinstance(part, ReasoningPart):
|
||||
block = {"type": "thinking", "thinking": part.text}
|
||||
pm = (part.provider_metadata or {}).get("anthropic", {})
|
||||
if "signature" in pm:
|
||||
block["signature"] = pm["signature"]
|
||||
parts.append(block)
|
||||
|
||||
if isinstance(part, ToolCallPart):
|
||||
fc_part = {"function_call": {"name": part.name, "args": part.arguments}}
|
||||
pm = (part.provider_metadata or {}).get("gemini", {})
|
||||
if "thoughtSignature" in pm:
|
||||
# Gemini expects thoughtSignature beside function_call,
|
||||
# not nested inside it.
|
||||
fc_part["thoughtSignature"] = pm["thoughtSignature"]
|
||||
parts.append(fc_part)
|
||||
```
|
||||
|
||||
If the key is missing (an older transcript that pre-dates your plugin's support), fall through — don't fail. Treat other providers' entries as opaque; don't parse them.
|
||||
|
||||
(advanced-model-plugins-usage)=
|
||||
|
||||
## Tracking token usage
|
||||
|
||||
Models that charge by the token should track the number of tokens used by each prompt. The ``response.set_usage()`` method can be used to record the number of tokens used by a response - these will then be made available through the Python API and logged to the SQLite database for command-line users.
|
||||
|
||||
`response` here is the response object that is passed to `.execute()` as an argument.
|
||||
|
||||
Call ``response.set_usage()`` at the end of your `.execute()` method. It accepts keyword arguments `input=`, `output=` and `details=` - all three are optional. `input` and `output` should be integers, and `details` should be a dictionary that provides additional information beyond the input and output token counts.
|
||||
|
||||
This example logs 15 input tokens, 340 output tokens and notes that 37 tokens were cached:
|
||||
|
||||
```python
|
||||
response.set_usage(input=15, output=340, details={"cached": 37})
|
||||
```
|
||||
(advanced-model-plugins-resolved-model)=
|
||||
|
||||
## Tracking resolved model names
|
||||
|
||||
In some cases the model ID that the user requested may not be the exact model that is executed. Many providers have a `model-latest` alias which may execute different models over time.
|
||||
|
||||
If those APIs return the _real_ model ID that was used, your plugin can record that in the `resources.resolved_model` column in the logs by calling this method and passing the string representing the resolved, final model ID:
|
||||
|
||||
```bash
|
||||
response.set_resolved_model(resolved_model_id)
|
||||
```
|
||||
This string will be recorded in the database and shown in the output of `llm logs` and `llm logs --json`.
|
||||
|
||||
(tutorial-model-plugin-raise-errors)=
|
||||
|
||||
## LLM_RAISE_ERRORS
|
||||
|
||||
While working on a plugin it can be useful to request that errors are raised instead of being caught and logged, so you can access them from the Python debugger.
|
||||
|
||||
Set the `LLM_RAISE_ERRORS` environment variable to enable this behavior, then run `llm` like this:
|
||||
|
||||
```bash
|
||||
LLM_RAISE_ERRORS=1 python -i -m llm ...
|
||||
```
|
||||
The `-i` option means Python will drop into an interactive shell if an error occurs. You can then open a debugger at the most recent error using:
|
||||
|
||||
```python
|
||||
import pdb; pdb.pm()
|
||||
```
|
||||
@@ -0,0 +1,96 @@
|
||||
(plugin-directory)=
|
||||
# Plugin directory
|
||||
|
||||
The following plugins are available for LLM. Here's {ref}`how to install them <installing-plugins>`.
|
||||
|
||||
(plugin-directory-local-models)=
|
||||
## Local models
|
||||
|
||||
These plugins all help you run LLMs directly on your own computer:
|
||||
|
||||
- **[llm-gguf](https://github.com/simonw/llm-gguf)** uses [llama.cpp](https://github.com/ggerganov/llama.cpp) to run models published in the GGUF format.
|
||||
- **[llm-mlx](https://github.com/simonw/llm-mlx)** (Mac only) uses Apple's MLX framework to provide extremely high performance access to a large number of local models.
|
||||
- **[llm-ollama](https://github.com/taketwo/llm-ollama)** adds support for local models run using [Ollama](https://ollama.ai/).
|
||||
- **[llm-llamafile](https://github.com/simonw/llm-llamafile)** adds support for local models that are running locally using [llamafile](https://github.com/Mozilla-Ocho/llamafile).
|
||||
- **[llm-mlc](https://github.com/simonw/llm-mlc)** can run local models released by the [MLC project](https://mlc.ai/mlc-llm/), including models that can take advantage of the GPU on Apple Silicon M1/M2 devices.
|
||||
- **[llm-gpt4all](https://github.com/simonw/llm-gpt4all)** adds support for various models released by the [GPT4All](https://gpt4all.io/) project that are optimized to run locally on your own machine. These models include versions of Vicuna, Orca, Falcon and MPT - here's [a full list of models](https://observablehq.com/@simonw/gpt4all-models).
|
||||
- **[llm-mpt30b](https://github.com/simonw/llm-mpt30b)** adds support for the [MPT-30B](https://huggingface.co/mosaicml/mpt-30b) local model.
|
||||
|
||||
(plugin-directory-remote-apis)=
|
||||
## Remote APIs
|
||||
|
||||
These plugins can be used to interact with remotely hosted models via their API:
|
||||
|
||||
- **[llm-mistral](https://github.com/simonw/llm-mistral)** adds support for [Mistral AI](https://mistral.ai/)'s language and embedding models.
|
||||
- **[llm-gemini](https://github.com/simonw/llm-gemini)** adds support for Google's [Gemini](https://ai.google.dev/docs) models.
|
||||
- **[llm-anthropic](https://github.com/simonw/llm-anthropic)** supports Anthropic's [Claude 3 family](https://www.anthropic.com/news/claude-3-family), [3.5 Sonnet](https://www.anthropic.com/news/claude-3-5-sonnet) and beyond.
|
||||
- **[llm-command-r](https://github.com/simonw/llm-command-r)** supports Cohere's Command R and [Command R Plus](https://txt.cohere.com/command-r-plus-microsoft-azure/) API models.
|
||||
- **[llm-reka](https://github.com/simonw/llm-reka)** supports the [Reka](https://www.reka.ai/) family of models via their API.
|
||||
- **[llm-perplexity](https://github.com/hex/llm-perplexity)** by Alexandru Geana supports the [Perplexity Labs](https://docs.perplexity.ai/) API models, including `llama-3-sonar-large-32k-online` which can search for things online and `llama-3-70b-instruct`.
|
||||
- **[llm-groq](https://github.com/angerman/llm-groq)** by Moritz Angermann provides access to fast models hosted by [Groq](https://console.groq.com/docs/models).
|
||||
- **[llm-grok](https://github.com/Hiepler/llm-grok)** by Benedikt Hiepler providing access to Grok model using the xAI API [Grok](https://x.ai/api).
|
||||
- **[llm-anyscale-endpoints](https://github.com/simonw/llm-anyscale-endpoints)** supports models hosted on the [Anyscale Endpoints](https://app.endpoints.anyscale.com/) platform, including Llama 2 70B.
|
||||
- **[llm-replicate](https://github.com/simonw/llm-replicate)** adds support for remote models hosted on [Replicate](https://replicate.com/), including Llama 2 from Meta AI.
|
||||
- **[llm-fireworks](https://github.com/simonw/llm-fireworks)** supports models hosted by [Fireworks AI](https://fireworks.ai/).
|
||||
- **[llm-openrouter](https://github.com/simonw/llm-openrouter)** provides access to models hosted on [OpenRouter](https://openrouter.ai/).
|
||||
- **[llm-cohere](https://github.com/Accudio/llm-cohere)** by Alistair Shepherd provides `cohere-generate` and `cohere-summarize` API models, powered by [Cohere](https://cohere.com/).
|
||||
- **[llm-bedrock](https://github.com/simonw/llm-bedrock)** adds support for Nova by Amazon via Amazon Bedrock.
|
||||
- **[llm-bedrock-anthropic](https://github.com/sblakey/llm-bedrock-anthropic)** by Sean Blakey adds support for Claude and Claude Instant by Anthropic via Amazon Bedrock.
|
||||
- **[llm-bedrock-meta](https://github.com/flabat/llm-bedrock-meta)** by Fabian Labat adds support for Llama 2 and Llama 3 by Meta via Amazon Bedrock.
|
||||
- **[llm-together](https://github.com/wearedevx/llm-together)** adds support for the [Together AI](https://www.together.ai/) extensive family of hosted openly licensed models.
|
||||
- **[llm-deepseek](https://github.com/abrasumente233/llm-deepseek)** adds support for the [DeepSeek](https://deepseek.com)'s DeepSeek-Chat and DeepSeek-Coder models.
|
||||
- **[llm-lambda-labs](https://github.com/simonw/llm-lambda-labs)** provides access to models hosted by [Lambda Labs](https://docs.lambdalabs.com/public-cloud/lambda-chat-api/), including the Nous Hermes 3 series.
|
||||
- **[llm-venice](https://github.com/ar-jan/llm-venice)** provides access to uncensored models hosted by privacy-focused [Venice AI](https://docs.venice.ai/), including Llama 3.1 405B.
|
||||
|
||||
If an API model host provides an OpenAI-compatible API you can also [configure LLM to talk to it](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models) without needing an extra plugin.
|
||||
|
||||
(plugin-directory-tools)=
|
||||
## Tools
|
||||
|
||||
The following plugins add new {ref}`tools <tools>` that can be used by models:
|
||||
|
||||
- **[llm-tools-simpleeval](https://github.com/simonw/llm-tools-simpleeval)** implements simple expression support for things like mathematics.
|
||||
- **[llm-tools-quickjs](https://github.com/simonw/llm-tools-quickjs)** provides access to a sandboxed QuickJS JavaScript interpreter, allowing LLMs to run JavaScript code. The environment persists between calls so the model can set variables and build functions and reuse them later on.
|
||||
- **[llm-tools-sqlite](https://github.com/simonw/llm-tools-sqlite)** can run read-only SQL queries against local SQLite databases.
|
||||
- **[llm-tools-datasette](https://github.com/simonw/llm-tools-datasette)** can run SQL queries against a remote [Datasette](https://datasette.io/) instance.
|
||||
- **[llm-tools-exa](https://github.com/daturkel/llm-tools-exa)** by Dan Turkel can perform web searches and question-answering using [exa.ai](https://exa.ai/).
|
||||
- **[llm-tools-rag](https://github.com/daturkel/llm-tools-rag)** by Dan Turkel can perform searches over your LLM embedding collections for simple RAG.
|
||||
|
||||
(plugin-directory-loaders)=
|
||||
## Fragments and template loaders
|
||||
|
||||
{ref}`LLM 0.24 <v0_24>` introduced support for plugins that define `-f prefix:value` or `-t prefix:value` custom loaders for fragments and templates.
|
||||
|
||||
- **[llm-video-frames](https://github.com/simonw/llm-video-frames)** uses `ffmpeg` to turn a video into a sequence of JPEG frames suitable for feeding into a vision model that doesn't support video inputs: `llm -f video-frames:video.mp4 'describe the key scenes in this video'`.
|
||||
- **[llm-templates-github](https://github.com/simonw/llm-templates-github)** supports loading templates shared on GitHub, e.g. `llm -t gh:simonw/pelican-svg`.
|
||||
- **[llm-templates-fabric](https://github.com/simonw/llm-templates-fabric)** provides access to the [Fabric](https://github.com/danielmiessler/fabric) collection of prompts: `cat setup.py | llm -t fabric:explain_code`.
|
||||
- **[llm-fragments-github](https://github.com/simonw/llm-fragments-github)** can load entire GitHub repositories in a single operation: `llm -f github:simonw/files-to-prompt 'explain this code'`. It can also fetch issue threads as Markdown using `llm -f issue:https://github.com/simonw/llm-fragments-github/issues/3`.
|
||||
- **[llm-hacker-news](https://github.com/simonw/llm-hacker-news)** imports conversations from Hacker News as fragments: `llm -f hn:43615912 'summary with illustrative direct quotes'`.
|
||||
- **[llm-fragments-pypi](https://github.com/samueldg/llm-fragments-pypi)** loads [PyPI](https://pypi.org/) packages' description and metadata as fragments: `llm -f pypi:ruff "What flake8 plugins does ruff re-implement?"`.
|
||||
- **[llm-fragments-pdf](https://github.com/daturkel/llm-fragments-pdf)** by Dan Turkel converts PDFs to markdown with [PyMuPDF4LLM](https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/index.html) to use as fragments: `llm -f pdf:something.pdf "what's this about?"`.
|
||||
- **[llm-fragments-site-text](https://github.com/daturkel/llm-fragments-site-text)** by Dan Turkel converts websites to markdown with [Trafilatura](https://trafilatura.readthedocs.io/en/latest/) to use as fragments: `llm -f site:https://example.com "summarize this"`.
|
||||
- **[llm-fragments-reader](https://github.com/simonw/llm-fragments-reader)** runs a URL theough the Jina Reader API: `llm -f 'reader:https://simonwillison.net/tags/jina/' summary`.
|
||||
|
||||
(plugin-directory-embeddings)=
|
||||
## Embedding models
|
||||
|
||||
{ref}`Embedding models <embeddings>` are models that can be used to generate and store embedding vectors for text.
|
||||
|
||||
- **[llm-sentence-transformers](https://github.com/simonw/llm-sentence-transformers)** adds support for embeddings using the [sentence-transformers](https://www.sbert.net/) library, which provides access to [a wide range](https://www.sbert.net/docs/pretrained_models.html) of embedding models.
|
||||
- **[llm-clip](https://github.com/simonw/llm-clip)** provides the [CLIP](https://openai.com/research/clip) model, which can be used to embed images and text in the same vector space, enabling text search against images. See [Build an image search engine with llm-clip](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) for more on this plugin.
|
||||
- **[llm-embed-jina](https://github.com/simonw/llm-embed-jina)** provides Jina AI's [8K text embedding models](https://jina.ai/news/jina-ai-launches-worlds-first-open-source-8k-text-embedding-rivaling-openai/).
|
||||
- **[llm-embed-onnx](https://github.com/simonw/llm-embed-onnx)** provides seven embedding models that can be executed using the ONNX model framework.
|
||||
|
||||
(plugin-directory-commands)=
|
||||
## Extra commands
|
||||
|
||||
- **[llm-cmd](https://github.com/simonw/llm-cmd)** accepts a prompt for a shell command, runs that prompt and populates the result in your shell so you can review it, edit it and then hit `<enter>` to execute or `ctrl+c` to cancel.
|
||||
- **[llm-cmd-comp](https://github.com/CGamesPlay/llm-cmd-comp)** provides a key binding for your shell that will launch a chat to build the command. When ready, hit `<enter>` and it will go right back into your shell command line, so you can run it.
|
||||
- **[llm-python](https://github.com/simonw/llm-python)** adds a `llm python` command for running a Python interpreter in the same virtual environment as LLM. This is useful for debugging, and also provides a convenient way to interact with the LLM {ref}`python-api` if you installed LLM using Homebrew or `pipx`.
|
||||
- **[llm-cluster](https://github.com/simonw/llm-cluster)** adds a `llm cluster` command for calculating clusters for a collection of embeddings. Calculated clusters can then be passed to a Large Language Model to generate a summary description.
|
||||
- **[llm-jq](https://github.com/simonw/llm-jq)** lets you pipe in JSON data and a prompt describing a `jq` program, then executes the generated program against the JSON.
|
||||
|
||||
(plugin-directory-fun)=
|
||||
## Just for fun
|
||||
|
||||
- **[llm-markov](https://github.com/simonw/llm-markov)** adds a simple model that generates output using a [Markov chain](https://en.wikipedia.org/wiki/Markov_chain). This example is used in the tutorial [Writing a plugin to support a new model](https://llm.datasette.io/en/latest/plugins/tutorial-model-plugin.html).
|
||||
@@ -0,0 +1,22 @@
|
||||
(plugins)=
|
||||
# Plugins
|
||||
|
||||
LLM plugins can enhance LLM by making alternative Large Language Models available, either via API or by running the models locally on your machine.
|
||||
|
||||
Plugins can also add new commands to the `llm` CLI tool.
|
||||
|
||||
The {ref}`plugin directory <plugin-directory>` lists available plugins that you can install and use.
|
||||
|
||||
{ref}`tutorial-model-plugin` describes how to build a new plugin in detail.
|
||||
|
||||
```{toctree}
|
||||
---
|
||||
maxdepth: 3
|
||||
---
|
||||
installing-plugins
|
||||
directory
|
||||
plugin-hooks
|
||||
tutorial-model-plugin
|
||||
advanced-model-plugins
|
||||
plugin-utilities
|
||||
```
|
||||
@@ -0,0 +1,101 @@
|
||||
(installing-plugins)=
|
||||
# Installing plugins
|
||||
|
||||
Plugins must be installed in the same virtual environment as LLM itself.
|
||||
|
||||
You can find names of plugins to install in the {ref}`plugin directory <plugin-directory>`
|
||||
|
||||
Use the `llm install` command (a thin wrapper around `pip install`) to install plugins in the correct environment:
|
||||
```bash
|
||||
llm install llm-gpt4all
|
||||
```
|
||||
Plugins can be uninstalled with `llm uninstall`:
|
||||
```bash
|
||||
llm uninstall llm-gpt4all -y
|
||||
```
|
||||
The `-y` flag skips asking for confirmation.
|
||||
|
||||
You can see additional models that have been added by plugins by running:
|
||||
```bash
|
||||
llm models
|
||||
```
|
||||
Or add `--options` to include details of the options available for each model:
|
||||
```bash
|
||||
llm models --options
|
||||
```
|
||||
To run a prompt against a newly installed model, pass its name as the `-m/--model` option:
|
||||
```bash
|
||||
llm -m orca-mini-3b-gguf2-q4_0 'What is the capital of France?'
|
||||
```
|
||||
|
||||
## Listing installed plugins
|
||||
|
||||
Run `llm plugins` to list installed plugins:
|
||||
|
||||
```bash
|
||||
llm plugins
|
||||
```
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "llm-anthropic",
|
||||
"hooks": [
|
||||
"register_models"
|
||||
],
|
||||
"version": "0.11"
|
||||
},
|
||||
{
|
||||
"name": "llm-gguf",
|
||||
"hooks": [
|
||||
"register_commands",
|
||||
"register_models"
|
||||
],
|
||||
"version": "0.1a0"
|
||||
},
|
||||
{
|
||||
"name": "llm-clip",
|
||||
"hooks": [
|
||||
"register_commands",
|
||||
"register_embedding_models"
|
||||
],
|
||||
"version": "0.1"
|
||||
},
|
||||
{
|
||||
"name": "llm-cmd",
|
||||
"hooks": [
|
||||
"register_commands"
|
||||
],
|
||||
"version": "0.2a0"
|
||||
},
|
||||
{
|
||||
"name": "llm-gemini",
|
||||
"hooks": [
|
||||
"register_embedding_models",
|
||||
"register_models"
|
||||
],
|
||||
"version": "0.3"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
(llm-load-plugins)=
|
||||
## Running with a subset of plugins
|
||||
|
||||
By default, LLM will load all plugins that are installed in the same virtual environment as LLM itself.
|
||||
|
||||
You can control the set of plugins that is loaded using the `LLM_LOAD_PLUGINS` environment variable.
|
||||
|
||||
Set that to the empty string to disable all plugins:
|
||||
|
||||
```bash
|
||||
LLM_LOAD_PLUGINS='' llm ...
|
||||
```
|
||||
Or to a comma-separated list of plugin names to load only those plugins:
|
||||
|
||||
```bash
|
||||
LLM_LOAD_PLUGINS='llm-gpt4all,llm-cluster' llm ...
|
||||
```
|
||||
You can use the `llm plugins` command to check that it is working correctly:
|
||||
```
|
||||
LLM_LOAD_PLUGINS='' llm plugins
|
||||
```
|
||||
@@ -0,0 +1,68 @@
|
||||
import llm
|
||||
import random
|
||||
import time
|
||||
from typing import Optional
|
||||
from pydantic import field_validator, Field
|
||||
|
||||
|
||||
@llm.hookimpl
|
||||
def register_models(register):
|
||||
register(Markov())
|
||||
|
||||
|
||||
def build_markov_table(text):
|
||||
words = text.split()
|
||||
transitions = {}
|
||||
# Loop through all but the last word
|
||||
for i in range(len(words) - 1):
|
||||
word = words[i]
|
||||
next_word = words[i + 1]
|
||||
transitions.setdefault(word, []).append(next_word)
|
||||
return transitions
|
||||
|
||||
|
||||
def generate(transitions, length, start_word=None):
|
||||
all_words = list(transitions.keys())
|
||||
next_word = start_word or random.choice(all_words)
|
||||
for i in range(length):
|
||||
yield next_word
|
||||
options = transitions.get(next_word) or all_words
|
||||
next_word = random.choice(options)
|
||||
|
||||
|
||||
class Markov(llm.Model):
|
||||
model_id = "markov"
|
||||
can_stream = True
|
||||
|
||||
class Options(llm.Options):
|
||||
length: Optional[int] = Field(
|
||||
description="Number of words to generate", default=None
|
||||
)
|
||||
delay: Optional[float] = Field(
|
||||
description="Seconds to delay between each token", default=None
|
||||
)
|
||||
|
||||
@field_validator("length")
|
||||
def validate_length(cls, length):
|
||||
if length is None:
|
||||
return None
|
||||
if length < 2:
|
||||
raise ValueError("length must be >= 2")
|
||||
return length
|
||||
|
||||
@field_validator("delay")
|
||||
def validate_delay(cls, delay):
|
||||
if delay is None:
|
||||
return None
|
||||
if not 0 <= delay <= 10:
|
||||
raise ValueError("delay must be between 0 and 10")
|
||||
return delay
|
||||
|
||||
def execute(self, prompt, stream, response, conversation):
|
||||
text = prompt.prompt
|
||||
transitions = build_markov_table(text)
|
||||
length = prompt.options.length or 20
|
||||
for word in generate(transitions, length):
|
||||
yield word + " "
|
||||
if prompt.options.delay:
|
||||
time.sleep(prompt.options.delay)
|
||||
@@ -0,0 +1,6 @@
|
||||
[project]
|
||||
name = "llm-markov"
|
||||
version = "0.1"
|
||||
|
||||
[project.entry-points.llm]
|
||||
markov = "llm_markov"
|
||||
@@ -0,0 +1,292 @@
|
||||
(plugin-hooks)=
|
||||
# Plugin hooks
|
||||
|
||||
Plugins use **plugin hooks** to customize LLM's behavior. These hooks are powered by the [Pluggy plugin system](https://pluggy.readthedocs.io/).
|
||||
|
||||
Each plugin can implement one or more hooks using the @hookimpl decorator against one of the hook function names described on this page.
|
||||
|
||||
LLM imitates the Datasette plugin system. The [Datasette plugin documentation](https://docs.datasette.io/en/stable/writing_plugins.html) describes how plugins work.
|
||||
|
||||
(plugin-hooks-register-commands)=
|
||||
## register_commands(cli)
|
||||
|
||||
This hook adds new commands to the `llm` CLI tool - for example `llm extra-command`.
|
||||
|
||||
This example plugin adds a new `hello-world` command that prints "Hello world!":
|
||||
|
||||
```python
|
||||
from llm import hookimpl
|
||||
import click
|
||||
|
||||
@hookimpl
|
||||
def register_commands(cli):
|
||||
@cli.command(name="hello-world")
|
||||
def hello_world():
|
||||
"Print hello world"
|
||||
click.echo("Hello world!")
|
||||
```
|
||||
This new command will be added to `llm --help` and can be run using `llm hello-world`.
|
||||
|
||||
(plugin-hooks-register-models)=
|
||||
## register_models(register, model_aliases)
|
||||
|
||||
This hook can be used to register one or more additional models.
|
||||
|
||||
```python
|
||||
import llm
|
||||
|
||||
@llm.hookimpl
|
||||
def register_models(register):
|
||||
register(HelloWorld())
|
||||
|
||||
class HelloWorld(llm.Model):
|
||||
model_id = "helloworld"
|
||||
|
||||
def execute(self, prompt, stream, response):
|
||||
return ["hello world"]
|
||||
```
|
||||
If your model includes an async version, you can register that too:
|
||||
|
||||
```python
|
||||
class AsyncHelloWorld(llm.AsyncModel):
|
||||
model_id = "helloworld"
|
||||
|
||||
async def execute(self, prompt, stream, response):
|
||||
return ["hello world"]
|
||||
|
||||
@llm.hookimpl
|
||||
def register_models(register):
|
||||
register(HelloWorld(), AsyncHelloWorld(), aliases=("hw",))
|
||||
```
|
||||
This demonstrates how to register a model with both sync and async versions, and how to specify an alias for that model.
|
||||
|
||||
The `model_aliases` parameter is a list of {class}`~llm.ModelWithAliases` objects representing all models registered so far by other plugins. Plugins that use `@llm.hookimpl(trylast=True)` can use this to inspect or modify models registered by other plugins. Both parameters are optional - plugins can accept just `register`, just `model_aliases`, or both.
|
||||
|
||||
The {ref}`model plugin tutorial <tutorial-model-plugin>` describes how to use this hook in detail. Asynchronous models {ref}`are described here <advanced-model-plugins-async>`.
|
||||
|
||||
```{eval-rst}
|
||||
.. autoclass:: llm.ModelWithAliases
|
||||
:exclude-members: matches
|
||||
```
|
||||
|
||||
(plugin-hooks-register-embedding-models)=
|
||||
## register_embedding_models(register)
|
||||
|
||||
This hook can be used to register one or more additional embedding models, as described in {ref}`embeddings-writing-plugins`.
|
||||
|
||||
```python
|
||||
import llm
|
||||
|
||||
@llm.hookimpl
|
||||
def register_embedding_models(register):
|
||||
register(HelloWorld())
|
||||
|
||||
class HelloWorld(llm.EmbeddingModel):
|
||||
model_id = "helloworld"
|
||||
|
||||
def embed_batch(self, items):
|
||||
return [[1, 2, 3], [4, 5, 6]]
|
||||
```
|
||||
|
||||
(plugin-hooks-register-tools)=
|
||||
## register_tools(register)
|
||||
|
||||
This hook can register one or more tool functions for use with LLM. See {ref}`the tools documentation <tools>` for more details.
|
||||
|
||||
This example registers two tools: `upper` and `count_character_in_word`.
|
||||
|
||||
```python
|
||||
import llm
|
||||
|
||||
def upper(text: str) -> str:
|
||||
"""Convert text to uppercase."""
|
||||
return text.upper()
|
||||
|
||||
def count_char(text: str, character: str) -> int:
|
||||
"""Count the number of occurrences of a character in a word."""
|
||||
return text.count(character)
|
||||
|
||||
@llm.hookimpl
|
||||
def register_tools(register):
|
||||
register(upper)
|
||||
# Here the name= argument is used to specify a different name for the tool:
|
||||
register(count_char, name="count_character_in_word")
|
||||
```
|
||||
|
||||
Tools can also be implemented as classes, as described in {ref}`Toolbox classes <python-api-toolbox>` in the Python API documentation.
|
||||
|
||||
You can register classes like the `Memory` example {ref}`from here <python-api-toolbox>` by passing the class (_not_ an instance of the class) to `register()`:
|
||||
|
||||
```python
|
||||
import llm
|
||||
|
||||
class Memory(llm.Toolbox):
|
||||
# Copy implementation from the Python API documentation
|
||||
|
||||
@llm.hookimpl
|
||||
def register_tools(register):
|
||||
register(Memory)
|
||||
```
|
||||
Once installed, this tool can be used like so:
|
||||
|
||||
```bash
|
||||
llm chat -T Memory
|
||||
```
|
||||
If a tool name starts with a capital letter it is assumed to be a toolbox class, not a regular tool function.
|
||||
|
||||
Here's an example session with the Memory tool:
|
||||
```
|
||||
Chatting with gpt-4.1-mini
|
||||
Type 'exit' or 'quit' to exit
|
||||
Type '!multi' to enter multiple lines, then '!end' to finish
|
||||
Type '!edit' to open your default editor and modify the prompt
|
||||
Type '!fragment <my_fragment> [<another_fragment> ...]' to insert one or more fragments
|
||||
> Remember my name is Henry
|
||||
|
||||
Tool call: Memory_set({'key': 'user_name', 'value': 'Henry'})
|
||||
null
|
||||
|
||||
Got it, Henry! I'll remember your name. How can I assist you today?
|
||||
> what keys are there?
|
||||
|
||||
Tool call: Memory_keys({})
|
||||
[
|
||||
"user_name"
|
||||
]
|
||||
|
||||
Currently, there is one key stored: "user_name". Would you like to add or retrieve any information?
|
||||
> read it
|
||||
|
||||
Tool call: Memory_get({'key': 'user_name'})
|
||||
Henry
|
||||
|
||||
The value stored under the key "user_name" is Henry. Is there anything else you'd like to do?
|
||||
> add Barrett to it
|
||||
|
||||
Tool call: Memory_append({'key': 'user_name', 'value': 'Barrett'})
|
||||
null
|
||||
|
||||
I have added "Barrett" to the key "user_name". If you want, I can now show you the updated value.
|
||||
> show value
|
||||
|
||||
Tool call: Memory_get({'key': 'user_name'})
|
||||
Henry
|
||||
Barrett
|
||||
|
||||
The value stored under the key "user_name" is now:
|
||||
Henry
|
||||
Barrett
|
||||
|
||||
Is there anything else you would like to do?
|
||||
```
|
||||
|
||||
(plugin-hooks-register-template-loaders)=
|
||||
## register_template_loaders(register)
|
||||
|
||||
Plugins can register new {ref}`template loaders <prompt-templates-loaders>` using the `register_template_loaders` hook.
|
||||
|
||||
Template loaders work with the `llm -t prefix:name` syntax. The prefix specifies the loader, then the registered loader function is called with the name as an argument. The loader function should return an `llm.Template()` object.
|
||||
|
||||
This example plugin registers `my-prefix` as a new template loader. Once installed it can be used like this:
|
||||
|
||||
```bash
|
||||
llm -t my-prefix:my-template
|
||||
```
|
||||
Here's the Python code:
|
||||
|
||||
```python
|
||||
import llm
|
||||
|
||||
@llm.hookimpl
|
||||
def register_template_loaders(register):
|
||||
register("my-prefix", my_template_loader)
|
||||
|
||||
def my_template_loader(template_path: str) -> llm.Template:
|
||||
"""
|
||||
Documentation for the template loader goes here. It will be displayed
|
||||
when users run the 'llm templates loaders' command.
|
||||
"""
|
||||
try:
|
||||
# Your logic to fetch the template content
|
||||
# This is just an example:
|
||||
prompt = "This is a sample prompt for {}".format(template_path)
|
||||
system = "You are an assistant specialized in {}".format(template_path)
|
||||
|
||||
# Return a Template object with the required fields
|
||||
return llm.Template(
|
||||
name=template_path,
|
||||
prompt=prompt,
|
||||
system=system,
|
||||
)
|
||||
except Exception as e:
|
||||
# Raise a ValueError with a clear message if the template cannot be found
|
||||
raise ValueError(f"Template '{template_path}' could not be loaded: {str(e)}")
|
||||
```
|
||||
The `llm.Template` class has the following constructor:
|
||||
|
||||
```{eval-rst}
|
||||
.. autoclass:: llm.Template
|
||||
```
|
||||
|
||||
The loader function should raise a `ValueError` if the template cannot be found or loaded correctly, providing a clear error message.
|
||||
|
||||
Note that `functions:` provided by templates using this plugin hook will not be made available, to avoid the risk of plugin hooks that load templates from remote sources introducing arbitrary code execution vulnerabilities.
|
||||
|
||||
(plugin-hooks-register-fragment-loaders)=
|
||||
## register_fragment_loaders(register)
|
||||
|
||||
Plugins can register new fragment loaders using the `register_fragment_loaders` hook. These can then be used with the `llm -f prefix:argument` syntax.
|
||||
|
||||
Fragment loader plugins differ from template loader plugins in that you can stack more than one fragment loader call together in the same prompt.
|
||||
|
||||
A fragment loader can return one or more string fragments or attachments, or a mixture of the two. The fragments will be concatenated together into the prompt string, while any attachments will be added to the list of attachments to be sent to the model.
|
||||
|
||||
The `prefix` specifies the loader. The `argument` will be passed to that registered callback..
|
||||
|
||||
The callback works in a very similar way to template loaders, but returns either a single `llm.Fragment`, a list of `llm.Fragment` objects, a single `llm.Attachment`, or a list that can mix `llm.Attachment` and `llm.Fragment` objects.
|
||||
|
||||
The `llm.Fragment` constructor takes a required string argument (the content of the fragment) and an optional second `source` argument, which is a string that may be displayed as debug information. For files this is a path and for URLs it is a URL. Your plugin can use anything you like for the `source` value.
|
||||
|
||||
See {ref}`the Python API documentation for attachments <python-api-attachments>` for details of the `llm.Attachment` class.
|
||||
|
||||
Here is some example code:
|
||||
|
||||
```python
|
||||
import llm
|
||||
|
||||
@llm.hookimpl
|
||||
def register_fragment_loaders(register):
|
||||
register("my-fragments", my_fragment_loader)
|
||||
|
||||
|
||||
def my_fragment_loader(argument: str) -> llm.Fragment:
|
||||
"""
|
||||
Documentation for the fragment loader goes here. It will be displayed
|
||||
when users run the 'llm fragments loaders' command.
|
||||
"""
|
||||
try:
|
||||
fragment = "Fragment content for {}".format(argument)
|
||||
source = "my-fragments:{}".format(argument)
|
||||
return llm.Fragment(fragment, source)
|
||||
except Exception as ex:
|
||||
# Raise a ValueError with a clear message if the fragment cannot be loaded
|
||||
raise ValueError(
|
||||
f"Fragment 'my-fragments:{argument}' could not be loaded: {str(ex)}"
|
||||
)
|
||||
|
||||
# Or for the case where you want to return multiple fragments and attachments:
|
||||
def my_fragment_loader(argument: str) -> list[llm.Fragment]:
|
||||
"Docs go here."
|
||||
return [
|
||||
llm.Fragment("Fragment 1 content", "my-fragments:{argument}"),
|
||||
llm.Fragment("Fragment 2 content", "my-fragments:{argument}"),
|
||||
llm.Attachment(path="/path/to/image.png"),
|
||||
]
|
||||
```
|
||||
A plugin like this one can be called like so:
|
||||
```bash
|
||||
llm -f my-fragments:argument
|
||||
```
|
||||
If multiple fragments are returned they will be used as if the user passed multiple `-f X` arguments to the command.
|
||||
|
||||
Multiple fragments are particularly useful for things like plugins that return every file in a directory. If these were concatenated together by the plugin, a change to a single file would invalidate the de-duplication cache for that whole fragment. Giving each file its own fragment means we can avoid storing multiple copies of that full collection if only a single file has changed.
|
||||
@@ -0,0 +1,92 @@
|
||||
(plugin-utilities)=
|
||||
# Utility functions for plugins
|
||||
|
||||
LLM provides some utility functions that may be useful to plugins.
|
||||
|
||||
(plugin-utilities-get-key)=
|
||||
## llm.get_key()
|
||||
|
||||
This method can be used to look up secrets that users have stored using the {ref}`llm keys set <help-keys-set>` command. If your plugin needs to access an API key or other secret this can be a convenient way to provide that.
|
||||
|
||||
This returns either a string containing the key or `None` if the key could not be resolved.
|
||||
|
||||
Use the `alias="name"` option to retrieve the key set with that alias:
|
||||
|
||||
```python
|
||||
github_key = llm.get_key(alias="github")
|
||||
```
|
||||
You can also add `env="ENV_VAR"` to fall back to looking in that environment variable if the key has not been configured:
|
||||
```python
|
||||
github_key = llm.get_key(alias="github", env="GITHUB_TOKEN")
|
||||
```
|
||||
In some cases you may allow users to provide a key as input, where they could input either the key itself or specify an alias to lookup in `keys.json`. Use the `input=` parameter for that:
|
||||
|
||||
```python
|
||||
github_key = llm.get_key(input=input_from_user, alias="github", env="GITHUB_TOKEN")
|
||||
```
|
||||
|
||||
An previous version of function used positional arguments in a confusing order. These are still supported but the new keyword arguments are recommended as a better way to use `llm.get_key()` going forward.
|
||||
|
||||
(plugin-utilities-user-dir)=
|
||||
## llm.user_dir()
|
||||
|
||||
LLM stores various pieces of logging and configuration data in a directory on the user's machine.
|
||||
|
||||
On macOS this directory is `~/Library/Application Support/io.datasette.llm`, but this will differ on other operating systems.
|
||||
|
||||
The `llm.user_dir()` function returns the path to this directory as a `pathlib.Path` object, after creating that directory if it does not yet exist.
|
||||
|
||||
Plugins can use this to store their own data in a subdirectory of this directory.
|
||||
|
||||
```python
|
||||
import llm
|
||||
user_dir = llm.user_dir()
|
||||
plugin_dir = data_path = user_dir / "my-plugin"
|
||||
plugin_dir.mkdir(exist_ok=True)
|
||||
data_path = plugin_dir / "plugin-data.db"
|
||||
```
|
||||
|
||||
(plugin-utilities-modelerror)=
|
||||
## llm.ModelError
|
||||
|
||||
If your model encounters an error that should be reported to the user you can raise this exception. For example:
|
||||
|
||||
```python
|
||||
import llm
|
||||
|
||||
raise ModelError("MPT model not installed - try running 'llm mpt30b download'")
|
||||
```
|
||||
This will be caught by the CLI layer and displayed to the user as an error message.
|
||||
|
||||
(plugin-utilities-response-fake)=
|
||||
## Response.fake()
|
||||
|
||||
When writing tests for a model it can be useful to generate fake response objects, for example in this test from [llm-mpt30b](https://github.com/simonw/llm-mpt30b):
|
||||
|
||||
```python
|
||||
def test_build_prompt_conversation():
|
||||
model = llm.get_model("mpt")
|
||||
conversation = model.conversation()
|
||||
conversation.responses = [
|
||||
llm.Response.fake(model, "prompt 1", "system 1", "response 1"),
|
||||
llm.Response.fake(model, "prompt 2", None, "response 2"),
|
||||
llm.Response.fake(model, "prompt 3", None, "response 3"),
|
||||
]
|
||||
lines = model.build_prompt(llm.Prompt("prompt 4", model), conversation)
|
||||
assert lines == [
|
||||
"<|im_start|>system\system 1<|im_end|>\n",
|
||||
"<|im_start|>user\nprompt 1<|im_end|>\n",
|
||||
"<|im_start|>assistant\nresponse 1<|im_end|>\n",
|
||||
"<|im_start|>user\nprompt 2<|im_end|>\n",
|
||||
"<|im_start|>assistant\nresponse 2<|im_end|>\n",
|
||||
"<|im_start|>user\nprompt 3<|im_end|>\n",
|
||||
"<|im_start|>assistant\nresponse 3<|im_end|>\n",
|
||||
"<|im_start|>user\nprompt 4<|im_end|>\n",
|
||||
"<|im_start|>assistant\n",
|
||||
]
|
||||
```
|
||||
The signature of `llm.Response.fake()` is:
|
||||
|
||||
```python
|
||||
def fake(cls, model: Model, prompt: str, system: str, response: str):
|
||||
```
|
||||
@@ -0,0 +1,614 @@
|
||||
(tutorial-model-plugin)=
|
||||
|
||||
# Developing a model plugin
|
||||
|
||||
This tutorial will walk you through developing a new plugin for LLM that adds support for a new Large Language Model.
|
||||
|
||||
We will be developing a plugin that implements a simple [Markov chain](https://en.wikipedia.org/wiki/Markov_chain) to generate words based on an input string. Markov chains are not technically large language models, but they provide a useful exercise for demonstrating how the LLM tool can be extended through plugins.
|
||||
|
||||
(tutorial-model-plugin-initial)=
|
||||
|
||||
## The initial structure of the plugin
|
||||
|
||||
First create a new directory with the name of your plugin - it should be called something like `llm-markov`.
|
||||
```bash
|
||||
mkdir llm-markov
|
||||
cd llm-markov
|
||||
```
|
||||
In that directory create a file called `llm_markov.py` containing this:
|
||||
|
||||
```python
|
||||
import llm
|
||||
|
||||
@llm.hookimpl
|
||||
def register_models(register):
|
||||
register(Markov())
|
||||
|
||||
class Markov(llm.Model):
|
||||
model_id = "markov"
|
||||
|
||||
def execute(self, prompt, stream, response, conversation):
|
||||
return ["hello world"]
|
||||
```
|
||||
|
||||
The `def register_models()` function here is called by the plugin system (thanks to the `@hookimpl` decorator). It uses the `register()` function passed to it to register an instance of the new model.
|
||||
|
||||
The `Markov` class implements the model. It sets a `model_id` - an identifier that can be passed to `llm -m` in order to identify the model to be executed.
|
||||
|
||||
The logic for executing the model goes in the `execute()` method. We'll extend this to do something more useful in a later step.
|
||||
|
||||
Next, create a `pyproject.toml` file. This is necessary to tell LLM how to load your plugin:
|
||||
|
||||
```toml
|
||||
[project]
|
||||
name = "llm-markov"
|
||||
version = "0.1"
|
||||
|
||||
[project.entry-points.llm]
|
||||
markov = "llm_markov"
|
||||
```
|
||||
|
||||
This is the simplest possible configuration. It defines a plugin name and provides an [entry point](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) for `llm` telling it how to load the plugin.
|
||||
|
||||
If you are comfortable with Python virtual environments you can create one now for your project, activate it and run `pip install llm` before the next step.
|
||||
|
||||
If you aren't familiar with virtual environments, don't worry: you can develop plugins without them. You'll need to have LLM installed using Homebrew or `pipx` or one of the [other installation options](https://llm.datasette.io/en/latest/setup.html#installation).
|
||||
|
||||
(tutorial-model-plugin-installing)=
|
||||
|
||||
## Installing your plugin to try it out
|
||||
|
||||
Having created a directory with a `pyproject.toml` file and an `llm_markov.py` file, you can install your plugin into LLM by running this from inside your `llm-markov` directory:
|
||||
|
||||
```bash
|
||||
llm install -e .
|
||||
```
|
||||
|
||||
The `-e` stands for "editable" - it means you'll be able to make further changes to the `llm_markov.py` file that will be reflected without you having to reinstall the plugin.
|
||||
|
||||
The `.` means the current directory. You can also install editable plugins by passing a path to their directory this:
|
||||
```bash
|
||||
llm install -e path/to/llm-markov
|
||||
```
|
||||
To confirm that your plugin has installed correctly, run this command:
|
||||
```bash
|
||||
llm plugins
|
||||
```
|
||||
The output should look like this:
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "llm-markov",
|
||||
"hooks": [
|
||||
"register_models"
|
||||
],
|
||||
"version": "0.1"
|
||||
},
|
||||
{
|
||||
"name": "llm.default_plugins.openai_models",
|
||||
"hooks": [
|
||||
"register_commands",
|
||||
"register_models"
|
||||
]
|
||||
}
|
||||
]
|
||||
```
|
||||
This command lists default plugins that are included with LLM as well as new plugins that have been installed.
|
||||
|
||||
Now let's try the plugin by running a prompt through it:
|
||||
```bash
|
||||
llm -m markov "the cat sat on the mat"
|
||||
```
|
||||
It outputs:
|
||||
```
|
||||
hello world
|
||||
```
|
||||
Next, we'll make it execute and return the results of a Markov chain.
|
||||
|
||||
(tutorial-model-plugin-building)=
|
||||
|
||||
## Building the Markov chain
|
||||
|
||||
Markov chains can be thought of as the simplest possible example of a generative language model. They work by building an index of words that have been seen following other words.
|
||||
|
||||
Here's what that index looks like for the phrase "the cat sat on the mat"
|
||||
```json
|
||||
{
|
||||
"the": ["cat", "mat"],
|
||||
"cat": ["sat"],
|
||||
"sat": ["on"],
|
||||
"on": ["the"]
|
||||
}
|
||||
```
|
||||
Here's a Python function that builds that data structure from a text input:
|
||||
```python
|
||||
def build_markov_table(text):
|
||||
words = text.split()
|
||||
transitions = {}
|
||||
# Loop through all but the last word
|
||||
for i in range(len(words) - 1):
|
||||
word = words[i]
|
||||
next_word = words[i + 1]
|
||||
transitions.setdefault(word, []).append(next_word)
|
||||
return transitions
|
||||
```
|
||||
We can try that out by pasting it into the interactive Python interpreter and running this:
|
||||
```pycon
|
||||
>>> transitions = build_markov_table("the cat sat on the mat")
|
||||
>>> transitions
|
||||
{'the': ['cat', 'mat'], 'cat': ['sat'], 'sat': ['on'], 'on': ['the']}
|
||||
```
|
||||
|
||||
(tutorial-model-plugin-executing)=
|
||||
|
||||
## Executing the Markov chain
|
||||
|
||||
To execute the model, we start with a word. We look at the options for words that might come next and pick one of those at random. Then we repeat that process until we have produced the desired number of output words.
|
||||
|
||||
Some words might not have any following words from our training sentence. For our implementation we will fall back on picking a random word from our collection.
|
||||
|
||||
We will implement this as a [Python generator](https://realpython.com/introduction-to-python-generators/), using the yield keyword to produce each token:
|
||||
```python
|
||||
def generate(transitions, length, start_word=None):
|
||||
all_words = list(transitions.keys())
|
||||
next_word = start_word or random.choice(all_words)
|
||||
for i in range(length):
|
||||
yield next_word
|
||||
options = transitions.get(next_word) or all_words
|
||||
next_word = random.choice(options)
|
||||
```
|
||||
If you aren't familiar with generators, the above code could also be implemented like this - creating a Python list and returning it at the end of the function:
|
||||
```python
|
||||
def generate_list(transitions, length, start_word=None):
|
||||
all_words = list(transitions.keys())
|
||||
next_word = start_word or random.choice(all_words)
|
||||
output = []
|
||||
for i in range(length):
|
||||
output.append(next_word)
|
||||
options = transitions.get(next_word) or all_words
|
||||
next_word = random.choice(options)
|
||||
return output
|
||||
```
|
||||
You can try out the `generate()` function like this:
|
||||
```python
|
||||
lookup = build_markov_table("the cat sat on the mat")
|
||||
for word in generate(transitions, 20):
|
||||
print(word)
|
||||
```
|
||||
Or you can generate a full string sentence with it like this:
|
||||
```python
|
||||
sentence = " ".join(generate(transitions, 20))
|
||||
```
|
||||
|
||||
(tutorial-model-plugin-register)=
|
||||
|
||||
## Adding that to the plugin
|
||||
|
||||
Our `execute()` method from earlier currently returns the list `["hello world"]`.
|
||||
|
||||
Update that to use our new Markov chain generator instead. Here's the full text of the new `llm_markov.py` file:
|
||||
|
||||
```python
|
||||
import llm
|
||||
import random
|
||||
|
||||
@llm.hookimpl
|
||||
def register_models(register):
|
||||
register(Markov())
|
||||
|
||||
def build_markov_table(text):
|
||||
words = text.split()
|
||||
transitions = {}
|
||||
# Loop through all but the last word
|
||||
for i in range(len(words) - 1):
|
||||
word = words[i]
|
||||
next_word = words[i + 1]
|
||||
transitions.setdefault(word, []).append(next_word)
|
||||
return transitions
|
||||
|
||||
def generate(transitions, length, start_word=None):
|
||||
all_words = list(transitions.keys())
|
||||
next_word = start_word or random.choice(all_words)
|
||||
for i in range(length):
|
||||
yield next_word
|
||||
options = transitions.get(next_word) or all_words
|
||||
next_word = random.choice(options)
|
||||
|
||||
class Markov(llm.Model):
|
||||
model_id = "markov"
|
||||
|
||||
def execute(self, prompt, stream, response, conversation):
|
||||
text = prompt.prompt
|
||||
transitions = build_markov_table(text)
|
||||
for word in generate(transitions, 20):
|
||||
yield word + ' '
|
||||
```
|
||||
The `execute()` method can access the text prompt that the user provided using` prompt.prompt` - `prompt` is a `Prompt` object that might include other more advanced input details as well.
|
||||
|
||||
Now when you run this you should see the output of the Markov chain!
|
||||
```bash
|
||||
llm -m markov "the cat sat on the mat"
|
||||
```
|
||||
```
|
||||
the mat the cat sat on the cat sat on the mat cat sat on the mat cat sat on
|
||||
```
|
||||
|
||||
(tutorial-model-plugin-execute)=
|
||||
|
||||
## Understanding execute()
|
||||
|
||||
The full signature of the `execute()` method is:
|
||||
```python
|
||||
def execute(self, prompt, stream, response, conversation):
|
||||
```
|
||||
The `prompt` argument is a `Prompt` object that contains the text that the user provided, the system prompt and the provided options.
|
||||
|
||||
`stream` is a boolean that says if the model is being run in streaming mode.
|
||||
|
||||
`response` is the `Response` object that is being created by the model. This is provided so you can write additional information to `response.response_json`, which may be logged to the database.
|
||||
|
||||
`conversation` is the `Conversation` that the prompt is a part of - or `None` if no conversation was provided. Some models may use `conversation.responses` to access previous prompts and responses in the conversation and use them to construct a call to the LLM that includes previous context.
|
||||
|
||||
(tutorial-model-plugin-logging)=
|
||||
|
||||
## Prompts and responses are logged to the database
|
||||
|
||||
The prompt and the response will be logged to a SQLite database automatically by LLM. You can see the single most recent addition to the logs using:
|
||||
```
|
||||
llm logs -n 1
|
||||
```
|
||||
The output should look something like this:
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": "01h52s4yez2bd1qk2deq49wk8h",
|
||||
"model": "markov",
|
||||
"prompt": "the cat sat on the mat",
|
||||
"system": null,
|
||||
"prompt_json": null,
|
||||
"options_json": {},
|
||||
"response": "on the cat sat on the cat sat on the mat cat sat on the cat sat on the cat ",
|
||||
"response_json": null,
|
||||
"conversation_id": "01h52s4yey7zc5rjmczy3ft75g",
|
||||
"duration_ms": 0,
|
||||
"datetime_utc": "2023-07-11T15:29:34.685868",
|
||||
"conversation_name": "the cat sat on the mat",
|
||||
"conversation_model": "markov"
|
||||
}
|
||||
]
|
||||
```
|
||||
Plugins can log additional information to the database by assigning a dictionary to the `response.response_json` property during the `execute()` method.
|
||||
|
||||
Here's how to include that full `transitions` table in the `response_json` in the log:
|
||||
```python
|
||||
def execute(self, prompt, stream, response, conversation):
|
||||
text = self.prompt.prompt
|
||||
transitions = build_markov_table(text)
|
||||
for word in generate(transitions, 20):
|
||||
yield word + ' '
|
||||
response.response_json = {"transitions": transitions}
|
||||
```
|
||||
|
||||
Now when you run the logs command you'll see that too:
|
||||
```bash
|
||||
llm logs -n 1
|
||||
```
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": 623,
|
||||
"model": "markov",
|
||||
"prompt": "the cat sat on the mat",
|
||||
"system": null,
|
||||
"prompt_json": null,
|
||||
"options_json": {},
|
||||
"response": "on the mat the cat sat on the cat sat on the mat sat on the cat sat on the ",
|
||||
"response_json": {
|
||||
"transitions": {
|
||||
"the": [
|
||||
"cat",
|
||||
"mat"
|
||||
],
|
||||
"cat": [
|
||||
"sat"
|
||||
],
|
||||
"sat": [
|
||||
"on"
|
||||
],
|
||||
"on": [
|
||||
"the"
|
||||
]
|
||||
}
|
||||
},
|
||||
"reply_to_id": null,
|
||||
"chat_id": null,
|
||||
"duration_ms": 0,
|
||||
"datetime_utc": "2023-07-06T01:34:45.376637"
|
||||
}
|
||||
]
|
||||
```
|
||||
In this particular case this isn't a great idea here though: the `transitions` table is duplicate information, since it can be reproduced from the input data - and it can get really large for longer prompts.
|
||||
|
||||
(tutorial-model-plugin-options)=
|
||||
|
||||
## Adding options
|
||||
|
||||
LLM models can take options. For large language models these can be things like `temperature` or `top_k`.
|
||||
|
||||
Options are passed using the `-o/--option` command line parameters, for example:
|
||||
```bash
|
||||
llm -m gpt4 "ten pet pelican names" -o temperature 1.5
|
||||
```
|
||||
We're going to add two options to our Markov chain model:
|
||||
|
||||
- `length`: Number of words to generate
|
||||
- `delay`: a floating point number of Delay in between output token
|
||||
|
||||
The `delay` token will let us simulate a streaming language model, where tokens take time to generate and are returned by the `execute()` function as they become ready.
|
||||
|
||||
Options are defined using an inner class on the model, called `Options`. It should extend the `llm.Options` class.
|
||||
|
||||
First, add this import to the top of your `llm_markov.py` file:
|
||||
```python
|
||||
from typing import Optional
|
||||
```
|
||||
Then add this `Options` class to your model:
|
||||
```python
|
||||
class Markov(Model):
|
||||
model_id = "markov"
|
||||
|
||||
class Options(llm.Options):
|
||||
length: Optional[int] = None
|
||||
delay: Optional[float] = None
|
||||
```
|
||||
Let's add extra validation rules to our options. Length must be at least 2. Duration must be between 0 and 10.
|
||||
|
||||
The `Options` class uses [Pydantic 2](https://pydantic.dev/), which can support all sorts of advanced validation rules.
|
||||
|
||||
We can also add inline documentation, which can then be displayed by the `llm models --options` command.
|
||||
|
||||
Add these imports to the top of `llm_markov.py`:
|
||||
```python
|
||||
from pydantic import field_validator, Field
|
||||
```
|
||||
|
||||
We can now add Pydantic field validators for our two new rules, plus inline documentation:
|
||||
|
||||
```python
|
||||
class Options(llm.Options):
|
||||
length: Optional[int] = Field(
|
||||
description="Number of words to generate",
|
||||
default=None
|
||||
)
|
||||
delay: Optional[float] = Field(
|
||||
description="Seconds to delay between each token",
|
||||
default=None
|
||||
)
|
||||
|
||||
@field_validator("length")
|
||||
def validate_length(cls, length):
|
||||
if length is None:
|
||||
return None
|
||||
if length < 2:
|
||||
raise ValueError("length must be >= 2")
|
||||
return length
|
||||
|
||||
@field_validator("delay")
|
||||
def validate_delay(cls, delay):
|
||||
if delay is None:
|
||||
return None
|
||||
if not 0 <= delay <= 10:
|
||||
raise ValueError("delay must be between 0 and 10")
|
||||
return delay
|
||||
```
|
||||
Lets test our options validation:
|
||||
```bash
|
||||
llm -m markov "the cat sat on the mat" -o length -1
|
||||
```
|
||||
```
|
||||
Error: length
|
||||
Value error, length must be >= 2
|
||||
```
|
||||
|
||||
Next, we will modify our `execute()` method to handle those options. Add this to the beginning of `llm_markov.py`:
|
||||
```python
|
||||
import time
|
||||
```
|
||||
Then replace the `execute()` method with this one:
|
||||
```python
|
||||
def execute(self, prompt, stream, response, conversation):
|
||||
text = prompt.prompt
|
||||
transitions = build_markov_table(text)
|
||||
length = prompt.options.length or 20
|
||||
for word in generate(transitions, length):
|
||||
yield word + ' '
|
||||
if prompt.options.delay:
|
||||
time.sleep(prompt.options.delay)
|
||||
```
|
||||
Add `can_stream = True` to the top of the `Markov` model class, on the line below `model_id = "markov". This tells LLM that the model is able to stream content to the console.
|
||||
|
||||
The full `llm_markov.py` file should now look like this:
|
||||
|
||||
```{literalinclude} llm-markov/llm_markov.py
|
||||
:language: python
|
||||
```
|
||||
|
||||
Now we can request a 20 word completion with a 0.1s delay between tokens like this:
|
||||
```bash
|
||||
llm -m markov "the cat sat on the mat" \
|
||||
-o length 20 -o delay 0.1
|
||||
```
|
||||
LLM provides a `--no-stream` option users can use to turn off streaming. Using that option causes LLM to gather the response from the stream and then return it to the console in one block. You can try that like this:
|
||||
```bash
|
||||
llm -m markov "the cat sat on the mat" \
|
||||
-o length 20 -o delay 0.1 --no-stream
|
||||
```
|
||||
In this case it will still delay for 2s total while it gathers the tokens, then output them all at once.
|
||||
|
||||
That `--no-stream` option causes the `stream` argument passed to `execute()` to be false. Your `execute()` method can then behave differently depending on whether it is streaming or not.
|
||||
|
||||
Options are also logged to the database. You can see those here:
|
||||
```bash
|
||||
llm logs -n 1
|
||||
```
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": 636,
|
||||
"model": "markov",
|
||||
"prompt": "the cat sat on the mat",
|
||||
"system": null,
|
||||
"prompt_json": null,
|
||||
"options_json": {
|
||||
"length": 20,
|
||||
"delay": 0.1
|
||||
},
|
||||
"response": "the mat on the mat on the cat sat on the mat sat on the mat cat sat on the ",
|
||||
"response_json": null,
|
||||
"reply_to_id": null,
|
||||
"chat_id": null,
|
||||
"duration_ms": 2063,
|
||||
"datetime_utc": "2023-07-07T03:02:28.232970"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
(tutorial-model-plugin-distributing)=
|
||||
|
||||
## Distributing your plugin
|
||||
|
||||
There are many different options for distributing your new plugin so other people can try it out.
|
||||
|
||||
You can create a downloadable wheel or `.zip` or `.tar.gz` files, or share the plugin through GitHub Gists or repositories.
|
||||
|
||||
You can also publish your plugin to PyPI, the Python Package Index.
|
||||
|
||||
(tutorial-model-plugin-wheels)=
|
||||
|
||||
### Wheels and sdist packages
|
||||
|
||||
The easiest option is to produce a distributable package is to use the `build` command. First, install the `build` package by running this:
|
||||
```bash
|
||||
python -m pip install build
|
||||
```
|
||||
Then run `build` in your plugin directory to create the packages:
|
||||
```bash
|
||||
python -m build
|
||||
```
|
||||
This will create two files: `dist/llm-markov-0.1.tar.gz` and `dist/llm-markov-0.1-py3-none-any.whl`.
|
||||
|
||||
Either of these files can be used to install the plugin:
|
||||
|
||||
```bash
|
||||
llm install dist/llm_markov-0.1-py3-none-any.whl
|
||||
```
|
||||
If you host this file somewhere online other people will be able to install it using `pip install` against the URL to your package:
|
||||
```bash
|
||||
llm install 'https://.../llm_markov-0.1-py3-none-any.whl'
|
||||
```
|
||||
You can run the following command at any time to uninstall your plugin, which is useful for testing out different installation methods:
|
||||
```bash
|
||||
llm uninstall llm-markov -y
|
||||
```
|
||||
|
||||
(tutorial-model-plugin-gists)=
|
||||
|
||||
### GitHub Gists
|
||||
|
||||
A neat quick option for distributing a simple plugin is to host it in a GitHub Gist. These are available for free with a GitHub account, and can be public or private. Gists can contain multiple files but don't support directory structures - which is OK, because our plugin is just two files, `pyproject.toml` and `llm_markov.py`.
|
||||
|
||||
Here's an example Gist I created for this tutorial:
|
||||
|
||||
[https://gist.github.com/simonw/6e56d48dc2599bffba963cef0db27b6d](https://gist.github.com/simonw/6e56d48dc2599bffba963cef0db27b6d)
|
||||
|
||||
You can turn a Gist into an installable `.zip` URL by right-clicking on the "Download ZIP" button and selecting "Copy Link". Here's that link for my example Gist:
|
||||
|
||||
`https://gist.github.com/simonw/6e56d48dc2599bffba963cef0db27b6d/archive/cc50c854414cb4deab3e3ab17e7e1e07d45cba0c.zip`
|
||||
|
||||
The plugin can be installed using the `llm install` command like this:
|
||||
```bash
|
||||
llm install 'https://gist.github.com/simonw/6e56d48dc2599bffba963cef0db27b6d/archive/cc50c854414cb4deab3e3ab17e7e1e07d45cba0c.zip'
|
||||
```
|
||||
|
||||
(tutorial-model-plugin-github)=
|
||||
|
||||
## GitHub repositories
|
||||
|
||||
The same trick works for regular GitHub repositories as well: the "Download ZIP" button can be found by clicking the green "Code" button at the top of the repository. The URL which that provides can then be used to install the plugin that lives in that repository.
|
||||
|
||||
(tutorial-model-plugin-pypi)=
|
||||
|
||||
## Publishing plugins to PyPI
|
||||
|
||||
The [Python Package Index (PyPI)](https://pypi.org/) is the official repository for Python packages. You can upload your plugin to PyPI and reserve a name for it - once you have done that, anyone will be able to install your plugin using `llm install <name>`.
|
||||
|
||||
Follow [these instructions](https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives) to publish a package to PyPI. The short version:
|
||||
```bash
|
||||
python -m pip install twine
|
||||
python -m twine upload dist/*
|
||||
```
|
||||
You will need an account on PyPI, then you can enter your username and password - or create a token in the PyPI settings and use `__token__` as the username and the token as the password.
|
||||
|
||||
(tutorial-model-plugin-metadata)=
|
||||
|
||||
## Adding metadata
|
||||
|
||||
Before uploading a package to PyPI it's a good idea to add documentation and expand `pyproject.toml` with additional metadata.
|
||||
|
||||
Create a `README.md` file in the root of your plugin directory with instructions about how to install, configure and use your plugin.
|
||||
|
||||
You can then replace `pyproject.toml` with something like this:
|
||||
|
||||
```toml
|
||||
[project]
|
||||
name = "llm-markov"
|
||||
version = "0.1"
|
||||
description = "Plugin for LLM adding a Markov chain generating model"
|
||||
readme = "README.md"
|
||||
authors = [{name = "Simon Willison"}]
|
||||
license = {text = "Apache-2.0"}
|
||||
classifiers = [
|
||||
"License :: OSI Approved :: Apache Software License"
|
||||
]
|
||||
dependencies = [
|
||||
"llm"
|
||||
]
|
||||
requires-python = ">3.7"
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/simonw/llm-markov"
|
||||
Changelog = "https://github.com/simonw/llm-markov/releases"
|
||||
Issues = "https://github.com/simonw/llm-markov/issues"
|
||||
|
||||
[project.entry-points.llm]
|
||||
markov = "llm_markov"
|
||||
```
|
||||
This will pull in your README to be displayed as part of your project's listing page on PyPI.
|
||||
|
||||
It adds `llm` as a dependency, ensuring it will be installed if someone tries to install your plugin package without it.
|
||||
|
||||
It adds some links to useful pages (you can drop the `project.urls` section if those links are not useful for your project).
|
||||
|
||||
You should drop a `LICENSE` file into the GitHub repository for your package as well. I like to use the Apache 2 license [like this](https://github.com/simonw/llm/blob/main/LICENSE).
|
||||
|
||||
(tutorial-model-plugin-breaks)=
|
||||
|
||||
## What to do if it breaks
|
||||
|
||||
Sometimes you may make a change to your plugin that causes it to break, preventing `llm` from starting. For example you may see an error like this one:
|
||||
|
||||
```
|
||||
$ llm 'hi'
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
File llm-markov/llm_markov.py", line 10
|
||||
register(Markov()):
|
||||
^
|
||||
SyntaxError: invalid syntax
|
||||
```
|
||||
You may find that you are unable to uninstall the plugin using `llm uninstall llm-markov` because the command itself fails with the same error.
|
||||
|
||||
Should this happen, you can uninstall the plugin after first disabling it using the {ref}`LLM_LOAD_PLUGINS <llm-load-plugins>` environment variable like this:
|
||||
```bash
|
||||
LLM_LOAD_PLUGINS='' llm uninstall llm-markov
|
||||
```
|
||||
+1006
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
||||
(related-tools)=
|
||||
# Related tools
|
||||
|
||||
The following tools are designed to be used with LLM:
|
||||
|
||||
(related-tools-strip-tags)=
|
||||
## strip-tags
|
||||
|
||||
[strip-tags](https://github.com/simonw/strip-tags) is a command for stripping tags from HTML. This is useful when working with LLMs because HTML tags can use up a lot of your token budget.
|
||||
|
||||
Here's how to summarize the front page of the New York Times, by both stripping tags and filtering to just the elements with `class="story-wrapper"`:
|
||||
|
||||
```bash
|
||||
curl -s https://www.nytimes.com/ \
|
||||
| strip-tags .story-wrapper \
|
||||
| llm -s 'summarize the news'
|
||||
```
|
||||
|
||||
[llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) describes ways to use `strip-tags` in more detail.
|
||||
|
||||
(related-tools-ttok)=
|
||||
## ttok
|
||||
|
||||
[ttok](https://github.com/simonw/ttok) is a command-line tool for counting OpenAI tokens. You can use it to check if input is likely to fit in the token limit for GPT 3.5 or GPT4:
|
||||
|
||||
```bash
|
||||
cat my-file.txt | ttok
|
||||
```
|
||||
```
|
||||
125
|
||||
```
|
||||
It can also truncate input down to a desired number of tokens:
|
||||
```bash
|
||||
ttok This is too many tokens -t 3
|
||||
```
|
||||
```
|
||||
This is too
|
||||
```
|
||||
This is useful for truncating a large document down to a size where it can be processed by an LLM.
|
||||
|
||||
(related-tools-symbex)=
|
||||
## Symbex
|
||||
|
||||
[Symbex](https://github.com/simonw/symbex) is a tool for searching for symbols in Python codebases. It's useful for extracting just the code for a specific problem and then piping that into LLM for explanation, refactoring or other tasks.
|
||||
|
||||
Here's how to use it to find all functions that match `test*csv*` and use those to guess what the software under test does:
|
||||
|
||||
```bash
|
||||
symbex 'test*csv*' | \
|
||||
llm --system 'based on these tests guess what this tool does'
|
||||
```
|
||||
It can also be used to export symbols in a format that can be piped to {ref}`llm embed-multi <embeddings-cli-embed-multi>` in order to create embeddings:
|
||||
```bash
|
||||
symbex '*' '*:*' --nl | \
|
||||
llm embed-multi symbols - \
|
||||
--format nl --database embeddings.db --store
|
||||
```
|
||||
For more examples see [Symbex: search Python code for functions and classes, then pipe them into a LLM](https://simonwillison.net/2023/Jun/18/symbex/).
|
||||
@@ -0,0 +1,7 @@
|
||||
sphinx==7.2.6
|
||||
furo==2023.9.10
|
||||
sphinx-autobuild
|
||||
sphinx-copybutton
|
||||
sphinx-markdown-builder==0.6.8
|
||||
myst-parser
|
||||
cogapp
|
||||
+599
@@ -0,0 +1,599 @@
|
||||
(schemas)=
|
||||
|
||||
# Schemas
|
||||
|
||||
Large Language Models are very good at producing structured output as JSON or other formats. LLM's **schemas** feature allows you to define the exact structure of JSON data you want to receive from a model.
|
||||
|
||||
This feature is supported by models from OpenAI, Anthropic, Google Gemini and can be implemented for others {ref}`via plugins <advanced-model-plugins-schemas>`.
|
||||
|
||||
This page describes schemas used via the `llm` command-line tool. Schemas can also be used from the {ref}`Python API <python-api-schemas>`.
|
||||
|
||||
(schemas-tutorial)=
|
||||
|
||||
## Schemas tutorial
|
||||
|
||||
In this tutorial we're going to use schemas to analyze some news stories.
|
||||
|
||||
But first, let's invent some dogs!
|
||||
|
||||
### Getting started with dogs
|
||||
|
||||
LLMs are great at creating test data. Let's define a simple schema for a dog, using LLM's {ref}`concise schema syntax <schemas-dsl>`. We'll pass that to LLm with `llm --schema` and prompt it to "invent a cool dog":
|
||||
```bash
|
||||
llm --schema 'name, age int, one_sentence_bio' 'invent a cool dog'
|
||||
```
|
||||
I got back Ziggy:
|
||||
```json
|
||||
{
|
||||
"name": "Ziggy",
|
||||
"age": 4,
|
||||
"one_sentence_bio": "Ziggy is a hyper-intelligent, bioluminescent dog who loves to perform tricks in the dark and guides his owner home using his glowing fur."
|
||||
}
|
||||
```
|
||||
The response matched my schema, with `name` and `one_sentence_bio` string columns and an integer for `age`.
|
||||
|
||||
We're using the default LLM model here - `gpt-4o-mini`. Add `-m model` to use another model - for example use `-m o3-mini` to have O3 mini invent some dogs.
|
||||
|
||||
For a list of available models that support schemas, run this command:
|
||||
```bash
|
||||
llm models --schemas
|
||||
```
|
||||
|
||||
Want several more dogs? You can pass in that same schema using `--schema-multi` and ask for several at once:
|
||||
```bash
|
||||
llm --schema-multi 'name, age int, one_sentence_bio' 'invent 3 really cool dogs'
|
||||
```
|
||||
Here's what I got:
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"name": "Echo",
|
||||
"age": 3,
|
||||
"one_sentence_bio": "Echo is a sleek, silvery-blue Siberian Husky with mesmerizing blue eyes and a talent for mimicking sounds, making him a natural entertainer."
|
||||
},
|
||||
{
|
||||
"name": "Nova",
|
||||
"age": 2,
|
||||
"one_sentence_bio": "Nova is a vibrant, spotted Dalmatian with an adventurous spirit and a knack for agility courses, always ready to leap into action."
|
||||
},
|
||||
{
|
||||
"name": "Pixel",
|
||||
"age": 4,
|
||||
"one_sentence_bio": "Pixel is a playful, tech-savvy Poodle with a rainbow-colored coat, known for her ability to interact with smart devices and her love for puzzle toys."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
So that's the basic idea: we can feed in a schema and LLM will pass it to the underlying model and (usually) get back JSON that conforms to that schema.
|
||||
|
||||
This stuff gets a _lot_ more useful when you start applying it to larger amounts of text, extracting structured details from unstructured content.
|
||||
|
||||
### Extracting people from a news articles
|
||||
|
||||
We are going to extract details of the people who are mentioned in different news stories, and then use those to compile a database.
|
||||
|
||||
Let's start by compiling a schema. For each person mentioned we want to extract the following details:
|
||||
|
||||
- Their name
|
||||
- The organization they work for
|
||||
- Their role
|
||||
- What we learned about them from the story
|
||||
|
||||
We will also record the article headline and the publication date, to make things easier for us later on.
|
||||
|
||||
Using LLM's custom, concise schema language, this time with newlines separating the individual fields (for the dogs example we used commas):
|
||||
```
|
||||
name: the person's name
|
||||
organization: who they represent
|
||||
role: their job title or role
|
||||
learned: what we learned about them from this story
|
||||
article_headline: the headline of the story
|
||||
article_date: the publication date in YYYY-MM-DD
|
||||
```
|
||||
As you can see, this schema definition is pretty simple - each line has the name of a property we want to capture, then an optional: followed by a description, which doubles as instructions for the model.
|
||||
|
||||
The full syntax is {ref}`described below <schemas-dsl>` - you can also include type information for things like numbers.
|
||||
|
||||
Let's run this against a news article.
|
||||
|
||||
Visit [AP News](https://apnews.com/) and grab the URL to an article. I'm using this one:
|
||||
|
||||
https://apnews.com/article/trump-federal-employees-firings-a85d1aaf1088e050d39dcf7e3664bb9f
|
||||
|
||||
There's quite a lot of HTML on that page, possibly even enough to exceed GPT-4o mini's 128,000 token input limit. We'll use another tool called [strip-tags](https://github.com/simonw/strip-tags) to reduce that. If you have [uv](https://docs.astral.sh/uv/) installed you can call it using `uvx strip-tags`, otherwise you'll need to install it first:
|
||||
|
||||
```
|
||||
uv tool install strip-tags
|
||||
# Or "pip install" or "pipx install"
|
||||
```
|
||||
Now we can run this command to extract the people from that article:
|
||||
|
||||
```bash
|
||||
curl 'https://apnews.com/article/trump-federal-employees-firings-a85d1aaf1088e050d39dcf7e3664bb9f' | \
|
||||
uvx strip-tags | \
|
||||
llm --schema-multi "
|
||||
name: the person's name
|
||||
organization: who they represent
|
||||
role: their job title or role
|
||||
learned: what we learned about them from this story
|
||||
article_headline: the headline of the story
|
||||
article_date: the publication date in YYYY-MM-DD
|
||||
" --system 'extract people mentioned in this article'
|
||||
```
|
||||
The output I got started like this:
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"name": "William Alsup",
|
||||
"organization": "U.S. District Court",
|
||||
"role": "Judge",
|
||||
"learned": "He ruled that the mass firings of probationary employees were likely unlawful and criticized the authority exercised by the Office of Personnel Management.",
|
||||
"article_headline": "Judge finds mass firings of federal probationary workers were likely unlawful",
|
||||
"article_date": "2025-02-26"
|
||||
},
|
||||
{
|
||||
"name": "Everett Kelley",
|
||||
"organization": "American Federation of Government Employees",
|
||||
"role": "National President",
|
||||
"learned": "He hailed the court's decision as a victory for employees who were illegally fired.",
|
||||
"article_headline": "Judge finds mass firings of federal probationary workers were likely unlawful",
|
||||
"article_date": "2025-02-26"
|
||||
}
|
||||
```
|
||||
This data has been logged to LLM's {ref}`SQLite database <logging>`. We can retrieve the data back out again using the {ref}`llm logs <logging-view>` command like this:
|
||||
```bash
|
||||
llm logs -c --data
|
||||
```
|
||||
The `-c` flag means "use most recent conversation", and the `--data` flag outputs just the JSON data that was captured in the response.
|
||||
|
||||
We're going to want to use the same schema for other things. Schemas that we use are automatically logged to the database - we can view them using `llm schemas`:
|
||||
|
||||
```bash
|
||||
llm schemas
|
||||
```
|
||||
Here's the output:
|
||||
```
|
||||
- id: 3b7702e71da3dd791d9e17b76c88730e
|
||||
summary: |
|
||||
{items: [{name, organization, role, learned, article_headline, article_date}]}
|
||||
usage: |
|
||||
1 time, most recently 2025-02-28T04:50:02.032081+00:00
|
||||
```
|
||||
To view the full schema, run that command with `--full`:
|
||||
|
||||
```bash
|
||||
llm schemas --full
|
||||
```
|
||||
Which outputs:
|
||||
```
|
||||
- id: 3b7702e71da3dd791d9e17b76c88730e
|
||||
schema: |
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "the person's name"
|
||||
},
|
||||
...
|
||||
```
|
||||
That `3b7702e71da3dd791d9e17b76c88730e` ID can be used to run the same schema again. Let's try that now on a different URL:
|
||||
|
||||
```bash
|
||||
curl 'https://apnews.com/article/bezos-katy-perry-blue-origin-launch-4a074e534baa664abfa6538159c12987' | \
|
||||
uvx strip-tags | \
|
||||
llm --schema 3b7702e71da3dd791d9e17b76c88730e \
|
||||
--system 'extract people mentioned in this article'
|
||||
```
|
||||
Here we are using `--schema` because our schema ID already corresponds to an array of items.
|
||||
|
||||
The result starts like this:
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"name": "Katy Perry",
|
||||
"organization": "Blue Origin",
|
||||
"role": "Singer",
|
||||
"learned": "Katy Perry will join the all-female celebrity crew for a spaceflight organized by Blue Origin.",
|
||||
"article_headline": "Katy Perry and Gayle King will join Jeff Bezos’ fiancee Lauren Sanchez on Blue Origin spaceflight",
|
||||
"article_date": "2023-10-15"
|
||||
},
|
||||
```
|
||||
One more trick: let's turn our schema and system prompt combination into a {ref}`template <prompt-templates>`.
|
||||
|
||||
```bash
|
||||
llm --schema 3b7702e71da3dd791d9e17b76c88730e \
|
||||
--system 'extract people mentioned in this article' \
|
||||
--save people
|
||||
```
|
||||
This creates a new template called "people". We can confirm the template was created correctly using:
|
||||
```bash
|
||||
llm templates show people
|
||||
```
|
||||
Which will output the YAML version of the template looking like this:
|
||||
```yaml
|
||||
name: people
|
||||
schema_object:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
properties:
|
||||
article_date:
|
||||
description: the publication date in YYYY-MM-DD
|
||||
type: string
|
||||
article_headline:
|
||||
description: the headline of the story
|
||||
type: string
|
||||
learned:
|
||||
description: what we learned about them from this story
|
||||
type: string
|
||||
name:
|
||||
description: the person's name
|
||||
type: string
|
||||
organization:
|
||||
description: who they represent
|
||||
type: string
|
||||
role:
|
||||
description: their job title or role
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
- organization
|
||||
- role
|
||||
- learned
|
||||
- article_headline
|
||||
- article_date
|
||||
type: object
|
||||
type: array
|
||||
required:
|
||||
- items
|
||||
type: object
|
||||
system: extract people mentioned in this article
|
||||
```
|
||||
We can now run our people extractor against another fresh URL. Let's use one from The Guardian:
|
||||
```bash
|
||||
curl https://www.theguardian.com/commentisfree/2025/feb/27/billy-mcfarland-new-fyre-festival-fantasist | \
|
||||
strip-tags | llm -t people
|
||||
```
|
||||
Storing the schema in a template means we can just use `llm -t people` to run the prompt. Here's what I got back:
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"name": "Billy McFarland",
|
||||
"organization": "Fyre Festival",
|
||||
"role": "Organiser",
|
||||
"learned": "Billy McFarland is known for organizing the infamous Fyre Festival and was sentenced to six years in prison for wire fraud related to it. He is attempting to revive the festival with Fyre 2.",
|
||||
"article_headline": "Welcome back Billy McFarland and a new Fyre festival. Shows you can’t keep a good fantasist down",
|
||||
"article_date": "2025-02-27"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
Depending on the model, schema extraction may work against images and PDF files as well.
|
||||
|
||||
I took a screenshot of part of [this story in the Onion](https://theonion.com/mark-zuckerberg-insists-anyone-with-same-skewed-values-1826829272/) and saved it to the following URL:
|
||||
|
||||
https://static.simonwillison.net/static/2025/onion-zuck.jpg
|
||||
|
||||
We can pass that as an {ref}`attachment <usage-attachments>` using the `-a` option. This time let's use GPT-4o:
|
||||
|
||||
```bash
|
||||
llm -t people -a https://static.simonwillison.net/static/2025/onion-zuck.jpg -m gpt-4o
|
||||
```
|
||||
Which gave me back this:
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"name": "Mark Zuckerberg",
|
||||
"organization": "Facebook",
|
||||
"role": "CEO",
|
||||
"learned": "He addressed criticism by suggesting anyone with similar values and thirst for power could make the same mistakes.",
|
||||
"article_headline": "Mark Zuckerberg Insists Anyone With Same Skewed Values And Unrelenting Thirst For Power Could Have Made Same Mistakes",
|
||||
"article_date": "2018-06-14"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
Now that we've extracted people from a number of different sources, let's load them into a database.
|
||||
|
||||
The {ref}`llm logs <logging-view>` command has several features for working with logged JSON objects. Since we've been recording multiple objects from each page in an `"items"` array using our `people` template we can access those using the following command:
|
||||
|
||||
```bash
|
||||
llm logs --schema t:people --data-key items
|
||||
```
|
||||
In place of `t:people` we could use the `3b7702e71da3dd791d9e17b76c88730e` schema ID or even the original schema string instead, see {ref}`specifying a schema <schemas-specify>`.
|
||||
|
||||
This command outputs newline-delimited JSON for every item that has been captured using the specified schema:
|
||||
```json
|
||||
{"name": "Katy Perry", "organization": "Blue Origin", "role": "Singer", "learned": "She is one of the passengers on the upcoming spaceflight with Blue Origin."}
|
||||
{"name": "Gayle King", "organization": "Blue Origin", "role": "TV Journalist", "learned": "She is participating in the upcoming Blue Origin spaceflight."}
|
||||
{"name": "Lauren Sanchez", "organization": "Blue Origin", "role": "Helicopter Pilot and former TV Journalist", "learned": "She selected the crew for the Blue Origin spaceflight."}
|
||||
{"name": "Aisha Bowe", "organization": "Engineering firm", "role": "Former NASA Rocket Scientist", "learned": "She is part of the crew for the spaceflight."}
|
||||
{"name": "Amanda Nguyen", "organization": "Research Scientist", "role": "Activist and Scientist", "learned": "She is included in the crew for the upcoming Blue Origin flight."}
|
||||
{"name": "Kerianne Flynn", "organization": "Movie Producer", "role": "Producer", "learned": "She will also be a passenger on the upcoming spaceflight."}
|
||||
{"name": "Billy McFarland", "organization": "Fyre Festival", "role": "Organiser", "learned": "He was sentenced to six years in prison for wire fraud in 2018 and has launched a new festival called Fyre 2.", "article_headline": "Welcome back Billy McFarland and a new Fyre festival. Shows you can\u2019t keep a good fantasist down", "article_date": "2025-02-27"}
|
||||
{"name": "Mark Zuckerberg", "organization": "Facebook", "role": "CEO", "learned": "He attempted to dismiss criticism by suggesting that anyone with similar values and thirst for power could have made the same mistakes.", "article_headline": "Mark Zuckerberg Insists Anyone With Same Skewed Values And Unrelenting Thirst For Power Could Have Made Same Mistakes", "article_date": "2018-06-14"}
|
||||
```
|
||||
If we add `--data-array` we'll get back a valid JSON array of objects instead:
|
||||
```bash
|
||||
llm logs --schema t:people --data-key items --data-array
|
||||
```
|
||||
Output starts:
|
||||
```json
|
||||
[{"name": "Katy Perry", "organization": "Blue Origin", "role": "Singer", "learned": "She is one of the passengers on the upcoming spaceflight with Blue Origin."},
|
||||
{"name": "Gayle King", "organization": "Blue Origin", "role": "TV Journalist", "learned": "She is participating in the upcoming Blue Origin spaceflight."},
|
||||
```
|
||||
|
||||
We can load this into a SQLite database using [sqlite-utils](https://sqlite-utils.datasette.io/), in particular the [sqlite-utils insert](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-json-data) command.
|
||||
|
||||
```bash
|
||||
uv tool install sqlite-utils
|
||||
# or pip install or pipx install
|
||||
```
|
||||
Now we can pipe the JSON into that tool to create a database with a `people` table:
|
||||
```bash
|
||||
llm logs --schema t:people --data-key items --data-array | \
|
||||
sqlite-utils insert data.db people -
|
||||
```
|
||||
To see a table of the name, organization and role columns use [sqlite-utils rows](https://sqlite-utils.datasette.io/en/stable/cli.html#returning-all-rows-in-a-table):
|
||||
```bash
|
||||
sqlite-utils rows data.db people -t -c name -c organization -c role
|
||||
```
|
||||
Which produces:
|
||||
```
|
||||
name organization role
|
||||
--------------- ------------------ -----------------------------------------
|
||||
Katy Perry Blue Origin Singer
|
||||
Gayle King Blue Origin TV Journalist
|
||||
Lauren Sanchez Blue Origin Helicopter Pilot and former TV Journalist
|
||||
Aisha Bowe Engineering firm Former NASA Rocket Scientist
|
||||
Amanda Nguyen Research Scientist Activist and Scientist
|
||||
Kerianne Flynn Movie Producer Producer
|
||||
Billy McFarland Fyre Festival Organiser
|
||||
Mark Zuckerberg Facebook CEO
|
||||
```
|
||||
We can also explore the database in a web interface using [Datasette](https://datasette.io/):
|
||||
|
||||
```bash
|
||||
uvx datasette data.db
|
||||
# Or install datasette first:
|
||||
uv tool install datasette # or pip install or pipx install
|
||||
datasette data.db
|
||||
```
|
||||
Visit `http://127.0.0.1:8001/data/people` to start navigating the data.
|
||||
|
||||
(schemas-json-schemas)=
|
||||
|
||||
## Using JSON schemas
|
||||
|
||||
The above examples have both used {ref}`concise schema syntax <schemas-dsl>`. LLM converts this format to [JSON schema](https://json-schema.org/), and you can use JSON schema directly yourself if you wish.
|
||||
|
||||
JSON schema covers the following:
|
||||
|
||||
- The data types of fields (string, number, array, object, etc.)
|
||||
- Required vs. optional fields
|
||||
- Nested data structures
|
||||
- Constraints on values (minimum/maximum, patterns, etc.)
|
||||
- Descriptions of those fields - these can be used to guide the language model
|
||||
|
||||
Different models may support different subsets of the overall JSON schema language. You should experiment to figure out what works for the model you are using.
|
||||
|
||||
LLM recommends that the top level of the schema is an object, not an array, for increased compatibility across multiple models. I suggest using `{"items": [array of objects]}` if you want to return an array.
|
||||
|
||||
The dogs schema above, `name, age int, one_sentence_bio`, would look like this as a full JSON schema:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"age": {
|
||||
"type": "integer"
|
||||
},
|
||||
"one_sentence_bio": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"age",
|
||||
"one_sentence_bio"
|
||||
]
|
||||
}
|
||||
```
|
||||
This JSON can be passed directly to the `--schema` option, or saved in a file and passed as the filename.
|
||||
```bash
|
||||
llm --schema '{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"age": {
|
||||
"type": "integer"
|
||||
},
|
||||
"one_sentence_bio": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"age",
|
||||
"one_sentence_bio"
|
||||
]
|
||||
}' 'a surprising dog'
|
||||
```
|
||||
Example output:
|
||||
```json
|
||||
{
|
||||
"name": "Baxter",
|
||||
"age": 3,
|
||||
"one_sentence_bio": "Baxter is a rescue dog who learned to skateboard and now performs tricks at local parks, astonishing everyone with his skill!"
|
||||
}
|
||||
```
|
||||
|
||||
(schemas-specify)=
|
||||
|
||||
## Ways to specify a schema
|
||||
|
||||
LLM accepts schema definitions for both running prompts and exploring logged responses, using the `--schema` option.
|
||||
|
||||
This option can take multiple forms:
|
||||
|
||||
- A string providing a JSON schema: `--schema '{"type": "object", ...}'`
|
||||
- A {ref}`condensed schema definition <schemas-dsl>`: `--schema 'name,age int'`
|
||||
- The name or path of a file on disk containing a JSON schema: `--schema dogs.schema.json`
|
||||
- The hexadecimal ID of a previously logged schema: `--schema 520f7aabb121afd14d0c6c237b39ba2d` - these IDs can be found using the `llm schemas` command.
|
||||
- A schema that has been {ref}`saved in a template <prompt-templates-save>`: `--schema t:name-of-template`, see {ref}`schemas-reusable`.
|
||||
|
||||
(schemas-dsl)=
|
||||
|
||||
## Concise LLM schema syntax
|
||||
|
||||
JSON schema's can be time-consuming to construct by hand. LLM also supports a concise alternative syntax for specifying a schema.
|
||||
|
||||
A simple schema for an object with two string properties called `name` and `bio` looks like this:
|
||||
|
||||
name, bio
|
||||
|
||||
You can include type information by adding a type indicator after the property name, separated by a space.
|
||||
|
||||
name, bio, age int
|
||||
|
||||
Supported types are `int` for integers, `float` for floating point numbers, `str` for strings (the default) and `bool` for true/false booleans.
|
||||
|
||||
To include a description of the field to act as a hint to the model, add one after a colon:
|
||||
|
||||
name: the person's name, age int: their age, bio: a short bio
|
||||
|
||||
If your schema is getting long you can switch from comma-separated to newline-separated, which also allows you to use commas in those descriptions:
|
||||
|
||||
name: the person's name
|
||||
age int: their age
|
||||
bio: a short bio, no more than three sentences
|
||||
|
||||
You can experiment with the syntax using the `llm schemas dsl` command, which converts the input into a JSON schema:
|
||||
```bash
|
||||
llm schemas dsl 'name, age int'
|
||||
```
|
||||
Output:
|
||||
```json
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"age": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"age"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The Python utility function `llm.schema_dsl(schema)` can be used to convert this syntax into the equivalent JSON schema dictionary when working with schemas {ref}`in the Python API <python-api-schemas>`.
|
||||
|
||||
(schemas-reusable)=
|
||||
|
||||
## Saving reusable schemas in templates
|
||||
|
||||
If you want to store a schema with a name so you can reuse it easily in the future, the easiest way to do so is to save it {ref}`in a template <prompt-templates-schemas>`.
|
||||
|
||||
The quickest way to do that is with the `llm --save` option:
|
||||
|
||||
```bash
|
||||
llm --schema 'name, age int, one_sentence_bio' --save dog
|
||||
```
|
||||
Now you can use it like this:
|
||||
```bash
|
||||
llm --schema t:dog 'invent a dog'
|
||||
```
|
||||
Or:
|
||||
```bash
|
||||
llm --schema-multi t:dog 'invent three dogs'
|
||||
```
|
||||
(schemas-logs)=
|
||||
|
||||
## Browsing logged JSON objects created using schemas
|
||||
|
||||
By default, all JSON produced using schemas is logged to {ref}`a SQLite database <logging>`. You can use special options to the `llm logs` command to extract just those JSON objects in a useful format.
|
||||
|
||||
The `llm logs --schema X` filter option can be used to filter just for responses that were created using the specified schema. You can pass the full schema JSON, a path to the schema on disk or the schema ID.
|
||||
|
||||
The `--data` option causes just the JSON data collected by that schema to be outputted, as newline-delimited JSON.
|
||||
|
||||
If you instead want a JSON array of objects (with starting and ending square braces) you can use `--data-array` instead.
|
||||
|
||||
Let's invent some dogs:
|
||||
|
||||
```bash
|
||||
llm --schema-multi 'name, ten_word_bio' 'invent 3 cool dogs'
|
||||
llm --schema-multi 'name, ten_word_bio' 'invent 2 cool dogs'
|
||||
```
|
||||
Having logged these cool dogs, you can see just the data that was returned by those prompts like this:
|
||||
```bash
|
||||
llm logs --schema-multi 'name, ten_word_bio' --data
|
||||
```
|
||||
We need to use `--schema-multi` here because we used that when we first created these records. The `--schema` option is also supported, and can be passed a filename or JSON schema or schema ID as well.
|
||||
|
||||
Output:
|
||||
```
|
||||
{"items": [{"name": "Robo", "ten_word_bio": "A cybernetic dog with laser eyes and super intelligence."}, {"name": "Flamepaw", "ten_word_bio": "Fire-resistant dog with a talent for agility and tricks."}]}
|
||||
{"items": [{"name": "Bolt", "ten_word_bio": "Lightning-fast border collie, loves frisbee and outdoor adventures."}, {"name": "Luna", "ten_word_bio": "Mystical husky with mesmerizing blue eyes, enjoys snow and play."}, {"name": "Ziggy", "ten_word_bio": "Quirky pug who loves belly rubs and quirky outfits."}]}
|
||||
```
|
||||
Note that the dogs are nested in that `"items"` key. To access the list of items from that key use `--data-key items`:
|
||||
```bash
|
||||
llm logs --schema-multi 'name, ten_word_bio' --data-key items
|
||||
```
|
||||
Output:
|
||||
```
|
||||
{"name": "Bolt", "ten_word_bio": "Lightning-fast border collie, loves frisbee and outdoor adventures."}
|
||||
{"name": "Luna", "ten_word_bio": "Mystical husky with mesmerizing blue eyes, enjoys snow and play."}
|
||||
{"name": "Ziggy", "ten_word_bio": "Quirky pug who loves belly rubs and quirky outfits."}
|
||||
{"name": "Robo", "ten_word_bio": "A cybernetic dog with laser eyes and super intelligence."}
|
||||
{"name": "Flamepaw", "ten_word_bio": "Fire-resistant dog with a talent for agility and tricks."}
|
||||
```
|
||||
Finally, to output a JSON array instead of newline-delimited JSON use `--data-array`:
|
||||
```bash
|
||||
llm logs --schema-multi 'name, ten_word_bio' --data-key items --data-array
|
||||
```
|
||||
Output:
|
||||
```json
|
||||
[{"name": "Bolt", "ten_word_bio": "Lightning-fast border collie, loves frisbee and outdoor adventures."},
|
||||
{"name": "Luna", "ten_word_bio": "Mystical husky with mesmerizing blue eyes, enjoys snow and play."},
|
||||
{"name": "Ziggy", "ten_word_bio": "Quirky pug who loves belly rubs and quirky outfits."},
|
||||
{"name": "Robo", "ten_word_bio": "A cybernetic dog with laser eyes and super intelligence."},
|
||||
{"name": "Flamepaw", "ten_word_bio": "Fire-resistant dog with a talent for agility and tricks."}]
|
||||
```
|
||||
Add `--data-ids` to include `"response_id"` and `"conversation_id"` fields in each of the returned objects reflecting the database IDs of the response and conversation they were a part of. This can be useful for tracking the source of each individual row.
|
||||
|
||||
```bash
|
||||
llm logs --schema-multi 'name, ten_word_bio' --data-key items --data-ids
|
||||
```
|
||||
Output:
|
||||
```json
|
||||
{"name": "Nebula", "ten_word_bio": "A cosmic puppy with starry fur, loves adventures in space.", "response_id": "01jn4dawj8sq0c6t3emf4k5ryx", "conversation_id": "01jn4dawj8sq0c6t3emf4k5ryx"}
|
||||
{"name": "Echo", "ten_word_bio": "A clever hound with extraordinary hearing, master of hide-and-seek.", "response_id": "01jn4dawj8sq0c6t3emf4k5ryx", "conversation_id": "01jn4dawj8sq0c6t3emf4k5ryx"}
|
||||
{"name": "Biscuit", "ten_word_bio": "An adorable chef dog, bakes treats that everyone loves.", "response_id": "01jn4dawj8sq0c6t3emf4k5ryx", "conversation_id": "01jn4dawj8sq0c6t3emf4k5ryx"}
|
||||
{"name": "Cosmo", "ten_word_bio": "Galactic explorer, loves adventures and chasing shooting stars.", "response_id": "01jn4daycb3svj0x7kvp7zrp4q", "conversation_id": "01jn4daycb3svj0x7kvp7zrp4q"}
|
||||
{"name": "Pixel", "ten_word_bio": "Tech-savvy pup, builds gadgets and loves virtual playtime.", "response_id": "01jn4daycb3svj0x7kvp7zrp4q", "conversation_id": "01jn4daycb3svj0x7kvp7zrp4q"}
|
||||
```
|
||||
If a row already has a property called `"conversation_id"` or `"response_id"` additional underscores will be appended to the ID key until it no longer overlaps with the existing keys.
|
||||
|
||||
The `--id-gt $ID` and `--id-gte $ID` options can be useful for ignoring logged schema data prior to a certain point, see {ref}`logging-filter-id` for details.
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
# Setup
|
||||
|
||||
## Installation
|
||||
|
||||
Install this tool using `pip`:
|
||||
```bash
|
||||
pip install llm
|
||||
```
|
||||
Or using [pipx](https://pypa.github.io/pipx/):
|
||||
```bash
|
||||
pipx install llm
|
||||
```
|
||||
Or using [uv](https://docs.astral.sh/uv/guides/tools/) ({ref}`more tips below <setup-uvx>`):
|
||||
```bash
|
||||
uv tool install llm
|
||||
```
|
||||
Or using [Homebrew](https://brew.sh/) (see {ref}`warning note <homebrew-warning>`):
|
||||
```bash
|
||||
brew install llm
|
||||
```
|
||||
|
||||
## Upgrading to the latest version
|
||||
|
||||
If you installed using `pip`:
|
||||
```bash
|
||||
pip install -U llm
|
||||
```
|
||||
For `pipx`:
|
||||
```bash
|
||||
pipx upgrade llm
|
||||
```
|
||||
For `uv`:
|
||||
```bash
|
||||
uv tool upgrade llm
|
||||
```
|
||||
For Homebrew:
|
||||
```bash
|
||||
brew upgrade llm
|
||||
```
|
||||
If the latest version is not yet available on Homebrew you can upgrade like this instead:
|
||||
```bash
|
||||
llm install -U llm
|
||||
```
|
||||
|
||||
(setup-uvx)=
|
||||
## Using uvx
|
||||
|
||||
If you have [uv](https://docs.astral.sh/uv/) installed you can also use the `uvx` command to try LLM without first installing it like this:
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY='sx-...'
|
||||
uvx llm 'fun facts about skunks'
|
||||
```
|
||||
This will install and run LLM using a temporary virtual environment.
|
||||
|
||||
You can use the `--with` option to add extra plugins. To use Anthropic's models, for example:
|
||||
```bash
|
||||
export ANTHROPIC_API_KEY='...'
|
||||
uvx --with llm-anthropic llm -m claude-3.5-haiku 'fun facts about skunks'
|
||||
```
|
||||
All of the usual LLM commands will work with `uvx llm`. Here's how to set your OpenAI key without needing an environment variable for example:
|
||||
```bash
|
||||
uvx llm keys set openai
|
||||
# Paste key here
|
||||
```
|
||||
|
||||
(homebrew-warning)=
|
||||
## A note about Homebrew and PyTorch
|
||||
|
||||
The version of LLM packaged for Homebrew currently uses Python 3.12. The PyTorch project do not yet have a stable release of PyTorch for that version of Python.
|
||||
|
||||
This means that LLM plugins that depend on PyTorch such as [llm-sentence-transformers](https://github.com/simonw/llm-sentence-transformers) may not install cleanly with the Homebrew version of LLM.
|
||||
|
||||
You can workaround this by manually installing PyTorch before installing `llm-sentence-transformers`:
|
||||
|
||||
```bash
|
||||
llm install llm-python
|
||||
llm python -m pip install \
|
||||
--pre torch torchvision \
|
||||
--index-url https://download.pytorch.org/whl/nightly/cpu
|
||||
llm install llm-sentence-transformers
|
||||
```
|
||||
This should produce a working installation of that plugin.
|
||||
|
||||
## Installing plugins
|
||||
|
||||
{ref}`plugins` can be used to add support for other language models, including models that can run on your own device.
|
||||
|
||||
For example, the [llm-gpt4all](https://github.com/simonw/llm-gpt4all) plugin adds support for 17 new models that can be installed on your own machine. You can install that like so:
|
||||
```bash
|
||||
llm install llm-gpt4all
|
||||
```
|
||||
|
||||
(api-keys)=
|
||||
## API key management
|
||||
|
||||
Many LLM models require an API key. These API keys can be provided to this tool using several different mechanisms.
|
||||
|
||||
You can obtain an API key for OpenAI's language models from [the API keys page](https://platform.openai.com/api-keys) on their site.
|
||||
|
||||
### Saving and using stored keys
|
||||
|
||||
The easiest way to store an API key is to use the `llm keys set` command:
|
||||
|
||||
```bash
|
||||
llm keys set openai
|
||||
```
|
||||
You will be prompted to enter the key like this:
|
||||
```
|
||||
% llm keys set openai
|
||||
Enter key:
|
||||
```
|
||||
Once stored, this key will be automatically used for subsequent calls to the API:
|
||||
|
||||
```bash
|
||||
llm "Five ludicrous names for a pet lobster"
|
||||
```
|
||||
|
||||
You can list the names of keys that have been set using this command:
|
||||
|
||||
```bash
|
||||
llm keys
|
||||
```
|
||||
|
||||
Keys that are stored in this way live in a file called `keys.json`. This file is located at the path shown when you run the following command:
|
||||
|
||||
```bash
|
||||
llm keys path
|
||||
```
|
||||
|
||||
On macOS this will be `~/Library/Application Support/io.datasette.llm/keys.json`. On Linux it may be something like `~/.config/io.datasette.llm/keys.json`.
|
||||
|
||||
### Passing keys using the --key option
|
||||
|
||||
Keys can be passed directly using the `--key` option, like this:
|
||||
|
||||
```bash
|
||||
llm "Five names for pet weasels" --key sk-my-key-goes-here
|
||||
```
|
||||
You can also pass the alias of a key stored in the `keys.json` file. For example, if you want to maintain a personal API key you could add that like this:
|
||||
```bash
|
||||
llm keys set personal
|
||||
```
|
||||
And then use it for prompts like so:
|
||||
|
||||
```bash
|
||||
llm "Five friendly names for a pet skunk" --key personal
|
||||
```
|
||||
|
||||
### Keys in environment variables
|
||||
|
||||
Keys can also be set using an environment variable. These are different for different models.
|
||||
|
||||
For OpenAI models the key will be read from the `OPENAI_API_KEY` environment variable.
|
||||
|
||||
The environment variable will be used if no `--key` option is passed to the command and there is not a key configured in `keys.json`
|
||||
|
||||
To use an environment variable in place of the `keys.json` key run the prompt like this:
|
||||
```bash
|
||||
llm 'my prompt' --key $OPENAI_API_KEY
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
You can configure LLM in a number of different ways.
|
||||
|
||||
(setup-default-model)=
|
||||
### Setting a custom default model
|
||||
|
||||
The model used when calling `llm` without the `-m/--model` option defaults to `gpt-4o-mini` - the fastest and least expensive OpenAI model.
|
||||
|
||||
You can use the `llm models default` command to set a different default model. For GPT-4o (slower and more expensive, but more capable) run this:
|
||||
|
||||
```bash
|
||||
llm models default gpt-4o
|
||||
```
|
||||
You can view the current model by running this:
|
||||
```
|
||||
llm models default
|
||||
```
|
||||
Any of the supported aliases for a model can be passed to this command.
|
||||
|
||||
### Setting a custom directory location
|
||||
|
||||
This tool stores various files - prompt templates, stored keys, preferences, a database of logs - in a directory on your computer.
|
||||
|
||||
On macOS this is `~/Library/Application Support/io.datasette.llm/`.
|
||||
|
||||
On Linux it may be something like `~/.config/io.datasette.llm/`.
|
||||
|
||||
You can set a custom location for this directory by setting the `LLM_USER_PATH` environment variable:
|
||||
|
||||
```bash
|
||||
export LLM_USER_PATH=/path/to/my/custom/directory
|
||||
```
|
||||
### Turning SQLite logging on and off
|
||||
|
||||
By default, LLM will log every prompt and response you make to a SQLite database - see {ref}`logging` for more details.
|
||||
|
||||
You can turn this behavior off by default by running:
|
||||
```bash
|
||||
llm logs off
|
||||
```
|
||||
Or turn it back on again with:
|
||||
```
|
||||
llm logs on
|
||||
```
|
||||
Run `llm logs status` to see the current states of the setting.
|
||||
@@ -0,0 +1,387 @@
|
||||
(prompt-templates)=
|
||||
# Templates
|
||||
|
||||
A **template** can combine a prompt, system prompt, model, default model options, schema, and fragments into a single reusable unit.
|
||||
|
||||
Only one template can be used at a time. To compose multiple shorter pieces of prompts together consider using {ref}`fragments <fragments>` instead.
|
||||
|
||||
(prompt-templates-save)=
|
||||
|
||||
## Getting started with <code>--save</code>
|
||||
|
||||
The easiest way to create a template is using the `--save template_name` option.
|
||||
|
||||
Here's how to create a template for summarizing text:
|
||||
|
||||
```bash
|
||||
llm '$input - summarize this' --save summarize
|
||||
```
|
||||
Put `$input` where you would like the user's input to be inserted. If you omit this their input will be added to the end of your regular prompt:
|
||||
```bash
|
||||
llm 'Summarize the following: ' --save summarize
|
||||
```
|
||||
You can also create templates using system prompts:
|
||||
```bash
|
||||
llm --system 'Summarize this' --save summarize
|
||||
```
|
||||
You can set the default model for a template using `--model`:
|
||||
|
||||
```bash
|
||||
llm --system 'Summarize this' --model gpt-4o --save summarize
|
||||
```
|
||||
You can also save default options:
|
||||
```bash
|
||||
llm --system 'Speak in French' -o temperature 1.8 --save wild-french
|
||||
```
|
||||
If you want to include a literal `$` sign in your prompt, use `$$` instead:
|
||||
```bash
|
||||
llm --system 'Estimate the cost in $$ of this: $input' --save estimate
|
||||
```
|
||||
Use `--tool/-T` one or more times to add tools to the template:
|
||||
```bash
|
||||
llm -T llm_time --system 'Always include the current time in the answer' --save time
|
||||
```
|
||||
You can also use `--functions` to add Python function code directly to the template:
|
||||
```bash
|
||||
llm --functions 'def reverse_string(s): return s[::-1]' --system 'reverse any input' --save reverse
|
||||
llm -t reverse 'Hello, world!'
|
||||
```
|
||||
|
||||
Add `--schema` to bake a {ref}`schema <usage-schemas>` into your template:
|
||||
|
||||
```bash
|
||||
llm --schema dog.schema.json 'invent a dog' --save dog
|
||||
```
|
||||
|
||||
If you add `--extract` the setting to {ref}`extract the first fenced code block <usage-extract-fenced-code>` will be persisted in the template.
|
||||
```bash
|
||||
llm --system 'write a Python function' --extract --save python-function
|
||||
llm -t python-function 'calculate haversine distance between two points'
|
||||
```
|
||||
In each of these cases the template will be saved in YAML format in a dedicated directory on disk.
|
||||
|
||||
(prompt-templates-using)=
|
||||
|
||||
## Using a template
|
||||
|
||||
You can execute a named template using the `-t/--template` option:
|
||||
|
||||
```bash
|
||||
curl -s https://example.com/ | llm -t summarize
|
||||
```
|
||||
|
||||
This can be combined with the `-m` option to specify a different model:
|
||||
```bash
|
||||
curl -s https://llm.datasette.io/en/latest/ | \
|
||||
llm -t summarize -m gpt-3.5-turbo-16k
|
||||
```
|
||||
Templates can also be specified as a direct path to a YAML file on disk:
|
||||
```bash
|
||||
llm -t path/to/template.yaml 'extra prompt here'
|
||||
```
|
||||
Or as a URL to a YAML file hosted online:
|
||||
```bash
|
||||
llm -t https://raw.githubusercontent.com/simonw/llm-templates/refs/heads/main/python-app.yaml \
|
||||
'Python app to pick a random line from a file'
|
||||
```
|
||||
Note that templates loaded via URLs will have any `functions:` keys ignored, to avoid accidentally executing arbitrary code. This restriction also applies to templates loaded via the {ref}`template loaders plugin mechanism <plugin-hooks-register-template-loaders>`.
|
||||
|
||||
(prompt-templates-list)=
|
||||
|
||||
## Listing available templates
|
||||
|
||||
This command lists all available templates:
|
||||
```bash
|
||||
llm templates
|
||||
```
|
||||
The output looks something like this:
|
||||
```
|
||||
cmd : system: reply with macos terminal commands only, no extra information
|
||||
glados : system: You are GlaDOS prompt: Summarize this:
|
||||
```
|
||||
|
||||
(prompt-templates-yaml)=
|
||||
|
||||
## Templates as YAML files
|
||||
|
||||
Templates are stored as YAML files on disk.
|
||||
|
||||
You can edit (or create) a YAML file for a template using the `llm templates edit` command:
|
||||
```
|
||||
llm templates edit summarize
|
||||
```
|
||||
This will open the system default editor.
|
||||
|
||||
:::{tip}
|
||||
You can control which editor will be used here using the `EDITOR` environment variable - for example, to use VS Code:
|
||||
```bash
|
||||
export EDITOR="code -w"
|
||||
```
|
||||
Add that to your `~/.zshrc` or `~/.bashrc` file depending on which shell you use (`zsh` is the default on macOS since macOS Catalina in 2019).
|
||||
:::
|
||||
|
||||
You can create or edit template files directly in the templates directory. The location of this directory is shown by the `llm templates path` command:
|
||||
```bash
|
||||
llm templates path
|
||||
```
|
||||
Example output:
|
||||
```
|
||||
/Users/simon/Library/Application Support/io.datasette.llm/templates
|
||||
```
|
||||
|
||||
A basic YAML template looks like this:
|
||||
|
||||
```yaml
|
||||
prompt: 'Summarize this: $input'
|
||||
```
|
||||
Or use YAML multi-line strings for longer inputs. I created this using `llm templates edit steampunk`:
|
||||
```yaml
|
||||
prompt: >
|
||||
Summarize the following text.
|
||||
|
||||
Insert frequent satirical steampunk-themed illustrative anecdotes.
|
||||
Really go wild with that.
|
||||
|
||||
Text to summarize: $input
|
||||
```
|
||||
The `prompt: >` causes the following indented text to be treated as a single string, with newlines collapsed to spaces. Use `prompt: |` to preserve newlines.
|
||||
|
||||
Running that with `llm -t steampunk` against GPT-4o (via [strip-tags](https://github.com/simonw/strip-tags) to remove HTML tags from the input and minify whitespace):
|
||||
```bash
|
||||
curl -s 'https://til.simonwillison.net/macos/imovie-slides-and-audio' | \
|
||||
strip-tags -m | llm -t steampunk -m gpt-4o
|
||||
```
|
||||
Output:
|
||||
> In a fantastical steampunk world, Simon Willison decided to merge an old MP3 recording with slides from the talk using iMovie. After exporting the slides as images and importing them into iMovie, he had to disable the default Ken Burns effect using the "Crop" tool. Then, Simon manually synchronized the audio by adjusting the duration of each image. Finally, he published the masterpiece to YouTube, with the whimsical magic of steampunk-infused illustrations leaving his viewers in awe.
|
||||
|
||||
(prompt-templates-system)=
|
||||
|
||||
### System prompts
|
||||
|
||||
When working with models that support system prompts you can set a system prompt using a `system:` key like so:
|
||||
|
||||
```yaml
|
||||
system: Summarize this
|
||||
```
|
||||
If you specify only a system prompt you don't need to use the `$input` variable - `llm` will use the user's input as the whole of the regular prompt, which will then be processed using the instructions set in that system prompt.
|
||||
|
||||
You can combine system and regular prompts like so:
|
||||
|
||||
```yaml
|
||||
system: You speak like an excitable Victorian adventurer
|
||||
prompt: 'Summarize this: $input'
|
||||
```
|
||||
|
||||
(prompt-templates-fragments)=
|
||||
|
||||
### Fragments
|
||||
|
||||
Templates can reference {ref}`Fragments <fragments>` using the `fragments:` and `system_fragments:` keys. These should be a list of fragment URLs, filepaths or hashes:
|
||||
|
||||
```yaml
|
||||
fragments:
|
||||
- https://example.com/robots.txt
|
||||
- /path/to/file.txt
|
||||
- 993fd38d898d2b59fd2d16c811da5bdac658faa34f0f4d411edde7c17ebb0680
|
||||
system_fragments:
|
||||
- https://example.com/systm-prompt.txt
|
||||
```
|
||||
|
||||
(prompt-templates-options)=
|
||||
|
||||
### Options
|
||||
|
||||
Default options can be set using the `options:` key:
|
||||
|
||||
```yaml
|
||||
name: wild-french
|
||||
system: Speak in French
|
||||
options:
|
||||
temperature: 1.8
|
||||
```
|
||||
|
||||
(prompt-templates-tools)=
|
||||
|
||||
### Tools
|
||||
|
||||
The `tools:` key can provide a list of tool names from other plugins - either function names or toolbox specifiers:
|
||||
```yaml
|
||||
name: time-plus
|
||||
tools:
|
||||
- llm_time
|
||||
- Datasette("https://example.com/timezone-lookup")
|
||||
```
|
||||
The `functions:` key can provide a multi-line string of Python code defining additional functions:
|
||||
```yaml
|
||||
name: my-functions
|
||||
functions: |
|
||||
def reverse_string(s: str):
|
||||
return s[::-1]
|
||||
|
||||
def greet(name: str):
|
||||
return f"Hello, {name}!"
|
||||
```
|
||||
(prompt-templates-schemas)=
|
||||
|
||||
### Schemas
|
||||
|
||||
Use the `schema_object:` key to embed a JSON schema (as YAML) in your template. The easiest way to create these is with the `llm --schema ... --save name-of-template` command - the result should look something like this:
|
||||
|
||||
```yaml
|
||||
name: dogs
|
||||
schema_object:
|
||||
properties:
|
||||
dogs:
|
||||
items:
|
||||
properties:
|
||||
bio:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
type: object
|
||||
type: array
|
||||
type: object
|
||||
```
|
||||
|
||||
(prompt-templates-variables)=
|
||||
|
||||
### Additional template variables
|
||||
|
||||
Templates that work against the user's normal prompt input (content that is either piped to the tool via standard input or passed as a command-line argument) can use the `$input` variable.
|
||||
|
||||
You can use additional named variables. These will then need to be provided using the `-p/--param` option when executing the template.
|
||||
|
||||
Here's an example YAML template called `recipe`, which you can create using `llm templates edit recipe`:
|
||||
|
||||
```yaml
|
||||
prompt: |
|
||||
Suggest a recipe using ingredients: $ingredients
|
||||
|
||||
It should be based on cuisine from this country: $country
|
||||
```
|
||||
This can be executed like so:
|
||||
|
||||
```bash
|
||||
llm -t recipe -p ingredients 'sausages, milk' -p country Germany
|
||||
```
|
||||
My output started like this:
|
||||
> Recipe: German Sausage and Potato Soup
|
||||
>
|
||||
> Ingredients:
|
||||
> - 4 German sausages
|
||||
> - 2 cups whole milk
|
||||
|
||||
This example combines input piped to the tool with additional parameters. Call this `summarize`:
|
||||
|
||||
```yaml
|
||||
system: Summarize this text in the voice of $voice
|
||||
```
|
||||
Then to run it:
|
||||
```bash
|
||||
curl -s 'https://til.simonwillison.net/macos/imovie-slides-and-audio' | \
|
||||
strip-tags -m | llm -t summarize -p voice GlaDOS
|
||||
```
|
||||
I got this:
|
||||
|
||||
> My previous test subject seemed to have learned something new about iMovie. They exported keynote slides as individual images [...] Quite impressive for a human.
|
||||
|
||||
(prompt-default-parameters)=
|
||||
|
||||
### Specifying default parameters
|
||||
|
||||
When creating a template using the `--save` option you can pass `-p name value` to store the default values for parameters:
|
||||
```bash
|
||||
llm --system 'Summarize this text in the voice of $voice' \
|
||||
--model gpt-4o -p voice GlaDOS --save summarize
|
||||
```
|
||||
|
||||
You can specify default values for parameters in the YAML using the `defaults:` key.
|
||||
|
||||
```yaml
|
||||
system: Summarize this text in the voice of $voice
|
||||
defaults:
|
||||
voice: GlaDOS
|
||||
```
|
||||
|
||||
When running without `-p` it will choose the default:
|
||||
|
||||
```bash
|
||||
curl -s 'https://til.simonwillison.net/macos/imovie-slides-and-audio' | \
|
||||
strip-tags -m | llm -t summarize
|
||||
```
|
||||
|
||||
But you can override the defaults with `-p`:
|
||||
|
||||
```bash
|
||||
curl -s 'https://til.simonwillison.net/macos/imovie-slides-and-audio' | \
|
||||
strip-tags -m | llm -t summarize -p voice Yoda
|
||||
```
|
||||
|
||||
I got this:
|
||||
|
||||
> Text, summarize in Yoda's voice, I will: "Hmm, young padawan. Summary of this text, you seek. Hmmm. ...
|
||||
|
||||
(prompt-templates-extract)=
|
||||
|
||||
### Configuring code extraction
|
||||
|
||||
To configure the {ref}`extract first fenced code block <usage-extract-fenced-code>` setting for the template, add this:
|
||||
|
||||
```yaml
|
||||
extract: true
|
||||
```
|
||||
|
||||
(prompt-templates-default-model)=
|
||||
|
||||
### Setting a default model for a template
|
||||
|
||||
Templates executed using `llm -t template-name` will execute using the default model that the user has configured for the tool - or `gpt-3.5-turbo` if they have not configured their own default.
|
||||
|
||||
You can specify a new default model for a template using the `model:` key in the associated YAML. Here's a template called `roast`:
|
||||
|
||||
```yaml
|
||||
model: gpt-4o
|
||||
system: roast the user at every possible opportunity, be succinct
|
||||
```
|
||||
Example:
|
||||
```bash
|
||||
llm -t roast 'How are you today?'
|
||||
```
|
||||
> I'm doing great but with your boring questions, I must admit, I've seen more life in a cemetery.
|
||||
|
||||
(prompt-templates-loaders)=
|
||||
|
||||
## Template loaders from plugins
|
||||
|
||||
LLM plugins can {ref}`register prefixes <plugin-hooks-register-template-loaders>` that can be used to load templates from external sources.
|
||||
|
||||
[llm-templates-github](https://github.com/simonw/llm-templates-github) is an example which adds a `gh:` prefix which can be used to load templates from GitHub.
|
||||
|
||||
You can install that plugin like this:
|
||||
```bash
|
||||
llm install llm-templates-github
|
||||
```
|
||||
|
||||
Use the `llm templates loaders` command to see details of the registered loaders.
|
||||
|
||||
```bash
|
||||
llm templates loaders
|
||||
```
|
||||
Output:
|
||||
```
|
||||
gh:
|
||||
Load a template from GitHub or local cache if available
|
||||
|
||||
Format: username/repo/template_name (without the .yaml extension)
|
||||
or username/template_name which means username/llm-templates/template_name
|
||||
```
|
||||
|
||||
Then you can then use it like this:
|
||||
```bash
|
||||
curl -sL 'https://llm.datasette.io/' | llm -t gh:simonw/summarize
|
||||
```
|
||||
The `-sL` flags to `curl` are used to follow redirects and suppress progress meters.
|
||||
|
||||
This command will fetch the content of the LLM index page and feed it to the template defined by [summarize.yaml](https://github.com/simonw/llm-templates/blob/main/summarize.yaml) in the [simonw/llm-templates](https://github.com/simonw/llm-templates) GitHub repository.
|
||||
|
||||
If two template loader plugins attempt to register the same prefix one of them will have `_1` added to the end of their prefix. Use `llm templates loaders` to check if this has occurred.
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
(tools)=
|
||||
|
||||
# Tools
|
||||
|
||||
Many Large Language Models have been trained to execute tools as part of responding to a prompt. LLM supports tool usage with both the command-line interface and the Python API.
|
||||
|
||||
Exposing tools to LLMs **carries risks**! Be sure to read the {ref}`warning below <tools-warning>`.
|
||||
|
||||
(tools-how-they-work)=
|
||||
|
||||
## How tools work
|
||||
|
||||
A tool is effectively a function that the model can request to be executed. Here's how that works:
|
||||
|
||||
1. The initial prompt to the model includes a list of available tools, containing their names, descriptions and parameters.
|
||||
2. The model can choose to call one (or sometimes more than one) of those tools, returning a request for the tool to execute.
|
||||
3. The code that calls the model - in this case LLM itself - then executes the specified tool with the provided arguments.
|
||||
4. LLM prompts the model a second time, this time including the output of the tool execution.
|
||||
5. The model can then use that output to generate its next response.
|
||||
|
||||
This sequence can run several times in a loop, allowing the LLM to access data, act on that data and then pass that data off to other tools for further processing.
|
||||
|
||||
:::{admonition} Tools can be dangerous
|
||||
:class: danger
|
||||
|
||||
(tools-warning)=
|
||||
|
||||
## Warning: Tools can be dangerous
|
||||
|
||||
Applications built on top of LLMs suffer from a class of attacks called [prompt injection](https://simonwillison.net/tags/prompt-injection/) attacks. These occur when a malicious third party injects content into the LLM which causes it to take tool-based actions that act against the interests of the user of that application.
|
||||
|
||||
Be very careful about which tools you enable when you potentially might be exposed to untrusted sources of content - web pages, GitHub issues posted by other people, email and messages that have been sent to you that could come from an attacker.
|
||||
|
||||
Watch out for [the lethal trifecta](https://simonwillison.net/2025/Jun/16/the-lethal-trifecta/) of prompt injection exfiltration attacks. If your tool-enabled LLM has the following:
|
||||
|
||||
- access to private data
|
||||
- exposure to malicious instructions
|
||||
- the ability to exfiltrate information
|
||||
|
||||
Anyone who can feed malicious instructions into your LLM - by leaving them on a web page it visits, or sending an email to an inbox that it monitors - could be able to trick your LLM into using other tools to access your private information and then exfiltrate (pass out) that data to somewhere the attacker can see it.
|
||||
:::
|
||||
|
||||
(tools-trying-out)=
|
||||
|
||||
## Trying out tools
|
||||
|
||||
LLM comes with a default tool installed, called `llm_version`. You can try that out like this:
|
||||
|
||||
```bash
|
||||
llm --tool llm_version "What version of LLM is this?" --td
|
||||
```
|
||||
You can also use `-T llm_version` as a shortcut for `--tool llm_version`.
|
||||
|
||||
The output should look like this:
|
||||
```
|
||||
Tool call: llm_version({})
|
||||
0.26a0
|
||||
|
||||
The installed version of the LLM is 0.26a0.
|
||||
```
|
||||
Further tools can be installed using plugins, or you can use the `llm --functions` option to pass tools implemented as PYthon functions directly, as {ref}`described here <usage-tools>`.
|
||||
|
||||
(tools-implementation)=
|
||||
|
||||
## LLM's implementation of tools
|
||||
|
||||
In LLM every tool is defined as a Python function. The function can take any number of arguments and can return a string or an object that can be converted to a string.
|
||||
|
||||
Tool functions should include a docstring that describes what the function does. This docstring will become the description that is passed to the model.
|
||||
|
||||
Tools can also be defined as {ref}`toolbox classes <python-api-toolbox>`, a subclass of `llm.Toolbox` that allows multiple related tools to be bundled together. Toolbox classes can be configured when they are instantiated, and can also maintain state in between multiple tool calls.
|
||||
|
||||
The Python API can accept functions directly. The command-line interface has two ways for tools to be defined: via plugins that implement the {ref}`register_tools() plugin hook <plugin-hooks-register-tools>`, or directly on the command-line using the `--functions` argument to specify a block of Python code defining one or more functions - or a path to a Python file containing the same.
|
||||
|
||||
You can use tools {ref}`with the LLM command-line tool <usage-tools>` or {ref}`with the Python API <python-api-tools>`.
|
||||
|
||||
(tools-default)=
|
||||
|
||||
## Default tools
|
||||
|
||||
LLM includes some default tools for you to try out:
|
||||
|
||||
- `llm_version()` returns the current version of LLM
|
||||
- `llm_time()` returns the current local and UTC time
|
||||
|
||||
Try them like this:
|
||||
|
||||
```bash
|
||||
llm -T llm_version -T llm_time 'Give me the current time and LLM version' --td
|
||||
```
|
||||
|
||||
(tools-tips)=
|
||||
|
||||
## Tips for implementing tools
|
||||
|
||||
Consult the {ref}`register_tools() plugin hook <plugin-hooks-register-tools>` documentation for examples of how to implement tools in plugins.
|
||||
|
||||
If your plugin needs access to API secrets I recommend storing those using `llm keys set api-name` and then reading them using the {ref}`plugin-utilities-get-key` utility function. This avoids secrets being logged to the database as part of tool calls.
|
||||
|
||||
If your tool implementation needs to know which tool call invoked it - for example to key state against the unique `tool_call_id` - add a parameter named `llm_tool_call` to your function. It will be passed the `llm.ToolCall` object for the current invocation, and is hidden from the schema the model sees. See {ref}`python-api-tools-llm-tool-call` for details.
|
||||
|
||||
<!-- Uncomment when this is true: The [llm-tools-datasette](https://github.com/simonw/llm-tools-datasette) plugin is a good example of this pattern in action. -->
|
||||
+1666
File diff suppressed because it is too large
Load Diff
+503
@@ -0,0 +1,503 @@
|
||||
from .hookspecs import hookimpl
|
||||
from .errors import (
|
||||
ModelError,
|
||||
NeedsKeyException,
|
||||
)
|
||||
from .models import (
|
||||
AsyncConversation,
|
||||
AsyncKeyModel,
|
||||
AsyncModel,
|
||||
AsyncResponse,
|
||||
Attachment,
|
||||
CancelToolCall,
|
||||
PauseChain,
|
||||
Conversation,
|
||||
EmbeddingModel,
|
||||
EmbeddingModelWithAliases,
|
||||
KeyModel,
|
||||
Model,
|
||||
ModelWithAliases,
|
||||
Options,
|
||||
Prompt,
|
||||
Response,
|
||||
Tool,
|
||||
Toolbox,
|
||||
ToolCall,
|
||||
ToolOutput,
|
||||
ToolResult,
|
||||
Usage,
|
||||
)
|
||||
from .parts import (
|
||||
Message,
|
||||
assistant,
|
||||
system,
|
||||
tool_message,
|
||||
user,
|
||||
)
|
||||
from .utils import schema_dsl, Fragment
|
||||
from .embeddings import Collection
|
||||
from .templates import Template
|
||||
from .plugins import pm, load_plugins
|
||||
import click
|
||||
from typing import Any, Dict, List, Optional, Callable, Type, Union
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import struct
|
||||
|
||||
__all__ = [
|
||||
"AsyncConversation",
|
||||
"AsyncKeyModel",
|
||||
"AsyncModel",
|
||||
"AsyncResponse",
|
||||
"assistant",
|
||||
"Attachment",
|
||||
"CancelToolCall",
|
||||
"Collection",
|
||||
"Conversation",
|
||||
"Fragment",
|
||||
"get_async_model",
|
||||
"get_key",
|
||||
"get_model",
|
||||
"hookimpl",
|
||||
"KeyModel",
|
||||
"Message",
|
||||
"Model",
|
||||
"ModelError",
|
||||
"NeedsKeyException",
|
||||
"Options",
|
||||
"PauseChain",
|
||||
"Prompt",
|
||||
"Response",
|
||||
"schema_dsl",
|
||||
"system",
|
||||
"Template",
|
||||
"Tool",
|
||||
"Toolbox",
|
||||
"ToolCall",
|
||||
"tool_message",
|
||||
"ToolOutput",
|
||||
"ToolResult",
|
||||
"Usage",
|
||||
"user",
|
||||
"user_dir",
|
||||
]
|
||||
DEFAULT_MODEL = "gpt-4o-mini"
|
||||
|
||||
|
||||
def get_plugins(all=False):
|
||||
plugins = []
|
||||
plugin_to_distinfo = dict(pm.list_plugin_distinfo())
|
||||
for plugin in pm.get_plugins():
|
||||
if not all and plugin.__name__.startswith("llm.default_plugins."):
|
||||
continue
|
||||
plugin_info = {
|
||||
"name": plugin.__name__,
|
||||
"hooks": [h.name for h in pm.get_hookcallers(plugin)],
|
||||
}
|
||||
distinfo = plugin_to_distinfo.get(plugin)
|
||||
if distinfo:
|
||||
plugin_info["version"] = distinfo.version
|
||||
plugin_info["name"] = (
|
||||
getattr(distinfo, "name", None) or distinfo.project_name
|
||||
)
|
||||
plugins.append(plugin_info)
|
||||
return plugins
|
||||
|
||||
|
||||
def get_models_with_aliases() -> List["ModelWithAliases"]:
|
||||
model_aliases = []
|
||||
|
||||
# Include aliases from aliases.json
|
||||
aliases_path = user_dir() / "aliases.json"
|
||||
extra_model_aliases: Dict[str, list] = {}
|
||||
if aliases_path.exists():
|
||||
configured_aliases = json.loads(aliases_path.read_text())
|
||||
for alias, model_id in configured_aliases.items():
|
||||
extra_model_aliases.setdefault(model_id, []).append(alias)
|
||||
|
||||
def register(model, async_model=None, aliases=None):
|
||||
alias_list = list(aliases or [])
|
||||
if model.model_id in extra_model_aliases:
|
||||
alias_list.extend(extra_model_aliases[model.model_id])
|
||||
model_aliases.append(ModelWithAliases(model, async_model, alias_list))
|
||||
|
||||
load_plugins()
|
||||
pm.hook.register_models(register=register, model_aliases=model_aliases)
|
||||
|
||||
return model_aliases
|
||||
|
||||
|
||||
def _get_loaders(hook_method) -> Dict[str, Callable]:
|
||||
load_plugins()
|
||||
loaders = {}
|
||||
|
||||
def register(prefix, loader):
|
||||
suffix = 0
|
||||
prefix_to_try = prefix
|
||||
while prefix_to_try in loaders:
|
||||
suffix += 1
|
||||
prefix_to_try = f"{prefix}_{suffix}"
|
||||
loaders[prefix_to_try] = loader
|
||||
|
||||
hook_method(register=register)
|
||||
return loaders
|
||||
|
||||
|
||||
def get_template_loaders() -> Dict[str, Callable[[str], Template]]:
|
||||
"""Get template loaders registered by plugins."""
|
||||
return _get_loaders(pm.hook.register_template_loaders)
|
||||
|
||||
|
||||
def get_fragment_loaders() -> Dict[
|
||||
str,
|
||||
Callable[[str], Union[Fragment, Attachment, List[Union[Fragment, Attachment]]]],
|
||||
]:
|
||||
"""Get fragment loaders registered by plugins."""
|
||||
return _get_loaders(pm.hook.register_fragment_loaders)
|
||||
|
||||
|
||||
def get_tools() -> Dict[str, Union[Tool, Type[Toolbox]]]:
|
||||
"""Return all tools (llm.Tool and llm.Toolbox) registered by plugins."""
|
||||
load_plugins()
|
||||
tools: Dict[str, Union[Tool, Type[Toolbox]]] = {}
|
||||
|
||||
# Variable to track current plugin name
|
||||
current_plugin_name = None
|
||||
|
||||
def register(
|
||||
tool_or_function: Union[Tool, Type[Toolbox], Callable[..., Any]],
|
||||
name: Optional[str] = None,
|
||||
) -> None:
|
||||
tool: Union[Tool, Type[Toolbox], None] = None
|
||||
|
||||
# If it's a Toolbox class, set the plugin field on it
|
||||
if inspect.isclass(tool_or_function):
|
||||
if issubclass(tool_or_function, Toolbox):
|
||||
tool = tool_or_function
|
||||
if current_plugin_name:
|
||||
tool.plugin = current_plugin_name
|
||||
tool.name = name or tool.__name__
|
||||
else:
|
||||
raise TypeError(
|
||||
"Toolbox classes must inherit from llm.Toolbox, {} does not.".format(
|
||||
tool_or_function.__name__
|
||||
)
|
||||
)
|
||||
|
||||
# If it's already a Tool instance, use it directly
|
||||
elif isinstance(tool_or_function, Tool):
|
||||
tool = tool_or_function
|
||||
if name:
|
||||
tool.name = name
|
||||
if current_plugin_name:
|
||||
tool.plugin = current_plugin_name
|
||||
|
||||
# If it's a bare function, wrap it in a Tool
|
||||
else:
|
||||
tool = Tool.function(tool_or_function, name=name)
|
||||
if current_plugin_name:
|
||||
tool.plugin = current_plugin_name
|
||||
|
||||
# Get the name for the tool/toolbox
|
||||
if tool:
|
||||
# For Toolbox classes, use their name attribute or class name
|
||||
if inspect.isclass(tool) and issubclass(tool, Toolbox):
|
||||
prefix = name or getattr(tool, "name", tool.__name__) or ""
|
||||
else:
|
||||
prefix = name or tool.name or ""
|
||||
|
||||
suffix = 0
|
||||
candidate = prefix
|
||||
|
||||
# Avoid name collisions
|
||||
while candidate in tools:
|
||||
suffix += 1
|
||||
candidate = f"{prefix}_{suffix}"
|
||||
|
||||
tools[candidate] = tool
|
||||
|
||||
# Call each plugin's register_tools hook individually to track current_plugin_name
|
||||
for plugin in pm.get_plugins():
|
||||
current_plugin_name = pm.get_name(plugin)
|
||||
hook_caller = pm.hook.register_tools
|
||||
plugin_impls = [
|
||||
impl for impl in hook_caller.get_hookimpls() if impl.plugin is plugin
|
||||
]
|
||||
for impl in plugin_impls:
|
||||
impl.function(register=register)
|
||||
|
||||
return tools
|
||||
|
||||
|
||||
def get_embedding_models_with_aliases() -> List["EmbeddingModelWithAliases"]:
|
||||
model_aliases = []
|
||||
|
||||
# Include aliases from aliases.json
|
||||
aliases_path = user_dir() / "aliases.json"
|
||||
extra_model_aliases: Dict[str, list] = {}
|
||||
if aliases_path.exists():
|
||||
configured_aliases = json.loads(aliases_path.read_text())
|
||||
for alias, model_id in configured_aliases.items():
|
||||
extra_model_aliases.setdefault(model_id, []).append(alias)
|
||||
|
||||
def register(model, aliases=None):
|
||||
alias_list = list(aliases or [])
|
||||
if model.model_id in extra_model_aliases:
|
||||
alias_list.extend(extra_model_aliases[model.model_id])
|
||||
model_aliases.append(EmbeddingModelWithAliases(model, alias_list))
|
||||
|
||||
load_plugins()
|
||||
pm.hook.register_embedding_models(register=register)
|
||||
|
||||
return model_aliases
|
||||
|
||||
|
||||
def get_embedding_models():
|
||||
models = []
|
||||
|
||||
def register(model, aliases=None):
|
||||
models.append(model)
|
||||
|
||||
load_plugins()
|
||||
pm.hook.register_embedding_models(register=register)
|
||||
return models
|
||||
|
||||
|
||||
def get_embedding_model(name):
|
||||
aliases = get_embedding_model_aliases()
|
||||
try:
|
||||
return aliases[name]
|
||||
except KeyError:
|
||||
raise UnknownModelError("Unknown model: " + str(name))
|
||||
|
||||
|
||||
def get_embedding_model_aliases() -> Dict[str, EmbeddingModel]:
|
||||
model_aliases = {}
|
||||
for model_with_aliases in get_embedding_models_with_aliases():
|
||||
for alias in model_with_aliases.aliases:
|
||||
model_aliases[alias] = model_with_aliases.model
|
||||
model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model
|
||||
return model_aliases
|
||||
|
||||
|
||||
def get_async_model_aliases() -> Dict[str, AsyncModel]:
|
||||
async_model_aliases = {}
|
||||
for model_with_aliases in get_models_with_aliases():
|
||||
if model_with_aliases.async_model:
|
||||
for alias in model_with_aliases.aliases:
|
||||
async_model_aliases[alias] = model_with_aliases.async_model
|
||||
async_model_aliases[model_with_aliases.model.model_id] = (
|
||||
model_with_aliases.async_model
|
||||
)
|
||||
return async_model_aliases
|
||||
|
||||
|
||||
def get_model_aliases() -> Dict[str, Model]:
|
||||
model_aliases = {}
|
||||
for model_with_aliases in get_models_with_aliases():
|
||||
if model_with_aliases.model:
|
||||
for alias in model_with_aliases.aliases:
|
||||
model_aliases[alias] = model_with_aliases.model
|
||||
model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model
|
||||
return model_aliases
|
||||
|
||||
|
||||
class UnknownModelError(KeyError):
|
||||
pass
|
||||
|
||||
|
||||
def get_models() -> List[Model]:
|
||||
"Get all registered models"
|
||||
models_with_aliases = get_models_with_aliases()
|
||||
return [mwa.model for mwa in models_with_aliases if mwa.model]
|
||||
|
||||
|
||||
def get_async_models() -> List[AsyncModel]:
|
||||
"Get all registered async models"
|
||||
models_with_aliases = get_models_with_aliases()
|
||||
return [mwa.async_model for mwa in models_with_aliases if mwa.async_model]
|
||||
|
||||
|
||||
def get_async_model(name: Optional[str] = None) -> AsyncModel:
|
||||
"Get an async model by name or alias"
|
||||
aliases = get_async_model_aliases()
|
||||
name = name or get_default_model()
|
||||
try:
|
||||
return aliases[name]
|
||||
except KeyError:
|
||||
# Does a sync model exist?
|
||||
sync_model = None
|
||||
try:
|
||||
sync_model = get_model(name, _skip_async=True)
|
||||
except UnknownModelError:
|
||||
pass
|
||||
if sync_model:
|
||||
raise UnknownModelError("Unknown async model (sync model exists): " + name)
|
||||
else:
|
||||
raise UnknownModelError("Unknown model: " + name)
|
||||
|
||||
|
||||
def get_model(name: Optional[str] = None, _skip_async: bool = False) -> Model:
|
||||
"Get a model by name or alias"
|
||||
aliases = get_model_aliases()
|
||||
name = name or get_default_model()
|
||||
try:
|
||||
return aliases[name]
|
||||
except KeyError:
|
||||
# Does an async model exist?
|
||||
if _skip_async:
|
||||
raise UnknownModelError("Unknown model: " + name)
|
||||
async_model = None
|
||||
try:
|
||||
async_model = get_async_model(name)
|
||||
except UnknownModelError:
|
||||
pass
|
||||
if async_model:
|
||||
raise UnknownModelError("Unknown model (async model exists): " + name)
|
||||
else:
|
||||
raise UnknownModelError("Unknown model: " + name)
|
||||
|
||||
|
||||
def get_key(
|
||||
explicit_key: Optional[str] = None,
|
||||
key_alias: Optional[str] = None,
|
||||
env_var: Optional[str] = None,
|
||||
*,
|
||||
alias: Optional[str] = None,
|
||||
env: Optional[str] = None,
|
||||
input: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Return an API key based on a hierarchy of potential sources. You should use the keyword arguments,
|
||||
the positional arguments are here purely for backwards-compatibility with older code.
|
||||
|
||||
:param input: Input provided by the user. This may be the key, or an alias of a key in keys.json.
|
||||
:param alias: The alias used to retrieve the key from the keys.json file.
|
||||
:param env: Name of the environment variable to check for the key as a final fallback.
|
||||
"""
|
||||
if alias:
|
||||
key_alias = alias
|
||||
if env:
|
||||
env_var = env
|
||||
if input:
|
||||
explicit_key = input
|
||||
stored_keys = load_keys()
|
||||
# If user specified an alias, use the key stored for that alias
|
||||
if explicit_key in stored_keys:
|
||||
return stored_keys[explicit_key]
|
||||
if explicit_key:
|
||||
# User specified a key that's not an alias, use that
|
||||
return explicit_key
|
||||
# Stored key over-rides environment variables over-ride the default key
|
||||
if key_alias in stored_keys:
|
||||
return stored_keys[key_alias]
|
||||
# Finally try environment variable
|
||||
if env_var and os.environ.get(env_var):
|
||||
return os.environ[env_var]
|
||||
# Couldn't find it
|
||||
return None
|
||||
|
||||
|
||||
def load_keys():
|
||||
path = user_dir() / "keys.json"
|
||||
if path.exists():
|
||||
return json.loads(path.read_text())
|
||||
else:
|
||||
return {}
|
||||
|
||||
|
||||
def user_dir():
|
||||
llm_user_path = os.environ.get("LLM_USER_PATH")
|
||||
if llm_user_path:
|
||||
path = pathlib.Path(llm_user_path)
|
||||
else:
|
||||
path = pathlib.Path(click.get_app_dir("io.datasette.llm"))
|
||||
path.mkdir(exist_ok=True, parents=True)
|
||||
return path
|
||||
|
||||
|
||||
def set_alias(alias, model_id_or_alias):
|
||||
"""
|
||||
Set an alias to point to the specified model.
|
||||
"""
|
||||
path = user_dir() / "aliases.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not path.exists():
|
||||
path.write_text("{}\n")
|
||||
try:
|
||||
current = json.loads(path.read_text())
|
||||
except json.decoder.JSONDecodeError:
|
||||
# We're going to write a valid JSON file in a moment:
|
||||
current = {}
|
||||
# Resolve model_id_or_alias to a model_id
|
||||
try:
|
||||
model = get_model(model_id_or_alias)
|
||||
model_id = model.model_id
|
||||
except UnknownModelError:
|
||||
# Try to resolve it to an embedding model
|
||||
try:
|
||||
model = get_embedding_model(model_id_or_alias)
|
||||
model_id = model.model_id
|
||||
except UnknownModelError:
|
||||
# Set the alias to the exact string they provided instead
|
||||
model_id = model_id_or_alias
|
||||
current[alias] = model_id
|
||||
path.write_text(json.dumps(current, indent=4) + "\n")
|
||||
|
||||
|
||||
def remove_alias(alias):
|
||||
"""
|
||||
Remove an alias.
|
||||
"""
|
||||
path = user_dir() / "aliases.json"
|
||||
if not path.exists():
|
||||
raise KeyError("No aliases.json file exists")
|
||||
try:
|
||||
current = json.loads(path.read_text())
|
||||
except json.decoder.JSONDecodeError:
|
||||
raise KeyError("aliases.json file is not valid JSON")
|
||||
if alias not in current:
|
||||
raise KeyError("No such alias: {}".format(alias))
|
||||
del current[alias]
|
||||
path.write_text(json.dumps(current, indent=4) + "\n")
|
||||
|
||||
|
||||
def encode(values):
|
||||
return struct.pack("<" + "f" * len(values), *values)
|
||||
|
||||
|
||||
def decode(binary):
|
||||
return struct.unpack("<" + "f" * (len(binary) // 4), binary)
|
||||
|
||||
|
||||
def cosine_similarity(a, b):
|
||||
dot_product = sum(x * y for x, y in zip(a, b))
|
||||
magnitude_a = sum(x * x for x in a) ** 0.5
|
||||
magnitude_b = sum(x * x for x in b) ** 0.5
|
||||
return dot_product / (magnitude_a * magnitude_b)
|
||||
|
||||
|
||||
def get_default_model(filename="default_model.txt", default=DEFAULT_MODEL):
|
||||
path = user_dir() / filename
|
||||
if path.exists():
|
||||
return path.read_text().strip()
|
||||
else:
|
||||
return default
|
||||
|
||||
|
||||
def set_default_model(model, filename="default_model.txt"):
|
||||
path = user_dir() / filename
|
||||
if model is None and path.exists():
|
||||
path.unlink()
|
||||
else:
|
||||
path.write_text(model)
|
||||
|
||||
|
||||
def get_default_embedding_model():
|
||||
return get_default_model("default_embedding_model.txt", None)
|
||||
|
||||
|
||||
def set_default_embedding_model(model):
|
||||
set_default_model(model, "default_embedding_model.txt")
|
||||
@@ -0,0 +1,4 @@
|
||||
from .cli import cli
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
+4214
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
import llm
|
||||
from llm.tools import llm_time, llm_version
|
||||
|
||||
|
||||
@llm.hookimpl
|
||||
def register_tools(register):
|
||||
register(llm_version)
|
||||
register(llm_time)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,367 @@
|
||||
from .models import EmbeddingModel
|
||||
from .embeddings_migrations import embeddings_migrations
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
from itertools import islice
|
||||
import json
|
||||
from sqlite_utils import Database
|
||||
from sqlite_utils.db import Table
|
||||
import time
|
||||
from typing import cast, Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
|
||||
@dataclass
|
||||
class Entry:
|
||||
id: str
|
||||
score: Optional[float]
|
||||
content: Optional[str] = None
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class Collection:
|
||||
class DoesNotExist(Exception):
|
||||
pass
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
db: Optional[Database] = None,
|
||||
*,
|
||||
model: Optional[EmbeddingModel] = None,
|
||||
model_id: Optional[str] = None,
|
||||
create: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
A collection of embeddings
|
||||
|
||||
Returns the collection with the given name, creating it if it does not exist.
|
||||
|
||||
If you set create=False a Collection.DoesNotExist exception will be raised if the
|
||||
collection does not already exist.
|
||||
|
||||
Args:
|
||||
db (sqlite_utils.Database): Database to store the collection in
|
||||
name (str): Name of the collection
|
||||
model (llm.models.EmbeddingModel, optional): Embedding model to use
|
||||
model_id (str, optional): Alternatively, ID of the embedding model to use
|
||||
create (bool, optional): Whether to create the collection if it does not exist
|
||||
"""
|
||||
import llm
|
||||
|
||||
self.db = db or Database(memory=True)
|
||||
self.name = name
|
||||
self._model = model
|
||||
|
||||
embeddings_migrations.apply(self.db)
|
||||
|
||||
rows = list(self.db["collections"].rows_where("name = ?", [self.name]))
|
||||
if rows:
|
||||
row = rows[0]
|
||||
self.id = row["id"]
|
||||
self.model_id = row["model"]
|
||||
else:
|
||||
if create:
|
||||
# Collection does not exist, so model or model_id is required
|
||||
if not model and not model_id:
|
||||
raise ValueError(
|
||||
"Either model= or model_id= must be provided when creating a new collection"
|
||||
)
|
||||
# Create it
|
||||
if model_id:
|
||||
# Resolve alias
|
||||
model = llm.get_embedding_model(model_id)
|
||||
self._model = model
|
||||
model_id = cast(EmbeddingModel, model).model_id
|
||||
self.id = (
|
||||
cast(Table, self.db["collections"])
|
||||
.insert(
|
||||
{
|
||||
"name": self.name,
|
||||
"model": model_id,
|
||||
}
|
||||
)
|
||||
.last_pk
|
||||
)
|
||||
else:
|
||||
raise self.DoesNotExist(f"Collection '{name}' does not exist")
|
||||
|
||||
def model(self) -> EmbeddingModel:
|
||||
"Return the embedding model used by this collection"
|
||||
import llm
|
||||
|
||||
if self._model is None:
|
||||
self._model = llm.get_embedding_model(self.model_id)
|
||||
|
||||
return cast(EmbeddingModel, self._model)
|
||||
|
||||
def count(self) -> int:
|
||||
"""
|
||||
Count the number of items in the collection.
|
||||
|
||||
Returns:
|
||||
int: Number of items in the collection
|
||||
"""
|
||||
return next(
|
||||
self.db.query(
|
||||
"""
|
||||
select count(*) as c from embeddings where collection_id = (
|
||||
select id from collections where name = ?
|
||||
)
|
||||
""",
|
||||
(self.name,),
|
||||
)
|
||||
)["c"]
|
||||
|
||||
def embed(
|
||||
self,
|
||||
id: str,
|
||||
value: Union[str, bytes],
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
store: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Embed value and store it in the collection with a given ID.
|
||||
|
||||
Args:
|
||||
id (str): ID for the value
|
||||
value (str or bytes): value to be embedded
|
||||
metadata (dict, optional): Metadata to be stored
|
||||
store (bool, optional): Whether to store the value in the content or content_blob column
|
||||
"""
|
||||
from llm import encode
|
||||
|
||||
content_hash = self.content_hash(value)
|
||||
if self.db["embeddings"].count_where(
|
||||
"content_hash = ? and collection_id = ?", [content_hash, self.id]
|
||||
):
|
||||
return
|
||||
embedding = self.model().embed(value)
|
||||
cast(Table, self.db["embeddings"]).insert(
|
||||
{
|
||||
"collection_id": self.id,
|
||||
"id": id,
|
||||
"embedding": encode(embedding),
|
||||
"content": value if (store and isinstance(value, str)) else None,
|
||||
"content_blob": value if (store and isinstance(value, bytes)) else None,
|
||||
"content_hash": content_hash,
|
||||
"metadata": json.dumps(metadata) if metadata else None,
|
||||
"updated": int(time.time()),
|
||||
},
|
||||
replace=True,
|
||||
)
|
||||
|
||||
def embed_multi(
|
||||
self,
|
||||
entries: Iterable[Tuple[str, Union[str, bytes]]],
|
||||
store: bool = False,
|
||||
batch_size: int = 100,
|
||||
) -> None:
|
||||
"""
|
||||
Embed multiple texts and store them in the collection with given IDs.
|
||||
|
||||
Args:
|
||||
entries (iterable): Iterable of (id: str, text: str) tuples
|
||||
store (bool, optional): Whether to store the text in the content column
|
||||
batch_size (int, optional): custom maximum batch size to use
|
||||
"""
|
||||
self.embed_multi_with_metadata(
|
||||
((id, value, None) for id, value in entries),
|
||||
store=store,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
def embed_multi_with_metadata(
|
||||
self,
|
||||
entries: Iterable[Tuple[str, Union[str, bytes], Optional[Dict[str, Any]]]],
|
||||
store: bool = False,
|
||||
batch_size: int = 100,
|
||||
) -> None:
|
||||
"""
|
||||
Embed multiple values along with metadata and store them in the collection with given IDs.
|
||||
|
||||
Args:
|
||||
entries (iterable): Iterable of (id: str, value: str or bytes, metadata: None or dict)
|
||||
store (bool, optional): Whether to store the value in the content or content_blob column
|
||||
batch_size (int, optional): custom maximum batch size to use
|
||||
"""
|
||||
import llm
|
||||
|
||||
batch_size = min(batch_size, (self.model().batch_size or batch_size))
|
||||
iterator = iter(entries)
|
||||
collection_id = self.id
|
||||
while True:
|
||||
batch = list(islice(iterator, batch_size))
|
||||
if not batch:
|
||||
break
|
||||
# Calculate hashes first
|
||||
items_and_hashes = [(item, self.content_hash(item[1])) for item in batch]
|
||||
# Any of those hashes already exist?
|
||||
existing_ids = [
|
||||
row["id"]
|
||||
for row in self.db.query(
|
||||
"""
|
||||
select id from embeddings
|
||||
where collection_id = ? and content_hash in ({})
|
||||
""".format(",".join("?" for _ in items_and_hashes)),
|
||||
[collection_id]
|
||||
+ [item_and_hash[1] for item_and_hash in items_and_hashes],
|
||||
)
|
||||
]
|
||||
filtered_batch = [item for item in batch if item[0] not in existing_ids]
|
||||
embeddings = list(
|
||||
self.model().embed_multi(item[1] for item in filtered_batch)
|
||||
)
|
||||
with self.db.conn:
|
||||
cast(Table, self.db["embeddings"]).insert_all(
|
||||
(
|
||||
{
|
||||
"collection_id": collection_id,
|
||||
"id": id,
|
||||
"embedding": llm.encode(embedding),
|
||||
"content": (
|
||||
value if (store and isinstance(value, str)) else None
|
||||
),
|
||||
"content_blob": (
|
||||
value if (store and isinstance(value, bytes)) else None
|
||||
),
|
||||
"content_hash": self.content_hash(value),
|
||||
"metadata": json.dumps(metadata) if metadata else None,
|
||||
"updated": int(time.time()),
|
||||
}
|
||||
for (embedding, (id, value, metadata)) in zip(
|
||||
embeddings, filtered_batch
|
||||
)
|
||||
),
|
||||
replace=True,
|
||||
)
|
||||
|
||||
def similar_by_vector(
|
||||
self,
|
||||
vector: List[float],
|
||||
number: int = 10,
|
||||
skip_id: Optional[str] = None,
|
||||
prefix: Optional[str] = None,
|
||||
) -> List[Entry]:
|
||||
"""
|
||||
Find similar items in the collection by a given vector.
|
||||
|
||||
Args:
|
||||
vector (list): Vector to search by
|
||||
number (int, optional): Number of similar items to return
|
||||
skip_id (str, optional): An ID to exclude from the results
|
||||
prefix: (str, optional): Filter results to IDs witih this prefix
|
||||
|
||||
Returns:
|
||||
list: List of Entry objects
|
||||
"""
|
||||
import llm
|
||||
|
||||
def distance_score(other_encoded):
|
||||
other_vector = llm.decode(other_encoded)
|
||||
return llm.cosine_similarity(other_vector, vector)
|
||||
|
||||
self.db.register_function(distance_score, replace=True)
|
||||
|
||||
where_bits = ["collection_id = ?"]
|
||||
where_args = [str(self.id)]
|
||||
|
||||
if prefix:
|
||||
where_bits.append("id LIKE ? || '%'")
|
||||
where_args.append(prefix)
|
||||
|
||||
if skip_id:
|
||||
where_bits.append("id != ?")
|
||||
where_args.append(skip_id)
|
||||
|
||||
return [
|
||||
Entry(
|
||||
id=row["id"],
|
||||
score=row["score"],
|
||||
content=row["content"],
|
||||
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
|
||||
)
|
||||
for row in self.db.query(
|
||||
"""
|
||||
select id, content, metadata, distance_score(embedding) as score
|
||||
from embeddings
|
||||
where {where}
|
||||
order by score desc limit {number}
|
||||
""".format(
|
||||
where=" and ".join(where_bits),
|
||||
number=number,
|
||||
),
|
||||
where_args,
|
||||
)
|
||||
]
|
||||
|
||||
def similar_by_id(
|
||||
self, id: str, number: int = 10, prefix: Optional[str] = None
|
||||
) -> List[Entry]:
|
||||
"""
|
||||
Find similar items in the collection by a given ID.
|
||||
|
||||
Args:
|
||||
id (str): ID to search by
|
||||
number (int, optional): Number of similar items to return
|
||||
prefix: (str, optional): Filter results to IDs with this prefix
|
||||
|
||||
Returns:
|
||||
list: List of Entry objects
|
||||
"""
|
||||
import llm
|
||||
|
||||
matches = list(
|
||||
self.db["embeddings"].rows_where(
|
||||
"collection_id = ? and id = ?", (self.id, id)
|
||||
)
|
||||
)
|
||||
if not matches:
|
||||
raise self.DoesNotExist("ID not found")
|
||||
embedding = matches[0]["embedding"]
|
||||
comparison_vector = llm.decode(embedding)
|
||||
return self.similar_by_vector(
|
||||
comparison_vector, number, skip_id=id, prefix=prefix
|
||||
)
|
||||
|
||||
def similar(
|
||||
self, value: Union[str, bytes], number: int = 10, prefix: Optional[str] = None
|
||||
) -> List[Entry]:
|
||||
"""
|
||||
Find similar items in the collection by a given value.
|
||||
|
||||
Args:
|
||||
value (str or bytes): value to search by
|
||||
number (int, optional): Number of similar items to return
|
||||
prefix: (str, optional): Filter results to IDs with this prefix
|
||||
|
||||
Returns:
|
||||
list: List of Entry objects
|
||||
"""
|
||||
comparison_vector = self.model().embed(value)
|
||||
return self.similar_by_vector(comparison_vector, number, prefix=prefix)
|
||||
|
||||
@classmethod
|
||||
def exists(cls, db: Database, name: str) -> bool:
|
||||
"""
|
||||
Does this collection exist in the database?
|
||||
|
||||
Args:
|
||||
name (str): Name of the collection
|
||||
"""
|
||||
rows = list(db["collections"].rows_where("name = ?", [name]))
|
||||
return bool(rows)
|
||||
|
||||
def delete(self):
|
||||
"""
|
||||
Delete the collection and its embeddings from the database
|
||||
"""
|
||||
with self.db.conn:
|
||||
self.db.execute("delete from embeddings where collection_id = ?", [self.id])
|
||||
self.db.execute("delete from collections where id = ?", [self.id])
|
||||
|
||||
@staticmethod
|
||||
def content_hash(input: Union[str, bytes]) -> bytes:
|
||||
"Hash content for deduplication. Override to change hashing behavior."
|
||||
if isinstance(input, str):
|
||||
input = input.encode("utf8")
|
||||
return hashlib.md5(input).digest()
|
||||
@@ -0,0 +1,89 @@
|
||||
from sqlite_migrate import Migrations
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
embeddings_migrations = Migrations("llm.embeddings")
|
||||
|
||||
|
||||
@embeddings_migrations()
|
||||
def m001_create_tables(db):
|
||||
db["collections"].create({"id": int, "name": str, "model": str}, pk="id")
|
||||
db["collections"].create_index(["name"], unique=True)
|
||||
db["embeddings"].create(
|
||||
{
|
||||
"collection_id": int,
|
||||
"id": str,
|
||||
"embedding": bytes,
|
||||
"content": str,
|
||||
"metadata": str,
|
||||
},
|
||||
pk=("collection_id", "id"),
|
||||
)
|
||||
|
||||
|
||||
@embeddings_migrations()
|
||||
def m002_foreign_key(db):
|
||||
db["embeddings"].add_foreign_key("collection_id", "collections", "id")
|
||||
|
||||
|
||||
@embeddings_migrations()
|
||||
def m003_add_updated(db):
|
||||
db["embeddings"].add_column("updated", int)
|
||||
# Pretty-print the schema
|
||||
db["embeddings"].transform()
|
||||
# Assume anything existing was last updated right now
|
||||
db.execute(
|
||||
"update embeddings set updated = ? where updated is null", [int(time.time())]
|
||||
)
|
||||
|
||||
|
||||
@embeddings_migrations()
|
||||
def m004_store_content_hash(db):
|
||||
db["embeddings"].add_column("content_hash", bytes)
|
||||
db["embeddings"].transform(
|
||||
column_order=(
|
||||
"collection_id",
|
||||
"id",
|
||||
"embedding",
|
||||
"content",
|
||||
"content_hash",
|
||||
"metadata",
|
||||
"updated",
|
||||
)
|
||||
)
|
||||
|
||||
# Register functions manually so we can de-register later
|
||||
def md5(text):
|
||||
return hashlib.md5(text.encode("utf8")).digest()
|
||||
|
||||
def random_md5():
|
||||
return hashlib.md5(str(time.time()).encode("utf8")).digest()
|
||||
|
||||
db.conn.create_function("temp_md5", 1, md5)
|
||||
db.conn.create_function("temp_random_md5", 0, random_md5)
|
||||
|
||||
with db.conn:
|
||||
db.execute("""
|
||||
update embeddings
|
||||
set content_hash = temp_md5(content)
|
||||
where content is not null
|
||||
""")
|
||||
db.execute("""
|
||||
update embeddings
|
||||
set content_hash = temp_random_md5()
|
||||
where content is null
|
||||
""")
|
||||
|
||||
db["embeddings"].create_index(["content_hash"])
|
||||
|
||||
# De-register functions
|
||||
db.conn.create_function("temp_md5", 1, None)
|
||||
db.conn.create_function("temp_random_md5", 0, None)
|
||||
|
||||
|
||||
@embeddings_migrations()
|
||||
def m005_add_content_blob(db):
|
||||
db["embeddings"].add_column("content_blob", bytes)
|
||||
db["embeddings"].transform(
|
||||
column_order=("collection_id", "id", "embedding", "content", "content_blob")
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
class ModelError(Exception):
|
||||
"Models can raise this error, which will be displayed to the user"
|
||||
|
||||
|
||||
class NeedsKeyException(ModelError):
|
||||
"Model needs an API key which has not been provided"
|
||||
@@ -0,0 +1,35 @@
|
||||
from pluggy import HookimplMarker
|
||||
from pluggy import HookspecMarker
|
||||
|
||||
hookspec = HookspecMarker("llm")
|
||||
hookimpl = HookimplMarker("llm")
|
||||
|
||||
|
||||
@hookspec
|
||||
def register_commands(cli):
|
||||
"""Register additional CLI commands, e.g. 'llm mycommand ...'"""
|
||||
|
||||
|
||||
@hookspec
|
||||
def register_models(register, model_aliases):
|
||||
"Register additional model instances representing LLM models that can be called"
|
||||
|
||||
|
||||
@hookspec
|
||||
def register_embedding_models(register):
|
||||
"Register additional model instances that can be used for embedding"
|
||||
|
||||
|
||||
@hookspec
|
||||
def register_template_loaders(register):
|
||||
"Register additional template loaders with prefixes"
|
||||
|
||||
|
||||
@hookspec
|
||||
def register_fragment_loaders(register):
|
||||
"Register additional fragment loaders with prefixes"
|
||||
|
||||
|
||||
@hookspec
|
||||
def register_tools(register):
|
||||
"Register functions that can be used as tools by the LLMs"
|
||||
@@ -0,0 +1,428 @@
|
||||
import datetime
|
||||
from typing import Callable, List
|
||||
|
||||
MIGRATIONS: List[Callable] = []
|
||||
migration = MIGRATIONS.append
|
||||
|
||||
|
||||
def migrate(db):
|
||||
ensure_migrations_table(db)
|
||||
already_applied = {r["name"] for r in db["_llm_migrations"].rows}
|
||||
for fn in MIGRATIONS:
|
||||
name = fn.__name__
|
||||
if name not in already_applied:
|
||||
fn(db)
|
||||
db["_llm_migrations"].insert(
|
||||
{
|
||||
"name": name,
|
||||
"applied_at": str(datetime.datetime.now(datetime.timezone.utc)),
|
||||
}
|
||||
)
|
||||
already_applied.add(name)
|
||||
|
||||
|
||||
def ensure_migrations_table(db):
|
||||
if not db["_llm_migrations"].exists():
|
||||
db["_llm_migrations"].create(
|
||||
{
|
||||
"name": str,
|
||||
"applied_at": str,
|
||||
},
|
||||
pk="name",
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m001_initial(db):
|
||||
# Ensure the original table design exists, so other migrations can run
|
||||
if db["log"].exists():
|
||||
# It needs to have the chat_id column
|
||||
if "chat_id" not in db["log"].columns_dict:
|
||||
db["log"].add_column("chat_id")
|
||||
return
|
||||
db["log"].create(
|
||||
{
|
||||
"provider": str,
|
||||
"system": str,
|
||||
"prompt": str,
|
||||
"chat_id": str,
|
||||
"response": str,
|
||||
"model": str,
|
||||
"timestamp": str,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m002_id_primary_key(db):
|
||||
db["log"].transform(pk="id")
|
||||
|
||||
|
||||
@migration
|
||||
def m003_chat_id_foreign_key(db):
|
||||
db["log"].transform(types={"chat_id": int})
|
||||
db["log"].add_foreign_key("chat_id", "log", "id")
|
||||
|
||||
|
||||
@migration
|
||||
def m004_column_order(db):
|
||||
db["log"].transform(
|
||||
column_order=(
|
||||
"id",
|
||||
"model",
|
||||
"timestamp",
|
||||
"prompt",
|
||||
"system",
|
||||
"response",
|
||||
"chat_id",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m004_drop_provider(db):
|
||||
db["log"].transform(drop=("provider",))
|
||||
|
||||
|
||||
@migration
|
||||
def m005_debug(db):
|
||||
db["log"].add_column("debug", str)
|
||||
db["log"].add_column("duration_ms", int)
|
||||
|
||||
|
||||
@migration
|
||||
def m006_new_logs_table(db):
|
||||
columns = db["log"].columns_dict
|
||||
for column, type in (
|
||||
("options_json", str),
|
||||
("prompt_json", str),
|
||||
("response_json", str),
|
||||
("reply_to_id", int),
|
||||
):
|
||||
# It's possible people running development code like myself
|
||||
# might have accidentally created these columns already
|
||||
if column not in columns:
|
||||
db["log"].add_column(column, type)
|
||||
|
||||
# Use .transform() to rename options and timestamp_utc, and set new order
|
||||
db["log"].transform(
|
||||
column_order=(
|
||||
"id",
|
||||
"model",
|
||||
"prompt",
|
||||
"system",
|
||||
"prompt_json",
|
||||
"options_json",
|
||||
"response",
|
||||
"response_json",
|
||||
"reply_to_id",
|
||||
"chat_id",
|
||||
"duration_ms",
|
||||
"timestamp_utc",
|
||||
),
|
||||
rename={
|
||||
"timestamp": "timestamp_utc",
|
||||
"options": "options_json",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m007_finish_logs_table(db):
|
||||
db["log"].transform(
|
||||
drop={"debug"},
|
||||
rename={"timestamp_utc": "datetime_utc"},
|
||||
drop_foreign_keys=("chat_id",),
|
||||
)
|
||||
with db.conn:
|
||||
db.execute("alter table log rename to logs")
|
||||
|
||||
|
||||
@migration
|
||||
def m008_reply_to_id_foreign_key(db):
|
||||
db["logs"].add_foreign_key("reply_to_id", "logs", "id")
|
||||
|
||||
|
||||
@migration
|
||||
def m008_fix_column_order_in_logs(db):
|
||||
# reply_to_id ended up at the end after foreign key added
|
||||
db["logs"].transform(
|
||||
column_order=(
|
||||
"id",
|
||||
"model",
|
||||
"prompt",
|
||||
"system",
|
||||
"prompt_json",
|
||||
"options_json",
|
||||
"response",
|
||||
"response_json",
|
||||
"reply_to_id",
|
||||
"chat_id",
|
||||
"duration_ms",
|
||||
"timestamp_utc",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m009_delete_logs_table_if_empty(db):
|
||||
# We moved to a new table design, but we don't delete the table
|
||||
# if someone has put data in it
|
||||
if not db["logs"].count:
|
||||
db["logs"].drop()
|
||||
|
||||
|
||||
@migration
|
||||
def m010_create_new_log_tables(db):
|
||||
db["conversations"].create(
|
||||
{
|
||||
"id": str,
|
||||
"name": str,
|
||||
"model": str,
|
||||
},
|
||||
pk="id",
|
||||
)
|
||||
db["responses"].create(
|
||||
{
|
||||
"id": str,
|
||||
"model": str,
|
||||
"prompt": str,
|
||||
"system": str,
|
||||
"prompt_json": str,
|
||||
"options_json": str,
|
||||
"response": str,
|
||||
"response_json": str,
|
||||
"conversation_id": str,
|
||||
"duration_ms": int,
|
||||
"datetime_utc": str,
|
||||
},
|
||||
pk="id",
|
||||
foreign_keys=(("conversation_id", "conversations", "id"),),
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m011_fts_for_responses(db):
|
||||
db["responses"].enable_fts(["prompt", "response"], create_triggers=True)
|
||||
|
||||
|
||||
@migration
|
||||
def m012_attachments_tables(db):
|
||||
db["attachments"].create(
|
||||
{
|
||||
"id": str,
|
||||
"type": str,
|
||||
"path": str,
|
||||
"url": str,
|
||||
"content": bytes,
|
||||
},
|
||||
pk="id",
|
||||
)
|
||||
db["prompt_attachments"].create(
|
||||
{
|
||||
"response_id": str,
|
||||
"attachment_id": str,
|
||||
"order": int,
|
||||
},
|
||||
foreign_keys=(
|
||||
("response_id", "responses", "id"),
|
||||
("attachment_id", "attachments", "id"),
|
||||
),
|
||||
pk=("response_id", "attachment_id"),
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m013_usage(db):
|
||||
db["responses"].add_column("input_tokens", int)
|
||||
db["responses"].add_column("output_tokens", int)
|
||||
db["responses"].add_column("token_details", str)
|
||||
|
||||
|
||||
@migration
|
||||
def m014_schemas(db):
|
||||
db["schemas"].create(
|
||||
{
|
||||
"id": str,
|
||||
"content": str,
|
||||
},
|
||||
pk="id",
|
||||
)
|
||||
db["responses"].add_column("schema_id", str, fk="schemas", fk_col="id")
|
||||
# Clean up SQL create table indentation
|
||||
db["responses"].transform()
|
||||
# These changes may have dropped the FTS configuration, fix that
|
||||
db["responses"].enable_fts(
|
||||
["prompt", "response"], create_triggers=True, replace=True
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m015_fragments_tables(db):
|
||||
db["fragments"].create(
|
||||
{
|
||||
"id": int,
|
||||
"hash": str,
|
||||
"content": str,
|
||||
"datetime_utc": str,
|
||||
"source": str,
|
||||
},
|
||||
pk="id",
|
||||
)
|
||||
db["fragments"].create_index(["hash"], unique=True)
|
||||
db["fragment_aliases"].create(
|
||||
{
|
||||
"alias": str,
|
||||
"fragment_id": int,
|
||||
},
|
||||
foreign_keys=(("fragment_id", "fragments", "id"),),
|
||||
pk="alias",
|
||||
)
|
||||
db["prompt_fragments"].create(
|
||||
{
|
||||
"response_id": str,
|
||||
"fragment_id": int,
|
||||
"order": int,
|
||||
},
|
||||
foreign_keys=(
|
||||
("response_id", "responses", "id"),
|
||||
("fragment_id", "fragments", "id"),
|
||||
),
|
||||
pk=("response_id", "fragment_id"),
|
||||
)
|
||||
db["system_fragments"].create(
|
||||
{
|
||||
"response_id": str,
|
||||
"fragment_id": int,
|
||||
"order": int,
|
||||
},
|
||||
foreign_keys=(
|
||||
("response_id", "responses", "id"),
|
||||
("fragment_id", "fragments", "id"),
|
||||
),
|
||||
pk=("response_id", "fragment_id"),
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m016_fragments_table_pks(db):
|
||||
# The same fragment can be attached to a response multiple times
|
||||
# https://github.com/simonw/llm/issues/863#issuecomment-2781720064
|
||||
db["prompt_fragments"].transform(pk=("response_id", "fragment_id", "order"))
|
||||
db["system_fragments"].transform(pk=("response_id", "fragment_id", "order"))
|
||||
|
||||
|
||||
@migration
|
||||
def m017_tools_tables(db):
|
||||
db["tools"].create(
|
||||
{
|
||||
"id": int,
|
||||
"hash": str,
|
||||
"name": str,
|
||||
"description": str,
|
||||
"input_schema": str,
|
||||
},
|
||||
pk="id",
|
||||
)
|
||||
db["tools"].create_index(["hash"], unique=True)
|
||||
# Many-to-many relationship between tools and responses
|
||||
db["tool_responses"].create(
|
||||
{
|
||||
"tool_id": int,
|
||||
"response_id": str,
|
||||
},
|
||||
foreign_keys=(
|
||||
("tool_id", "tools", "id"),
|
||||
("response_id", "responses", "id"),
|
||||
),
|
||||
pk=("tool_id", "response_id"),
|
||||
)
|
||||
# tool_calls and tool_results are one-to-many against responses
|
||||
db["tool_calls"].create(
|
||||
{
|
||||
"id": int,
|
||||
"response_id": str,
|
||||
"tool_id": int,
|
||||
"name": str,
|
||||
"arguments": str,
|
||||
"tool_call_id": str,
|
||||
},
|
||||
pk="id",
|
||||
foreign_keys=(
|
||||
("response_id", "responses", "id"),
|
||||
("tool_id", "tools", "id"),
|
||||
),
|
||||
)
|
||||
db["tool_results"].create(
|
||||
{
|
||||
"id": int,
|
||||
"response_id": str,
|
||||
"tool_id": int,
|
||||
"name": str,
|
||||
"output": str,
|
||||
"tool_call_id": str,
|
||||
},
|
||||
pk="id",
|
||||
foreign_keys=(
|
||||
("response_id", "responses", "id"),
|
||||
("tool_id", "tools", "id"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m017_tools_plugin(db):
|
||||
db["tools"].add_column("plugin")
|
||||
|
||||
|
||||
@migration
|
||||
def m018_tool_instances(db):
|
||||
# Used to track instances of Toolbox classes that may be
|
||||
# used multiple times by different tools
|
||||
db["tool_instances"].create(
|
||||
{
|
||||
"id": int,
|
||||
"plugin": str,
|
||||
"name": str,
|
||||
"arguments": str,
|
||||
},
|
||||
pk="id",
|
||||
)
|
||||
# We record which instance was used only on the results
|
||||
db["tool_results"].add_column("instance_id", fk="tool_instances")
|
||||
|
||||
|
||||
@migration
|
||||
def m019_resolved_model(db):
|
||||
# For models like gemini-1.5-flash-latest where we wish to record
|
||||
# the resolved model name in addition to the alias
|
||||
db["responses"].add_column("resolved_model", str)
|
||||
|
||||
|
||||
@migration
|
||||
def m020_tool_results_attachments(db):
|
||||
db["tool_results_attachments"].create(
|
||||
{
|
||||
"tool_result_id": int,
|
||||
"attachment_id": str,
|
||||
"order": int,
|
||||
},
|
||||
foreign_keys=(
|
||||
("tool_result_id", "tool_results", "id"),
|
||||
("attachment_id", "attachments", "id"),
|
||||
),
|
||||
pk=("tool_result_id", "attachment_id"),
|
||||
)
|
||||
|
||||
|
||||
@migration
|
||||
def m021_tool_results_exception(db):
|
||||
db["tool_results"].add_column("exception", str)
|
||||
|
||||
|
||||
@migration
|
||||
def m022_response_reasoning(db):
|
||||
# Concatenated visible reasoning text emitted during the response.
|
||||
# NULL/empty when no reasoning was emitted or when the provider
|
||||
# only reported an opaque token count (the redacted-marker case).
|
||||
db["responses"].add_column("reasoning", str)
|
||||
+3459
File diff suppressed because it is too large
Load Diff
+352
@@ -0,0 +1,352 @@
|
||||
"""Part, Message, and StreamEvent value types.
|
||||
|
||||
Parts represent the structured content of model interactions: text,
|
||||
reasoning, tool calls, tool results, and attachments. A Message wraps a
|
||||
list of Parts with a role. StreamEvent wraps a streaming chunk with type
|
||||
information so consumers can distinguish text from reasoning from tool
|
||||
call fragments as they arrive.
|
||||
|
||||
These types are pure values — identity (ids, parent links, storage keys)
|
||||
is a storage concern that lives elsewhere. Two Messages with identical
|
||||
content are equal.
|
||||
"""
|
||||
|
||||
import base64
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .models import Attachment
|
||||
from .serialization import (
|
||||
AttachmentDict,
|
||||
AttachmentPartDict,
|
||||
MessageDict,
|
||||
PartDict,
|
||||
ReasoningPartDict,
|
||||
TextPartDict,
|
||||
ToolCallPartDict,
|
||||
ToolResultPartDict,
|
||||
)
|
||||
|
||||
|
||||
def _attachment_to_dict(att: Attachment) -> AttachmentDict:
|
||||
d: Dict[str, Any] = {}
|
||||
if att.type:
|
||||
d["type"] = att.type
|
||||
if att.url:
|
||||
d["url"] = att.url
|
||||
if att.path:
|
||||
d["path"] = att.path
|
||||
if att.content:
|
||||
d["content"] = base64.b64encode(att.content).decode("ascii")
|
||||
return d # type: ignore[return-value]
|
||||
|
||||
|
||||
def _attachment_from_dict(d: AttachmentDict) -> Attachment:
|
||||
raw_content = d.get("content")
|
||||
content_bytes: Optional[bytes] = None
|
||||
if isinstance(raw_content, str):
|
||||
content_bytes = base64.b64decode(raw_content)
|
||||
return Attachment(
|
||||
type=d.get("type"),
|
||||
path=d.get("path"),
|
||||
url=d.get("url"),
|
||||
content=content_bytes,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Part:
|
||||
"""Base class for all parts. Role lives on the enclosing Message."""
|
||||
|
||||
def to_dict(self) -> PartDict:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def from_dict(d: PartDict) -> "Part":
|
||||
if d["type"] == "text":
|
||||
return TextPart(
|
||||
text=d["text"],
|
||||
provider_metadata=d.get("provider_metadata"),
|
||||
)
|
||||
if d["type"] == "reasoning":
|
||||
return ReasoningPart(
|
||||
text=d["text"],
|
||||
redacted=d.get("redacted", False),
|
||||
provider_metadata=d.get("provider_metadata"),
|
||||
)
|
||||
if d["type"] == "tool_call":
|
||||
return ToolCallPart(
|
||||
name=d["name"],
|
||||
arguments=d["arguments"],
|
||||
tool_call_id=d.get("tool_call_id"),
|
||||
server_executed=d.get("server_executed", False),
|
||||
provider_metadata=d.get("provider_metadata"),
|
||||
)
|
||||
if d["type"] == "tool_result":
|
||||
return ToolResultPart(
|
||||
name=d["name"],
|
||||
output=d["output"],
|
||||
tool_call_id=d.get("tool_call_id"),
|
||||
server_executed=d.get("server_executed", False),
|
||||
exception=d.get("exception"),
|
||||
attachments=[
|
||||
_attachment_from_dict(a) for a in d.get("attachments", [])
|
||||
],
|
||||
provider_metadata=d.get("provider_metadata"),
|
||||
)
|
||||
if d["type"] == "attachment":
|
||||
att_dict = d.get("attachment")
|
||||
attachment = _attachment_from_dict(att_dict) if att_dict else None
|
||||
return AttachmentPart(
|
||||
attachment=attachment,
|
||||
provider_metadata=d.get("provider_metadata"),
|
||||
)
|
||||
raise ValueError(f"Unknown part type: {d['type']!r}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class TextPart(Part):
|
||||
text: str = ""
|
||||
provider_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
def to_dict(self) -> TextPartDict:
|
||||
d: Dict[str, Any] = {"type": "text", "text": self.text}
|
||||
if self.provider_metadata:
|
||||
d["provider_metadata"] = self.provider_metadata
|
||||
return d # type: ignore[return-value]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReasoningPart(Part):
|
||||
"""Reasoning/thinking tokens from the model.
|
||||
|
||||
`redacted=True, text=""` is the marker for the opaque-reasoning
|
||||
case (OpenAI GPT-5 series, Gemini without `includeThoughts`) where
|
||||
the provider reports that reasoning happened but withholds the
|
||||
content. The actual token total lives on response.token_details
|
||||
(e.g. `reasoning_tokens`); the Part only records the structural
|
||||
fact that reasoning occurred.
|
||||
"""
|
||||
|
||||
text: str = ""
|
||||
redacted: bool = False
|
||||
provider_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
def to_dict(self) -> ReasoningPartDict:
|
||||
d: Dict[str, Any] = {"type": "reasoning", "text": self.text}
|
||||
if self.redacted:
|
||||
d["redacted"] = True
|
||||
if self.provider_metadata:
|
||||
d["provider_metadata"] = self.provider_metadata
|
||||
return d # type: ignore[return-value]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallPart(Part):
|
||||
"""A request by the model to call a tool.
|
||||
|
||||
`server_executed=True` marks calls the provider executed on the
|
||||
server (Anthropic web search, Gemini code execution) rather than
|
||||
the LLM tool framework.
|
||||
"""
|
||||
|
||||
name: str = ""
|
||||
arguments: Dict[str, Any] = field(default_factory=dict)
|
||||
tool_call_id: Optional[str] = None
|
||||
server_executed: bool = False
|
||||
provider_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
def to_dict(self) -> ToolCallPartDict:
|
||||
d: Dict[str, Any] = {
|
||||
"type": "tool_call",
|
||||
"name": self.name,
|
||||
"arguments": self.arguments,
|
||||
}
|
||||
if self.tool_call_id is not None:
|
||||
d["tool_call_id"] = self.tool_call_id
|
||||
if self.server_executed:
|
||||
d["server_executed"] = True
|
||||
if self.provider_metadata:
|
||||
d["provider_metadata"] = self.provider_metadata
|
||||
return d # type: ignore[return-value]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolResultPart(Part):
|
||||
"""The result of a tool call."""
|
||||
|
||||
name: str = ""
|
||||
output: str = ""
|
||||
tool_call_id: Optional[str] = None
|
||||
server_executed: bool = False
|
||||
attachments: List[Any] = field(default_factory=list)
|
||||
exception: Optional[str] = None
|
||||
provider_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
def to_dict(self) -> ToolResultPartDict:
|
||||
d: Dict[str, Any] = {
|
||||
"type": "tool_result",
|
||||
"name": self.name,
|
||||
"output": self.output,
|
||||
}
|
||||
if self.tool_call_id is not None:
|
||||
d["tool_call_id"] = self.tool_call_id
|
||||
if self.server_executed:
|
||||
d["server_executed"] = True
|
||||
if self.exception is not None:
|
||||
d["exception"] = self.exception
|
||||
if self.attachments:
|
||||
d["attachments"] = [_attachment_to_dict(a) for a in self.attachments]
|
||||
if self.provider_metadata:
|
||||
d["provider_metadata"] = self.provider_metadata
|
||||
return d # type: ignore[return-value]
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttachmentPart(Part):
|
||||
"""An inline attachment (image, audio, file)."""
|
||||
|
||||
attachment: Optional[Attachment] = None
|
||||
provider_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
def to_dict(self) -> AttachmentPartDict:
|
||||
d: Dict[str, Any] = {"type": "attachment"}
|
||||
if self.attachment:
|
||||
d["attachment"] = _attachment_to_dict(self.attachment)
|
||||
if self.provider_metadata:
|
||||
d["provider_metadata"] = self.provider_metadata
|
||||
return d # type: ignore[return-value]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
"""A single turn in a conversation: role + list of parts.
|
||||
|
||||
`parts` contains one or more Part objects. `provider_metadata`
|
||||
carries opaque provider-specific data attached to the message as a
|
||||
whole; part-level data lives on the individual Part's
|
||||
`provider_metadata`.
|
||||
"""
|
||||
|
||||
role: str
|
||||
parts: List[Part] = field(default_factory=list)
|
||||
provider_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
def to_dict(self) -> MessageDict:
|
||||
d: Dict[str, Any] = {
|
||||
"role": self.role,
|
||||
"parts": [p.to_dict() for p in self.parts],
|
||||
}
|
||||
if self.provider_metadata:
|
||||
d["provider_metadata"] = self.provider_metadata
|
||||
return d # type: ignore[return-value]
|
||||
|
||||
@staticmethod
|
||||
def from_dict(d: MessageDict) -> "Message":
|
||||
return Message(
|
||||
role=d["role"],
|
||||
parts=[Part.from_dict(p) for p in d.get("parts", [])],
|
||||
provider_metadata=d.get("provider_metadata"),
|
||||
)
|
||||
|
||||
|
||||
def normalize_parts(items: Any) -> List[Part]:
|
||||
"""Normalize helper inputs to a list of Part objects.
|
||||
|
||||
Accepts str (→ TextPart), Attachment (→ AttachmentPart), Part
|
||||
(passed through), or a list/tuple of those (flattened one level).
|
||||
"""
|
||||
out: List[Part] = []
|
||||
for item in items:
|
||||
if isinstance(item, Part):
|
||||
out.append(item)
|
||||
elif isinstance(item, str):
|
||||
out.append(TextPart(text=item))
|
||||
elif isinstance(item, Attachment):
|
||||
out.append(AttachmentPart(attachment=item))
|
||||
elif isinstance(item, (list, tuple)):
|
||||
out.extend(normalize_parts(item))
|
||||
else:
|
||||
raise TypeError(f"Cannot convert {item!r} to an llm Part")
|
||||
return out
|
||||
|
||||
|
||||
def system(*items: Any, provider_metadata: Optional[Dict[str, Any]] = None) -> Message:
|
||||
"Build a Message with role='system'."
|
||||
return Message(
|
||||
role="system",
|
||||
parts=normalize_parts(items),
|
||||
provider_metadata=provider_metadata,
|
||||
)
|
||||
|
||||
|
||||
def user(*items: Any, provider_metadata: Optional[Dict[str, Any]] = None) -> Message:
|
||||
"Build a Message with role='user'."
|
||||
return Message(
|
||||
role="user",
|
||||
parts=normalize_parts(items),
|
||||
provider_metadata=provider_metadata,
|
||||
)
|
||||
|
||||
|
||||
def assistant(
|
||||
*items: Any, provider_metadata: Optional[Dict[str, Any]] = None
|
||||
) -> Message:
|
||||
"Build a Message with role='assistant'."
|
||||
return Message(
|
||||
role="assistant",
|
||||
parts=normalize_parts(items),
|
||||
provider_metadata=provider_metadata,
|
||||
)
|
||||
|
||||
|
||||
def tool_message(
|
||||
*items: Any, provider_metadata: Optional[Dict[str, Any]] = None
|
||||
) -> Message:
|
||||
"Build a Message with role='tool' (typically wrapping ToolResultParts)."
|
||||
return Message(
|
||||
role="tool",
|
||||
parts=normalize_parts(items),
|
||||
provider_metadata=provider_metadata,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamEvent:
|
||||
"""A streaming event from a model response.
|
||||
|
||||
`part_index` groups events into parts. When left at its default of
|
||||
`None`, the framework allocates an index automatically: consecutive
|
||||
same-family text/reasoning events concatenate, tool-call events
|
||||
group by `tool_call_id`, and `tool_result` always starts its own
|
||||
part. Pass an explicit integer only to override the default
|
||||
grouping (e.g. forcing a single TextPart across non-adjacent text
|
||||
bursts).
|
||||
|
||||
`redacted=True` (only meaningful on `type="reasoning"` events with
|
||||
an empty `chunk`) signals that opaque reasoning happened — content
|
||||
withheld by the provider, token total on response.token_details.
|
||||
The framework hoists redacted reasoning Parts to the start of the
|
||||
assembled message regardless of when they were emitted in the
|
||||
stream, so UIs can render them before the visible content.
|
||||
|
||||
`provider_metadata` carries opaque provider data (Anthropic
|
||||
`signature`, Gemini `thoughtSignature`, OpenAI `encrypted_content`)
|
||||
that must be echoed back on the next request; the framework merges
|
||||
it onto the finalized Part (last non-None wins per top-level key).
|
||||
|
||||
`message_index` is for providers that emit multiple assistant
|
||||
messages in a single response (Anthropic server-side tool
|
||||
execution); most plugins leave it at 0.
|
||||
"""
|
||||
|
||||
type: str # "text" / "reasoning" / "tool_call_name" /
|
||||
# "tool_call_args" / "tool_result"
|
||||
chunk: str
|
||||
part_index: Optional[int] = None
|
||||
tool_call_id: Optional[str] = None
|
||||
server_executed: bool = False
|
||||
tool_name: Optional[str] = None
|
||||
redacted: bool = False
|
||||
provider_metadata: Optional[Dict[str, Any]] = None
|
||||
message_index: int = 0
|
||||
@@ -0,0 +1,50 @@
|
||||
import importlib
|
||||
from importlib import metadata
|
||||
import os
|
||||
import pluggy
|
||||
import sys
|
||||
from . import hookspecs
|
||||
|
||||
DEFAULT_PLUGINS = (
|
||||
"llm.default_plugins.openai_models",
|
||||
"llm.default_plugins.default_tools",
|
||||
)
|
||||
|
||||
pm = pluggy.PluginManager("llm")
|
||||
pm.add_hookspecs(hookspecs)
|
||||
|
||||
LLM_LOAD_PLUGINS = os.environ.get("LLM_LOAD_PLUGINS", None)
|
||||
|
||||
_loaded = False
|
||||
|
||||
|
||||
def load_plugins():
|
||||
global _loaded
|
||||
if _loaded:
|
||||
return
|
||||
_loaded = True
|
||||
if not hasattr(sys, "_called_from_test") and LLM_LOAD_PLUGINS is None:
|
||||
# Only load plugins if not running tests
|
||||
pm.load_setuptools_entrypoints("llm")
|
||||
|
||||
# Load any plugins specified in LLM_LOAD_PLUGINS")
|
||||
if LLM_LOAD_PLUGINS is not None:
|
||||
for package_name in [
|
||||
name for name in LLM_LOAD_PLUGINS.split(",") if name.strip()
|
||||
]:
|
||||
try:
|
||||
distribution = metadata.distribution(package_name) # Updated call
|
||||
llm_entry_points = [
|
||||
ep for ep in distribution.entry_points if ep.group == "llm"
|
||||
]
|
||||
for entry_point in llm_entry_points:
|
||||
mod = entry_point.load()
|
||||
pm.register(mod, name=entry_point.name)
|
||||
# Ensure name can be found in plugin_to_distinfo later:
|
||||
pm._plugin_distinfo.append((mod, distribution)) # type: ignore
|
||||
except metadata.PackageNotFoundError:
|
||||
sys.stderr.write(f"Plugin {package_name} could not be found\n")
|
||||
|
||||
for plugin in DEFAULT_PLUGINS:
|
||||
mod = importlib.import_module(plugin)
|
||||
pm.register(mod, plugin)
|
||||
@@ -0,0 +1,182 @@
|
||||
"""TypedDict spec for the JSON-safe wire form of Part, Message, and Response.
|
||||
|
||||
These are the exact shapes returned by ``Part.to_dict()``,
|
||||
``Message.to_dict()``, and ``Response.to_dict()`` — and accepted by the
|
||||
matching ``from_dict`` classmethods. They are the canonical wire format;
|
||||
use them to annotate any code that reads or writes serialized llm data.
|
||||
|
||||
Example::
|
||||
|
||||
from llm.serialization import MessageDict
|
||||
|
||||
def save_messages(conn, messages: list[MessageDict]) -> None:
|
||||
for m in messages:
|
||||
conn.execute(
|
||||
"INSERT INTO messages(role, parts_json) VALUES (?, ?)",
|
||||
(m["role"], json.dumps(m["parts"])),
|
||||
)
|
||||
|
||||
Or pair with Pydantic's TypeAdapter for runtime validation::
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from llm.serialization import MessageDict
|
||||
|
||||
msg = TypeAdapter(MessageDict).validate_python(incoming_dict)
|
||||
|
||||
Or export JSON Schema for cross-language consumers::
|
||||
|
||||
schema = TypeAdapter(MessageDict).json_schema()
|
||||
|
||||
The TypedDicts are erased at runtime — zero overhead. ``NotRequired``
|
||||
keys may be absent from a serialized payload; required keys must
|
||||
always be present.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Literal, Union
|
||||
|
||||
# NotRequired moved to typing in 3.11; use typing_extensions for 3.10
|
||||
# support. typing_extensions is a transitive dep via pydantic.
|
||||
from typing_extensions import NotRequired, TypedDict
|
||||
|
||||
__all__ = [
|
||||
"AttachmentDict",
|
||||
"AttachmentPartDict",
|
||||
"MessageDict",
|
||||
"PartDict",
|
||||
"PromptDict",
|
||||
"ReasoningPartDict",
|
||||
"ResponseDict",
|
||||
"TextPartDict",
|
||||
"ToolCallPartDict",
|
||||
"ToolResultPartDict",
|
||||
"UsageDict",
|
||||
]
|
||||
|
||||
|
||||
# ---- Attachment payload (nested inside AttachmentPartDict + tool results) ----
|
||||
|
||||
|
||||
class AttachmentDict(TypedDict, total=False):
|
||||
"""Nested attachment payload. All fields optional — an Attachment
|
||||
may carry a type, a url, a path, and/or base64-encoded content.
|
||||
"""
|
||||
|
||||
type: str
|
||||
url: str
|
||||
path: str
|
||||
# base64-encoded bytes when the attachment was constructed with raw
|
||||
# content= bytes.
|
||||
content: str
|
||||
|
||||
|
||||
# ---- Per-Part TypedDicts (discriminated by the `type` field) -----------------
|
||||
|
||||
|
||||
class TextPartDict(TypedDict):
|
||||
type: Literal["text"]
|
||||
text: str
|
||||
provider_metadata: NotRequired[Dict[str, Any]]
|
||||
|
||||
|
||||
class ReasoningPartDict(TypedDict):
|
||||
type: Literal["reasoning"]
|
||||
text: str
|
||||
# `redacted=True` with `text=""` is the marker for opaque
|
||||
# reasoning (OpenAI GPT-5, Gemini without thoughts). The token
|
||||
# total lives on response usage, not on the Part.
|
||||
redacted: NotRequired[bool]
|
||||
provider_metadata: NotRequired[Dict[str, Any]]
|
||||
|
||||
|
||||
class ToolCallPartDict(TypedDict):
|
||||
type: Literal["tool_call"]
|
||||
name: str
|
||||
arguments: Dict[str, Any]
|
||||
tool_call_id: NotRequired[str]
|
||||
# True for provider-executed calls (Anthropic web search, Gemini code
|
||||
# execution). Adapters use this to restore provider-side blocks on
|
||||
# the next turn.
|
||||
server_executed: NotRequired[bool]
|
||||
provider_metadata: NotRequired[Dict[str, Any]]
|
||||
|
||||
|
||||
class ToolResultPartDict(TypedDict):
|
||||
type: Literal["tool_result"]
|
||||
name: str
|
||||
output: str
|
||||
tool_call_id: NotRequired[str]
|
||||
server_executed: NotRequired[bool]
|
||||
exception: NotRequired[str]
|
||||
attachments: NotRequired[List[AttachmentDict]]
|
||||
provider_metadata: NotRequired[Dict[str, Any]]
|
||||
|
||||
|
||||
class AttachmentPartDict(TypedDict):
|
||||
type: Literal["attachment"]
|
||||
attachment: NotRequired[AttachmentDict]
|
||||
provider_metadata: NotRequired[Dict[str, Any]]
|
||||
|
||||
|
||||
PartDict = Union[
|
||||
TextPartDict,
|
||||
ReasoningPartDict,
|
||||
ToolCallPartDict,
|
||||
ToolResultPartDict,
|
||||
AttachmentPartDict,
|
||||
]
|
||||
"""Discriminated union of Part dict shapes. Use with
|
||||
``pydantic.TypeAdapter(PartDict)`` to validate / dispatch by ``type``.
|
||||
"""
|
||||
|
||||
|
||||
# ---- Message ----------------------------------------------------------------
|
||||
|
||||
|
||||
class MessageDict(TypedDict):
|
||||
"""JSON-safe form of ``llm.Message``.
|
||||
|
||||
``role`` is one of "user", "assistant", "system", "tool" in practice
|
||||
— typed as ``str`` here to leave room for provider-specific values.
|
||||
"""
|
||||
|
||||
role: str
|
||||
parts: List[PartDict]
|
||||
provider_metadata: NotRequired[Dict[str, Any]]
|
||||
|
||||
|
||||
# ---- Response + nested shapes -----------------------------------------------
|
||||
|
||||
|
||||
class PromptDict(TypedDict):
|
||||
"""The ``prompt`` sub-dict of ``Response.to_dict()`` — captures the
|
||||
full input chain that was sent for this turn plus any options that
|
||||
apply."""
|
||||
|
||||
messages: List[MessageDict]
|
||||
options: NotRequired[Dict[str, Any]]
|
||||
system: NotRequired[str]
|
||||
|
||||
|
||||
class UsageDict(TypedDict, total=False):
|
||||
"""Optional usage block on ``ResponseDict``. All fields optional;
|
||||
providers vary in which they report."""
|
||||
|
||||
input: int
|
||||
output: int
|
||||
details: Dict[str, Any]
|
||||
|
||||
|
||||
class ResponseDict(TypedDict):
|
||||
"""JSON-safe form of ``llm.Response`` — everything needed for
|
||||
``Response.from_dict`` to rehydrate and ``response.reply()`` to
|
||||
continue a conversation across a process boundary.
|
||||
"""
|
||||
|
||||
model: str
|
||||
prompt: PromptDict
|
||||
messages: List[MessageDict]
|
||||
# Audit fields — present on a freshly-serialized response, optional
|
||||
# on hand-constructed ones.
|
||||
id: NotRequired[str]
|
||||
usage: NotRequired[UsageDict]
|
||||
datetime_utc: NotRequired[str]
|
||||
@@ -0,0 +1,92 @@
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
import string
|
||||
from typing import Optional, Any, Dict, List, Tuple
|
||||
|
||||
|
||||
class AttachmentType(BaseModel):
|
||||
type: str
|
||||
value: str
|
||||
|
||||
|
||||
class Template(BaseModel):
|
||||
"""A reusable prompt template."""
|
||||
|
||||
name: str
|
||||
prompt: Optional[str] = None
|
||||
system: Optional[str] = None
|
||||
attachments: Optional[List[str]] = None
|
||||
attachment_types: Optional[List[AttachmentType]] = None
|
||||
model: Optional[str] = None
|
||||
defaults: Optional[Dict[str, Any]] = None
|
||||
options: Optional[Dict[str, Any]] = None
|
||||
extract: Optional[bool] = None # For extracting fenced code blocks
|
||||
extract_last: Optional[bool] = None
|
||||
schema_object: Optional[dict] = None
|
||||
fragments: Optional[List[str]] = None
|
||||
system_fragments: Optional[List[str]] = None
|
||||
tools: Optional[List[str]] = None
|
||||
functions: Optional[str] = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
class MissingVariables(Exception):
|
||||
pass
|
||||
|
||||
def __init__(self, **data):
|
||||
super().__init__(**data)
|
||||
# Not a pydantic field to avoid YAML being able to set it
|
||||
# this controls if Python inline functions code is trusted
|
||||
self._functions_is_trusted = False
|
||||
|
||||
def evaluate(
|
||||
self, input: str, params: Optional[Dict[str, Any]] = None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Evaluate the template with the given input and parameters, returning (prompt, system)."""
|
||||
params = params or {}
|
||||
params["input"] = input
|
||||
if self.defaults:
|
||||
for k, v in self.defaults.items():
|
||||
if k not in params:
|
||||
params[k] = v
|
||||
prompt: Optional[str] = None
|
||||
system: Optional[str] = None
|
||||
if not self.prompt:
|
||||
system = self.interpolate(self.system, params)
|
||||
prompt = input
|
||||
else:
|
||||
prompt = self.interpolate(self.prompt, params)
|
||||
system = self.interpolate(self.system, params)
|
||||
return prompt, system
|
||||
|
||||
def vars(self) -> set:
|
||||
"""Return the set of variable names used in the prompt and system templates."""
|
||||
all_vars = set()
|
||||
for text in [self.prompt, self.system]:
|
||||
if not text:
|
||||
continue
|
||||
all_vars.update(self.extract_vars(string.Template(text)))
|
||||
return all_vars
|
||||
|
||||
@classmethod
|
||||
def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[str]:
|
||||
"""Substitute template variables in text with values from params, raising MissingVariables if any are absent."""
|
||||
if not text:
|
||||
return text
|
||||
# Confirm all variables in text are provided
|
||||
string_template = string.Template(text)
|
||||
vars = cls.extract_vars(string_template)
|
||||
missing = [p for p in vars if p not in params]
|
||||
if missing:
|
||||
raise cls.MissingVariables(
|
||||
"Missing variables: {}".format(", ".join(missing))
|
||||
)
|
||||
return string_template.substitute(**params)
|
||||
|
||||
@staticmethod
|
||||
def extract_vars(string_template: string.Template) -> List[str]:
|
||||
"""Extract and return the list of named variable identifiers from a string.Template."""
|
||||
return [
|
||||
match.group("named")
|
||||
for match in string_template.pattern.finditer(string_template.template)
|
||||
if match.group("named")
|
||||
]
|
||||
@@ -0,0 +1,37 @@
|
||||
from datetime import datetime, timezone
|
||||
from importlib.metadata import version
|
||||
import time
|
||||
|
||||
|
||||
def llm_version() -> str:
|
||||
"Return the installed version of llm"
|
||||
return version("llm")
|
||||
|
||||
|
||||
def llm_time() -> dict:
|
||||
"Returns the current time, as local time and UTC"
|
||||
# Get current times
|
||||
utc_time = datetime.now(timezone.utc)
|
||||
local_time = datetime.now()
|
||||
|
||||
# Get timezone information
|
||||
local_tz_name = time.tzname[time.localtime().tm_isdst]
|
||||
is_dst = bool(time.localtime().tm_isdst)
|
||||
|
||||
# Calculate offset
|
||||
offset_seconds = -time.timezone if not is_dst else -time.altzone
|
||||
offset_hours = offset_seconds // 3600
|
||||
offset_minutes = (offset_seconds % 3600) // 60
|
||||
|
||||
timezone_offset = (
|
||||
f"UTC{'+' if offset_hours >= 0 else ''}{offset_hours:02d}:{offset_minutes:02d}"
|
||||
)
|
||||
|
||||
return {
|
||||
"utc_time": utc_time.strftime("%Y-%m-%d %H:%M:%S UTC"),
|
||||
"utc_time_iso": utc_time.isoformat(),
|
||||
"local_timezone": local_tz_name,
|
||||
"local_time": local_time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"timezone_offset": timezone_offset,
|
||||
"is_dst": is_dst,
|
||||
}
|
||||
+735
@@ -0,0 +1,735 @@
|
||||
import click
|
||||
import hashlib
|
||||
import httpx
|
||||
import itertools
|
||||
import json
|
||||
import pathlib
|
||||
import puremagic
|
||||
import re
|
||||
import sqlite_utils
|
||||
import textwrap
|
||||
from typing import Any, List, Dict, Optional, Tuple, Type
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Final
|
||||
|
||||
from ulid import ULID
|
||||
|
||||
MIME_TYPE_FIXES = {
|
||||
"audio/wave": "audio/wav",
|
||||
}
|
||||
|
||||
|
||||
class Fragment(str):
|
||||
def __new__(cls, content, *args, **kwargs):
|
||||
# For immutable classes like str, __new__ creates the string object
|
||||
return super().__new__(cls, content)
|
||||
|
||||
def __init__(self, content, source=""):
|
||||
# Initialize our custom attributes
|
||||
self.source = source
|
||||
|
||||
def id(self):
|
||||
return hashlib.sha256(self.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def mimetype_from_string(content) -> Optional[str]:
|
||||
try:
|
||||
type_ = puremagic.from_string(content, mime=True)
|
||||
return MIME_TYPE_FIXES.get(type_, type_)
|
||||
except puremagic.PureError:
|
||||
return None
|
||||
|
||||
|
||||
def mimetype_from_path(path) -> Optional[str]:
|
||||
try:
|
||||
type_ = puremagic.from_file(path, mime=True)
|
||||
return MIME_TYPE_FIXES.get(type_, type_)
|
||||
except puremagic.PureError:
|
||||
return None
|
||||
|
||||
|
||||
def dicts_to_table_string(
|
||||
headings: List[str], dicts: List[Dict[str, str]]
|
||||
) -> List[str]:
|
||||
max_lengths = [len(h) for h in headings]
|
||||
|
||||
# Compute maximum length for each column
|
||||
for d in dicts:
|
||||
for i, h in enumerate(headings):
|
||||
if h in d and len(str(d[h])) > max_lengths[i]:
|
||||
max_lengths[i] = len(str(d[h]))
|
||||
|
||||
# Generate formatted table strings
|
||||
res = []
|
||||
res.append(" ".join(h.ljust(max_lengths[i]) for i, h in enumerate(headings)))
|
||||
|
||||
for d in dicts:
|
||||
row = []
|
||||
for i, h in enumerate(headings):
|
||||
row.append(str(d.get(h, "")).ljust(max_lengths[i]))
|
||||
res.append(" ".join(row))
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def remove_dict_none_values(d):
|
||||
"""
|
||||
Recursively remove keys with value of None or value of a dict that is all values of None
|
||||
"""
|
||||
if not isinstance(d, dict):
|
||||
return d
|
||||
new_dict = {}
|
||||
for key, value in d.items():
|
||||
if value is not None:
|
||||
if isinstance(value, dict):
|
||||
nested = remove_dict_none_values(value)
|
||||
if nested:
|
||||
new_dict[key] = nested
|
||||
elif isinstance(value, list):
|
||||
new_dict[key] = [remove_dict_none_values(v) for v in value]
|
||||
else:
|
||||
new_dict[key] = value
|
||||
return new_dict
|
||||
|
||||
|
||||
class _LogResponse(httpx.Response):
|
||||
def iter_bytes(self, *args, **kwargs):
|
||||
for chunk in super().iter_bytes(*args, **kwargs):
|
||||
click.echo(chunk.decode(), err=True)
|
||||
yield chunk
|
||||
|
||||
|
||||
class _LogTransport(httpx.BaseTransport):
|
||||
def __init__(self, transport: httpx.BaseTransport):
|
||||
self.transport = transport
|
||||
|
||||
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
||||
response = self.transport.handle_request(request)
|
||||
return _LogResponse(
|
||||
status_code=response.status_code,
|
||||
headers=response.headers,
|
||||
stream=response.stream,
|
||||
extensions=response.extensions,
|
||||
)
|
||||
|
||||
|
||||
def _no_accept_encoding(request: httpx.Request):
|
||||
request.headers.pop("accept-encoding", None)
|
||||
|
||||
|
||||
def _log_response(response: httpx.Response):
|
||||
request = response.request
|
||||
click.echo(f"Request: {request.method} {request.url}", err=True)
|
||||
click.echo(" Headers:", err=True)
|
||||
for key, value in request.headers.items():
|
||||
if key.lower() == "authorization":
|
||||
value = "[...]"
|
||||
if key.lower() == "cookie":
|
||||
value = value.split("=")[0] + "=..."
|
||||
click.echo(f" {key}: {value}", err=True)
|
||||
click.echo(" Body:", err=True)
|
||||
try:
|
||||
request_body = json.loads(request.content)
|
||||
click.echo(
|
||||
textwrap.indent(json.dumps(request_body, indent=2), " "), err=True
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
click.echo(textwrap.indent(request.content.decode(), " "), err=True)
|
||||
click.echo(f"Response: status_code={response.status_code}", err=True)
|
||||
click.echo(" Headers:", err=True)
|
||||
for key, value in response.headers.items():
|
||||
if key.lower() == "set-cookie":
|
||||
value = value.split("=")[0] + "=..."
|
||||
click.echo(f" {key}: {value}", err=True)
|
||||
click.echo(" Body:", err=True)
|
||||
|
||||
|
||||
def logging_client() -> httpx.Client:
|
||||
return httpx.Client(
|
||||
transport=_LogTransport(httpx.HTTPTransport()),
|
||||
event_hooks={"request": [_no_accept_encoding], "response": [_log_response]},
|
||||
)
|
||||
|
||||
|
||||
def simplify_usage_dict(d):
|
||||
# Recursively remove keys with value 0 and empty dictionaries
|
||||
def remove_empty_and_zero(obj):
|
||||
if isinstance(obj, dict):
|
||||
cleaned = {
|
||||
k: remove_empty_and_zero(v)
|
||||
for k, v in obj.items()
|
||||
if v != 0 and v != {}
|
||||
}
|
||||
return {k: v for k, v in cleaned.items() if v is not None and v != {}}
|
||||
return obj
|
||||
|
||||
return remove_empty_and_zero(d) or {}
|
||||
|
||||
|
||||
def token_usage_string(input_tokens, output_tokens, token_details) -> str:
|
||||
bits = []
|
||||
if input_tokens is not None:
|
||||
bits.append(f"{format(input_tokens, ',')} input")
|
||||
if output_tokens is not None:
|
||||
bits.append(f"{format(output_tokens, ',')} output")
|
||||
if token_details:
|
||||
bits.append(json.dumps(token_details))
|
||||
return ", ".join(bits)
|
||||
|
||||
|
||||
def extract_fenced_code_block(text: str, last: bool = False) -> Optional[str]:
|
||||
"""
|
||||
Extracts and returns Markdown fenced code block found in the given text.
|
||||
|
||||
The function handles fenced code blocks that:
|
||||
- Use at least three backticks (`).
|
||||
- May include a language tag immediately after the opening backticks.
|
||||
- Use more than three backticks as long as the closing fence has the same number.
|
||||
|
||||
If no fenced code block is found, the function returns None.
|
||||
|
||||
Args:
|
||||
text (str): The input text to search for a fenced code block.
|
||||
last (bool): Extract the last code block if True, otherwise the first.
|
||||
|
||||
Returns:
|
||||
Optional[str]: The content of the fenced code block, or None if not found.
|
||||
"""
|
||||
# Regex pattern to match fenced code blocks
|
||||
# - ^ or \n ensures that the fence is at the start of a line
|
||||
# - (`{3,}) captures the opening backticks (at least three)
|
||||
# - (\w+)? optionally captures the language tag
|
||||
# - \n matches the newline after the opening fence
|
||||
# - (.*?) non-greedy match for the code block content
|
||||
# - (?P=fence) ensures that the closing fence has the same number of backticks
|
||||
# - [ ]* allows for optional spaces between the closing fence and newline
|
||||
# - (?=\n|$) ensures that the closing fence is followed by a newline or end of string
|
||||
pattern = re.compile(
|
||||
r"""(?m)^(?P<fence>`{3,})(?P<lang>\w+)?\n(?P<code>.*?)^(?P=fence)[ ]*(?=\n|$)""",
|
||||
re.DOTALL,
|
||||
)
|
||||
matches = list(pattern.finditer(text))
|
||||
if matches:
|
||||
match = matches[-1] if last else matches[0]
|
||||
return match.group("code")
|
||||
return None
|
||||
|
||||
|
||||
def make_schema_id(schema: dict) -> Tuple[str, str]:
|
||||
schema_json = json.dumps(schema, separators=(",", ":"))
|
||||
schema_id = hashlib.blake2b(schema_json.encode(), digest_size=16).hexdigest()
|
||||
return schema_id, schema_json
|
||||
|
||||
|
||||
def output_rows_as_json(rows, nl=False, compact=False, json_cols=()):
|
||||
"""
|
||||
Output rows as JSON - either newline-delimited or an array
|
||||
|
||||
Parameters:
|
||||
- rows: Iterable of dictionaries to output
|
||||
- nl: Boolean, if True, use newline-delimited JSON
|
||||
- compact: Boolean, if True uses [{"...": "..."}\n {"...": "..."}] format
|
||||
- json_cols: Iterable of columns that contain JSON
|
||||
|
||||
Yields:
|
||||
- Stream of strings to be output
|
||||
"""
|
||||
current_iter, next_iter = itertools.tee(rows, 2)
|
||||
next(next_iter, None)
|
||||
first = True
|
||||
|
||||
for row, next_row in itertools.zip_longest(current_iter, next_iter):
|
||||
is_last = next_row is None
|
||||
for col in json_cols:
|
||||
row[col] = json.loads(row[col])
|
||||
|
||||
if nl:
|
||||
# Newline-delimited JSON: one JSON object per line
|
||||
yield json.dumps(row)
|
||||
elif compact:
|
||||
# Compact array format: [{"...": "..."}\n {"...": "..."}]
|
||||
yield "{firstchar}{serialized}{maybecomma}{lastchar}".format(
|
||||
firstchar="[" if first else " ",
|
||||
serialized=json.dumps(row),
|
||||
maybecomma="," if not is_last else "",
|
||||
lastchar="]" if is_last else "",
|
||||
)
|
||||
else:
|
||||
# Pretty-printed array format with indentation
|
||||
yield "{firstchar}{serialized}{maybecomma}{lastchar}".format(
|
||||
firstchar="[\n" if first else "",
|
||||
serialized=textwrap.indent(json.dumps(row, indent=2), " "),
|
||||
maybecomma="," if not is_last else "",
|
||||
lastchar="\n]" if is_last else "",
|
||||
)
|
||||
first = False
|
||||
|
||||
if first and not nl:
|
||||
# We didn't output any rows, so yield the empty list
|
||||
yield "[]"
|
||||
|
||||
|
||||
def resolve_schema_input(db, schema_input, load_template):
|
||||
# schema_input might be JSON or a filepath or an ID or t:name
|
||||
if not schema_input:
|
||||
return
|
||||
if schema_input.strip().startswith("t:"):
|
||||
name = schema_input.strip()[2:]
|
||||
schema_object = None
|
||||
try:
|
||||
template = load_template(name)
|
||||
schema_object = template.schema_object
|
||||
except ValueError:
|
||||
raise click.ClickException("Invalid template: {}".format(name))
|
||||
if not schema_object:
|
||||
raise click.ClickException("Template '{}' has no schema".format(name))
|
||||
return template.schema_object
|
||||
if schema_input.strip().startswith("{"):
|
||||
try:
|
||||
return json.loads(schema_input)
|
||||
except ValueError:
|
||||
pass
|
||||
if " " in schema_input.strip() or "," in schema_input:
|
||||
# Treat it as schema DSL
|
||||
return schema_dsl(schema_input)
|
||||
# Is it a file on disk?
|
||||
path = pathlib.Path(schema_input)
|
||||
if path.exists():
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except ValueError:
|
||||
raise click.ClickException("Schema file contained invalid JSON")
|
||||
# Last attempt: is it an ID in the DB?
|
||||
try:
|
||||
row = db["schemas"].get(schema_input)
|
||||
return json.loads(row["content"])
|
||||
except (sqlite_utils.db.NotFoundError, ValueError):
|
||||
raise click.BadParameter("Invalid schema")
|
||||
|
||||
|
||||
def schema_summary(schema: dict) -> str:
|
||||
"""
|
||||
Extract property names from a JSON schema and format them in a
|
||||
concise way that highlights the array/object structure.
|
||||
|
||||
Args:
|
||||
schema (dict): A JSON schema dictionary
|
||||
|
||||
Returns:
|
||||
str: A human-friendly summary of the schema structure
|
||||
"""
|
||||
if not schema or not isinstance(schema, dict):
|
||||
return ""
|
||||
|
||||
schema_type = schema.get("type", "")
|
||||
|
||||
if schema_type == "object":
|
||||
props = schema.get("properties", {})
|
||||
prop_summaries = []
|
||||
|
||||
for name, prop_schema in props.items():
|
||||
prop_type = prop_schema.get("type", "")
|
||||
|
||||
if prop_type == "array":
|
||||
items = prop_schema.get("items", {})
|
||||
items_summary = schema_summary(items)
|
||||
prop_summaries.append(f"{name}: [{items_summary}]")
|
||||
elif prop_type == "object":
|
||||
nested_summary = schema_summary(prop_schema)
|
||||
prop_summaries.append(f"{name}: {nested_summary}")
|
||||
else:
|
||||
prop_summaries.append(name)
|
||||
|
||||
return "{" + ", ".join(prop_summaries) + "}"
|
||||
|
||||
elif schema_type == "array":
|
||||
items = schema.get("items", {})
|
||||
return schema_summary(items)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def schema_dsl(schema_dsl: str, multi: bool = False) -> Dict[str, Any]:
|
||||
"""
|
||||
Build a JSON schema from a concise schema string.
|
||||
|
||||
Args:
|
||||
schema_dsl: A string representing a schema in the concise format.
|
||||
Can be comma-separated or newline-separated.
|
||||
multi: Boolean, return a schema for an "items" array of these
|
||||
|
||||
Returns:
|
||||
A dictionary representing the JSON schema.
|
||||
"""
|
||||
# Type mapping dictionary
|
||||
type_mapping = {
|
||||
"int": "integer",
|
||||
"float": "number",
|
||||
"bool": "boolean",
|
||||
"str": "string",
|
||||
}
|
||||
|
||||
# Initialize the schema dictionary with required elements
|
||||
json_schema: Dict[str, Any] = {"type": "object", "properties": {}, "required": []}
|
||||
|
||||
# Check if the schema is newline-separated or comma-separated
|
||||
if "\n" in schema_dsl:
|
||||
fields = [field.strip() for field in schema_dsl.split("\n") if field.strip()]
|
||||
else:
|
||||
fields = [field.strip() for field in schema_dsl.split(",") if field.strip()]
|
||||
|
||||
# Process each field
|
||||
for field in fields:
|
||||
# Extract field name, type, and description
|
||||
if ":" in field:
|
||||
field_info, description = field.split(":", 1)
|
||||
description = description.strip()
|
||||
else:
|
||||
field_info = field
|
||||
description = ""
|
||||
|
||||
# Process field name and type
|
||||
field_parts = field_info.strip().split()
|
||||
field_name = field_parts[0].strip()
|
||||
|
||||
# Default type is string
|
||||
field_type = "string"
|
||||
|
||||
# If type is specified, use it
|
||||
if len(field_parts) > 1:
|
||||
type_indicator = field_parts[1].strip()
|
||||
if type_indicator in type_mapping:
|
||||
field_type = type_mapping[type_indicator]
|
||||
|
||||
# Add field to properties
|
||||
json_schema["properties"][field_name] = {"type": field_type}
|
||||
|
||||
# Add description if provided
|
||||
if description:
|
||||
json_schema["properties"][field_name]["description"] = description
|
||||
|
||||
# Add field to required list
|
||||
json_schema["required"].append(field_name)
|
||||
|
||||
if multi:
|
||||
return multi_schema(json_schema)
|
||||
else:
|
||||
return json_schema
|
||||
|
||||
|
||||
def multi_schema(schema: dict) -> dict:
|
||||
"Wrap JSON schema in an 'items': [] array"
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {"items": {"type": "array", "items": schema}},
|
||||
"required": ["items"],
|
||||
}
|
||||
|
||||
|
||||
def find_unused_key(item: dict, key: str) -> str:
|
||||
'Return unused key, e.g. for {"id": "1"} and key "id" returns "id_"'
|
||||
while key in item:
|
||||
key += "_"
|
||||
return key
|
||||
|
||||
|
||||
def truncate_string(
|
||||
text: str,
|
||||
max_length: int = 100,
|
||||
normalize_whitespace: bool = False,
|
||||
keep_end: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Truncate a string to a maximum length, with options to normalize whitespace and keep both start and end.
|
||||
|
||||
Args:
|
||||
text: The string to truncate
|
||||
max_length: Maximum length of the result string
|
||||
normalize_whitespace: If True, replace all whitespace with a single space
|
||||
keep_end: If True, keep both beginning and end of string
|
||||
|
||||
Returns:
|
||||
Truncated string
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
|
||||
if normalize_whitespace:
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
|
||||
if len(text) <= max_length:
|
||||
return text
|
||||
|
||||
# Minimum sensible length for keep_end is 9 characters: "a... z"
|
||||
min_keep_end_length = 9
|
||||
|
||||
if keep_end and max_length >= min_keep_end_length:
|
||||
# Calculate how much text to keep at each end
|
||||
# Subtract 5 for the "... " separator
|
||||
cutoff = (max_length - 5) // 2
|
||||
return text[:cutoff] + "... " + text[-cutoff:]
|
||||
else:
|
||||
# Fall back to simple truncation for very small max_length
|
||||
return text[: max_length - 3] + "..."
|
||||
|
||||
|
||||
def ensure_fragment(db, content):
|
||||
sql = """
|
||||
insert into fragments (hash, content, datetime_utc, source)
|
||||
values (:hash, :content, datetime('now'), :source)
|
||||
on conflict(hash) do nothing
|
||||
"""
|
||||
hash_id = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
source = None
|
||||
if isinstance(content, Fragment):
|
||||
source = content.source
|
||||
with db.conn:
|
||||
db.execute(sql, {"hash": hash_id, "content": content, "source": source})
|
||||
return list(
|
||||
db.query("select id from fragments where hash = :hash", {"hash": hash_id})
|
||||
)[0]["id"]
|
||||
|
||||
|
||||
def ensure_tool(db, tool):
|
||||
sql = """
|
||||
insert into tools (hash, name, description, input_schema, plugin)
|
||||
values (:hash, :name, :description, :input_schema, :plugin)
|
||||
on conflict(hash) do nothing
|
||||
"""
|
||||
with db.conn:
|
||||
db.execute(
|
||||
sql,
|
||||
{
|
||||
"hash": tool.hash(),
|
||||
"name": tool.name,
|
||||
"description": tool.description,
|
||||
"input_schema": json.dumps(tool.input_schema),
|
||||
"plugin": tool.plugin,
|
||||
},
|
||||
)
|
||||
return list(
|
||||
db.query("select id from tools where hash = :hash", {"hash": tool.hash()})
|
||||
)[0]["id"]
|
||||
|
||||
|
||||
def maybe_fenced_code(content: str) -> str:
|
||||
"Return the content as a fenced code block if it looks like code"
|
||||
is_code = False
|
||||
if content.count("<") > 10:
|
||||
is_code = True
|
||||
if not is_code:
|
||||
# Are 90% of the lines under 120 chars?
|
||||
lines = content.splitlines()
|
||||
if len(lines) > 3:
|
||||
num_short = sum(1 for line in lines if len(line) < 120)
|
||||
if num_short / len(lines) > 0.9:
|
||||
is_code = True
|
||||
if is_code:
|
||||
# Find number of backticks not already present
|
||||
num_backticks = 3
|
||||
while "`" * num_backticks in content:
|
||||
num_backticks += 1
|
||||
# Add backticks
|
||||
content = (
|
||||
"\n"
|
||||
+ "`" * num_backticks
|
||||
+ "\n"
|
||||
+ content.strip()
|
||||
+ "\n"
|
||||
+ "`" * num_backticks
|
||||
)
|
||||
return content
|
||||
|
||||
|
||||
_plugin_prefix_re = re.compile(r"^[a-zA-Z0-9_-]+:")
|
||||
|
||||
|
||||
def has_plugin_prefix(value: str) -> bool:
|
||||
"Check if value starts with alphanumeric prefix followed by a colon"
|
||||
return bool(_plugin_prefix_re.match(value))
|
||||
|
||||
|
||||
def _parse_kwargs(arg_str: str) -> Dict[str, Any]:
|
||||
"""Parse key=value pairs where each value is valid JSON."""
|
||||
tokens = []
|
||||
buf = []
|
||||
depth = 0
|
||||
in_string = False
|
||||
string_char = ""
|
||||
escape = False
|
||||
|
||||
for ch in arg_str:
|
||||
if in_string:
|
||||
buf.append(ch)
|
||||
if escape:
|
||||
escape = False
|
||||
elif ch == "\\":
|
||||
escape = True
|
||||
elif ch == string_char:
|
||||
in_string = False
|
||||
else:
|
||||
if ch in "\"'":
|
||||
in_string = True
|
||||
string_char = ch
|
||||
buf.append(ch)
|
||||
elif ch in "{[(":
|
||||
depth += 1
|
||||
buf.append(ch)
|
||||
elif ch in "}])":
|
||||
depth -= 1
|
||||
buf.append(ch)
|
||||
elif ch == "," and depth == 0:
|
||||
tokens.append("".join(buf).strip())
|
||||
buf = []
|
||||
else:
|
||||
buf.append(ch)
|
||||
if buf:
|
||||
tokens.append("".join(buf).strip())
|
||||
|
||||
kwargs: Dict[str, Any] = {}
|
||||
for token in tokens:
|
||||
if not token:
|
||||
continue
|
||||
if "=" not in token:
|
||||
raise ValueError(f"Invalid keyword spec segment: '{token}'")
|
||||
key, value_str = token.split("=", 1)
|
||||
key = key.strip()
|
||||
value_str = value_str.strip()
|
||||
try:
|
||||
value = json.loads(value_str)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"Value for '{key}' is not valid JSON: {value_str}") from e
|
||||
kwargs[key] = value
|
||||
return kwargs
|
||||
|
||||
|
||||
def instantiate_from_spec(class_map: Dict[str, Type], spec: str):
|
||||
"""
|
||||
Instantiate a class from a specification string with flexible argument formats.
|
||||
|
||||
This function parses a specification string that defines a class name and its
|
||||
constructor arguments, then instantiates the class using the provided class
|
||||
mapping. The specification supports multiple argument formats for flexibility.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
class_map : Dict[str, Type]
|
||||
A mapping from class names (strings) to their corresponding class objects.
|
||||
Only classes present in this mapping can be instantiated.
|
||||
spec : str
|
||||
A specification string defining the class to instantiate and its arguments.
|
||||
|
||||
Format: "ClassName" or "ClassName(arguments)"
|
||||
|
||||
Supported argument formats:
|
||||
- Empty: ClassName() - calls constructor with no arguments
|
||||
- JSON object: ClassName({"key": "value", "other": 42}) - unpacked as **kwargs
|
||||
- Single JSON value: ClassName("hello") or ClassName([1,2,3]) - passed as single positional argument
|
||||
- Key-value pairs: ClassName(name="test", count=5, items=[1,2]) - parsed as individual kwargs
|
||||
where values must be valid JSON
|
||||
|
||||
Returns
|
||||
-------
|
||||
object
|
||||
An instance of the specified class, constructed with the parsed arguments.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If the spec string format is invalid, if the class name is not found in
|
||||
class_map, if JSON parsing fails, or if argument parsing encounters errors.
|
||||
"""
|
||||
m = re.fullmatch(r"\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\((.*)\))?\s*$", spec)
|
||||
if not m:
|
||||
raise ValueError(f"Invalid spec string: '{spec}'")
|
||||
class_name, arg_body = m.group(1), (m.group(2) or "").strip()
|
||||
if class_name not in class_map:
|
||||
raise ValueError(f"Unknown class '{class_name}'")
|
||||
|
||||
cls = class_map[class_name]
|
||||
|
||||
# No arguments at all
|
||||
if arg_body == "":
|
||||
return cls()
|
||||
|
||||
# Starts with { -> JSON object to kwargs
|
||||
if arg_body.lstrip().startswith("{"):
|
||||
try:
|
||||
kw = json.loads(arg_body)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError("Argument JSON object is not valid JSON") from e
|
||||
if not isinstance(kw, dict):
|
||||
raise ValueError("Top-level JSON must be an object when using {} form")
|
||||
return cls(**kw)
|
||||
|
||||
# Starts with quote / number / [ / t f n for single positional JSON value
|
||||
if re.match(r'\s*(["\[\d\-]|true|false|null)', arg_body, re.I):
|
||||
try:
|
||||
positional_value = json.loads(arg_body)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError("Positional argument must be valid JSON") from e
|
||||
return cls(positional_value)
|
||||
|
||||
# Otherwise treat as key=value pairs
|
||||
kwargs = _parse_kwargs(arg_body)
|
||||
return cls(**kwargs)
|
||||
|
||||
|
||||
NANOSECS_IN_MILLISECS = 1000000
|
||||
TIMESTAMP_LEN = 6
|
||||
RANDOMNESS_LEN = 10
|
||||
|
||||
_lock: Final = threading.Lock()
|
||||
_last: Optional[bytes] = None # 16-byte last produced ULID
|
||||
|
||||
|
||||
def monotonic_ulid() -> ULID:
|
||||
"""
|
||||
Return a ULID instance that is guaranteed to be *strictly larger* than every
|
||||
other ULID returned by this function inside the same process.
|
||||
|
||||
It works the same way the reference JavaScript `monotonicFactory` does:
|
||||
* If the current call happens in the same millisecond as the previous
|
||||
one, the 80-bit randomness part is incremented by exactly one.
|
||||
* As soon as the system clock moves forward, a brand-new ULID with
|
||||
cryptographically secure randomness is generated.
|
||||
* If more than 2**80 ULIDs are requested within a single millisecond
|
||||
an `OverflowError` is raised (practically impossible).
|
||||
"""
|
||||
global _last
|
||||
|
||||
now_ms = time.time_ns() // NANOSECS_IN_MILLISECS
|
||||
|
||||
with _lock:
|
||||
# First call
|
||||
if _last is None:
|
||||
_last = _fresh(now_ms)
|
||||
return ULID(_last)
|
||||
|
||||
# Decode timestamp from the last ULID we handed out
|
||||
last_ms = int.from_bytes(_last[:TIMESTAMP_LEN], "big")
|
||||
|
||||
# If the millisecond is the same, increment the randomness
|
||||
if now_ms == last_ms:
|
||||
rand_int = int.from_bytes(_last[TIMESTAMP_LEN:], "big") + 1
|
||||
if rand_int >= 1 << (RANDOMNESS_LEN * 8):
|
||||
raise OverflowError(
|
||||
"Randomness overflow: > 2**80 ULIDs requested "
|
||||
"in one millisecond!"
|
||||
)
|
||||
randomness = rand_int.to_bytes(RANDOMNESS_LEN, "big")
|
||||
_last = _last[:TIMESTAMP_LEN] + randomness
|
||||
return ULID(_last)
|
||||
|
||||
# New millisecond, start fresh
|
||||
_last = _fresh(now_ms)
|
||||
return ULID(_last)
|
||||
|
||||
|
||||
def _fresh(ms: int) -> bytes:
|
||||
"""Build a brand-new 16-byte ULID for the given millisecond."""
|
||||
timestamp = int.to_bytes(ms, TIMESTAMP_LEN, "big")
|
||||
randomness = os.urandom(RANDOMNESS_LEN)
|
||||
return timestamp + randomness
|
||||
@@ -0,0 +1,10 @@
|
||||
[mypy]
|
||||
|
||||
[mypy-pluggy.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-click_default_group.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-sqlite_migrate.*]
|
||||
ignore_missing_imports = True
|
||||
@@ -0,0 +1,86 @@
|
||||
[project]
|
||||
name = "llm"
|
||||
version = "0.32a3"
|
||||
description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine."
|
||||
readme = { file = "README.md", content-type = "text/markdown" }
|
||||
authors = [
|
||||
{ name = "Simon Willison" },
|
||||
]
|
||||
license = "Apache-2.0"
|
||||
requires-python = ">=3.10"
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: End Users/Desktop",
|
||||
"Intended Audience :: Science/Research",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
"Topic :: Text Processing :: Linguistic",
|
||||
"Topic :: Utilities",
|
||||
]
|
||||
|
||||
dependencies = [
|
||||
"click",
|
||||
"condense-json>=0.1.3",
|
||||
"openai>=2.32.0",
|
||||
"click-default-group>=1.2.3",
|
||||
"sqlite-utils>=3.37",
|
||||
"sqlite-migrate>=0.1a2",
|
||||
"pydantic>=2.0.0",
|
||||
"PyYAML",
|
||||
"pluggy",
|
||||
"python-ulid",
|
||||
"setuptools",
|
||||
"pip",
|
||||
"pyreadline3; sys_platform == 'win32'",
|
||||
"puremagic",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"build",
|
||||
"click>=8.2.0",
|
||||
"pytest",
|
||||
"numpy",
|
||||
"pytest-httpx>=0.33.0",
|
||||
"pytest-asyncio",
|
||||
"cogapp",
|
||||
"mypy>=1.10.0",
|
||||
"black>=26.3.1",
|
||||
"pytest-recording",
|
||||
"ruff",
|
||||
"syrupy",
|
||||
"types-click",
|
||||
"types-PyYAML",
|
||||
"types-setuptools",
|
||||
"llm-echo==0.3a3",
|
||||
# docs
|
||||
"sphinx==7.2.6",
|
||||
"furo==2023.9.10",
|
||||
"sphinx-autobuild",
|
||||
"sphinx-copybutton",
|
||||
"sphinx-markdown-builder==0.6.8",
|
||||
"myst-parser",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/simonw/llm"
|
||||
Documentation = "https://llm.datasette.io/"
|
||||
Issues = "https://github.com/simonw/llm/issues"
|
||||
CI = "https://github.com/simonw/llm/actions"
|
||||
Changelog = "https://github.com/simonw/llm/releases"
|
||||
|
||||
[project.scripts]
|
||||
llm = "llm.cli:cli"
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["llm*"]
|
||||
@@ -0,0 +1,2 @@
|
||||
[pytest]
|
||||
asyncio_default_fixture_loop_scope = function
|
||||
@@ -0,0 +1,109 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Reply
|
||||
with exactly: pong"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '182'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAA3RU247jIAx971dEPE9HSS9J2l8ZrZADTsuWAAumM9Wo/74KadJmpn1LfPDB9vHh
|
||||
e5FlTEm2z5jH4HheS9mut0W1LcpNJcu8KHctNEVTQwWiLiCvi2aHq02eY121Tblmbz2Fbf6ioJHG
|
||||
moBDXHgEQsmhx4qqqvN1VVRlwgIBxdDnCNs5jYRySGpAnA7eRtPX1YIOOISV1soc2D77XmRZljEH
|
||||
F/R9vsQzauvQs0WWXYeLR8ofV1cJRe9tn2mi1inQevwX0YgLd2hA04Xts/w9T5gyIxmXSKB0eMxU
|
||||
JpCPgpQ1s3gHX9xGcpE42RP+BslazQXoOV1nJeq+p4Oj5fZ9u1zlq3KZb5ar26T7Ax766x7ThpvY
|
||||
PvtIoxkGNEnbhcNrZXf1qk7KNtiWuw0IWcntusjLdF9ioYvDxIMhwAHvwCsJEyisITT3oh4Lm9GO
|
||||
Y8IvmrLTATDGEoyj/fgzA7U9OG+bJ0gi2mfMWXNgE3C9fU1nmTtCSAW0yoDmYMIn+nv93uqEQggq
|
||||
EBgauHqexMEceNAa9VxK8nHYV+cxoBH4ZKWcx7OyMfDRLTwJNcnpvO0ccQHiiPyEl5eYx37EwzKw
|
||||
1eY47IhHCNbMvIJta32airaftx5ZiF0HfmSfvBOgRbpwJXvqVuHMKQH9WQnkpEbvtRD1IBwLZD3O
|
||||
TEvYuX5fY4oXt/5vCt1qa63v4P7/sBnp3DD1W8ln9I0NKk2TdShV7O6mH3Q4WiUG4SJZNgH3RWFk
|
||||
HX9Yn3wKuqTRrh4CPhoxOo1JFaDR4xMVkw+mDpSZ+bwo3n7HHx6Pqc8korwn5rNefz4f22fxZ7ST
|
||||
/q+YyRLoh3rLaYQxzOXukEACQU9/XVz/AwAA//8DAEAnS6QwBgAA
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f74ce61c87211fd-ORD
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 03:12:57 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '1036'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=UYuMLYCou_tA0.A8bkTTDs0INBmHlG2j7U9W9gdY8rQ-1778037177-1.0.1.1-5g.mxYNPGVDfBz2paukjljQbRq5PH1x6hIXpFFHvfnbkZxklDSJMF7lUnwNKNv48F1tAMpQxDGkwbCVoTbZJsbFM324N18q7IanrQ5hcaB8;
|
||||
path=/; expires=Wed, 06-May-26 03:42:57 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=j9wkRiR3FXF4b0PnFv7DnktylnlM._0WSw8ZUpHPQmQ-1778037177704-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
vary:
|
||||
- Accept-Encoding
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_c6713bbdac8d4639a5ea09cb6c5eb5a9
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,136 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Reply
|
||||
with exactly: pong"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '181'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: 'event: response.created
|
||||
|
||||
data: {"type":"response.created","response":{"id":"resp_00592e63e61b66660169fab1b9f8e481a2b321356198d7ac1b","object":"response","created_at":1778037177,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0}
|
||||
|
||||
|
||||
event: response.in_progress
|
||||
|
||||
data: {"type":"response.in_progress","response":{"id":"resp_00592e63e61b66660169fab1b9f8e481a2b321356198d7ac1b","object":"response","created_at":1778037177,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1}
|
||||
|
||||
|
||||
event: response.output_item.added
|
||||
|
||||
data: {"type":"response.output_item.added","item":{"id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":2}
|
||||
|
||||
|
||||
event: response.content_part.added
|
||||
|
||||
data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3}
|
||||
|
||||
|
||||
event: response.output_text.delta
|
||||
|
||||
data: {"type":"response.output_text.delta","content_index":0,"delta":"pong","item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","logprobs":[],"obfuscation":"mZhSFjZXJr0X","output_index":0,"sequence_number":4}
|
||||
|
||||
|
||||
event: response.output_text.done
|
||||
|
||||
data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","logprobs":[],"output_index":0,"sequence_number":5,"text":"pong"}
|
||||
|
||||
|
||||
event: response.content_part.done
|
||||
|
||||
data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"},"sequence_number":6}
|
||||
|
||||
|
||||
event: response.output_item.done
|
||||
|
||||
data: {"type":"response.output_item.done","item":{"id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":7}
|
||||
|
||||
|
||||
event: response.completed
|
||||
|
||||
data: {"type":"response.completed","response":{"id":"resp_00592e63e61b66660169fab1b9f8e481a2b321356198d7ac1b","object":"response","created_at":1778037177,"status":"completed","background":false,"completed_at":1778037178,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"msg_00592e63e61b66660169fab1bada8481a28ef92eb62c168ead","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"pong"}],"phase":"final_answer","role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":11,"input_tokens_details":{"cached_tokens":0},"output_tokens":5,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":16},"user":null,"metadata":{}},"sequence_number":8}
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f74ce69fcf0cb75-DFW
|
||||
content-type:
|
||||
- text/event-stream; charset=utf-8
|
||||
date:
|
||||
- Wed, 06 May 2026 03:12:58 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '512'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=Xlm1i2d4hNcY9FYDrgbksx5Ak.InLICBWuWA7k.o5eU-1778037177.9137826-1.0.1.1-GD76qNB2IKnNrhlsjKWgXNF6BpybgPQmNoOTYCMh4ejIhEDWMMph.l_WZvek4hrh5qeLqHfJ28EH5H2c10KBdqGhET8jLthIUOa15fXX7oxYNG_MZ.CHZ9Fuj4bT9GU.;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
03:42:58 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '39999826'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_445d87d531af499daeb09f7826886b8c
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
+540
@@ -0,0 +1,540 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve
|
||||
this puzzle: call db_lookup(''start''), then follow each instruction step by
|
||||
step. Each lookup tells you the next key to use. Compute each step in your head.
|
||||
State only the final integer."}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look
|
||||
up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '551'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAA/31WWbOaShB+z6+wfA4pQNb7prKouCKuNylqgGGR1ZnBLXX++wXMUc9Jcn2woD+6
|
||||
p6e7Z77v55dWqx157X9abQRxYdMcK7uM7/q86/igw9OMIPvA8SHf4UVXYmSxehYAgILvcLzvO3L7
|
||||
ax0idw7QJe9h8gzDu91FEBDo2aDGGFGUaI6WaL7BMAGkxLWPm6dFAqvv7k4OcOMA5WVW5+WDBMO7
|
||||
OUqSKAsq28/qtTIU4ApR7e/BE0zyonqpgLf7wu8hPy0tNihEKK89szJJGoOP4LGEmXu1C5iBhFwr
|
||||
kP5GN1iUvQezPUhAlOBXzyjDBJUuiapNv9pTcLHzkhQlsUkew99BkueJ7YLkY7g092BS7ykoCMV/
|
||||
4ymWZgWK5ii20358gEC93KvbfaXK8m9TmnuBnq3Ff2+swAK5aazMcIIoQiADlmM5QDfLNUHItYD3
|
||||
1gKcZ3ULHlBVMnQt6jK7eUZg1oxA0K1/vYJCdqaommIXY0+YqLd81telmQHjI6WepFPmzK14GUHO
|
||||
zE99E2+Zsot2OKWvhnHtdtVdKBHztl8bKx8anjCwdupRR+cVHMpTTzQYrtQQve5LdlfVZlveNtPo
|
||||
Fo6G2cGfjHv8BbHCMhg6y7i3K9Sxvrvl3IDKku5ofc5sK8giygD8aXoY7XHIUWf+vNouOlNgcAfa
|
||||
2Y52ypkYWrLgk/FtDjgjuMWbUSzQg4shceKe369jcYMH8tJQlPlyrGKXpNvLjj+Wo1jerpb6lI9N
|
||||
2pobOnaHF2tLbsp84JZeP+oePFM/0YFtDUfJcLUQcl/ZbhaJ5HjFLj8uho567RjhMezE2MiXushk
|
||||
9AqvzpTg0UPYmyxSmlu7etRfhcoAGKCjhtol5CfTIC3X6kBZFlzveobrsyHqnJ5FhidNTzEotas5
|
||||
W7Cz9JgdbGFigI3rDsFinfK7s7xZ0fZ8ONeUmXHFUHRTzJLkqhfaBSqBFSy7Jr6MwV7dT4YbN9pb
|
||||
jAJuXXmMkMJ0QF/h4Hyw6Zsjp3BiZSBqk47FWshiBEqn435U2LNtKmv7/WV06gar7cTJ1ipmlmWw
|
||||
6fXZzOr7OzQ2z9vD9baPOzyMbydhPLyK+ygIaULZpjXboF48vXQp8XAK+nN7p/GMqzNZzoWWpg7C
|
||||
0C3Pu2U6WU4m3djdRLQSUtrWuYqxTA4rfWJYB8yYU3k6lIjLJstz2lW9cO/F9qhQs3XiF4qKD6Zm
|
||||
DlXB9INQo5ZmxgZad+Wb7g0OjCJJyWZCQ8WIgCFsjYE1XwCC6fTG9BgBJOZopkLNOU/miy4zud5E
|
||||
QAxH2vNDe4gSvRcyg1x2SrCzTc3vFzpM5+5MhibS8724GJ+Xi4FQnJkJUkq8twvCnYxdv3NUyg5S
|
||||
97YwG5y1XWbO1wd7UXIOkX0eIGNuGd2CDVMV74y+cgIb2CvYJFBnjiDgU4c7DoQjpvK8353pIkrJ
|
||||
YTnur0tJo02GmUaaG51PaZKq/dnZkj0BpoFxyGSLs9l+D6zSXEaeGk39I+Wm4UQXASscdHDDw0Qe
|
||||
3qgClRcxXEuSyES7ZKqEmT1BkhDH/TTOjDHfIbpDsfLWNaiDtcqSsY4GaZj1Elro+7RPuaqILUFh
|
||||
yQG4CgzluRCQkoZZz5875Wgg8nmX5ehBZKzEwOx5VLzLzkJ4CvsTYxUfHElaHXhmps0uPfqkjDON
|
||||
s4WrhPD5usnnawhLc7IZjEhuQ4TNZVRustv6MOrmnGfIk7HfM8xwSEJho/RKwYRjlhf5hX9b7VTN
|
||||
0repPt92OrP1mabSIH1edbhMU4BqYvj3R2N7+/qnu9Z3/37XirTHSPVdK/kCBB4nsjIPKkZlf79r
|
||||
/TJrOKVhiJck/sKZDQhQUKbVHdzgP7+3Y3j93v7ne+2EyPf22/PLOqh9z7d5HAoinXZvptDd4ZuX
|
||||
H8FQFo4BHT89MpA2eXmOneR5XBbtew2q/x8NAxUAVZFg8pHXKmq8k3dR6YKKLOAf+LWCTlFeYvtd
|
||||
Otwze3BbgaqtkiqkG0K72tNfMQRrBrozY5vlwjthPlnrIRyg7+eooaqwumN+bfKlwXX4h5LAwIfk
|
||||
WuVUx/Yj+EE3YIhOUbUrEr0rER+UCWn/Ejg5gh8kDIFpUbN32diZXwUg8EKeyVWppeD5/jITzXev
|
||||
o9c+QeTkOGrK2U6hF5XpUwLdGxHmVX41DEqStx8A/l0rfB69Z/M9iF0UFe+lHVcD0CqLFmidQFLC
|
||||
lnNtVW1pRVmLhLBVlLdbAlseIMABGH77vxl6QPX0pFX7EH7Z+L2/VcGq6n60V8h9EF5NL1uoJFnd
|
||||
8Bfw7fH89vUZvlZ9EYLeoxgvwR/vP14c3hf4pXdfEOB5UV0gkMxfc256/+XTyk1+jVyuz8eng0Ty
|
||||
oipPUO3cqQPQD2PRnBlZuhtQ1aV3Gdj2Igyc5F0/lxgE8DlQUfZBhErS19/tL8r25/OSqA6V93Sk
|
||||
P4zeZ20r8H8C/hT3cSCfzhz/ITbJCUieKMN3HkNd4o8HsBoaUI9avcDbl7f/ADPaWJDQDAAA
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f7526f32c3c0044-ORD
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 04:13:27 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '2091'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=N7TAdwEKwrRihZ4HPzOd8tAdwT5JnZWp1Mj2.UfCC3g-1778040804.348554-1.0.1.1-NFuNpo0J7tlz8j4YDeoThOIY2CDX.4CkvnAcV4AwMaERfiFLSTPjDY8gcSPYiwIXP3TeplBxIS2vJK_VrxoTlp4TAVqwN8bgJ6CMnQFDfKuS7S0zRVqTDUsqYdgW6ccN;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
04:43:27 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_c48ea45fc57a407bb6b100db8c59af68
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve
|
||||
this puzzle: call db_lookup(''start''), then follow each instruction step by
|
||||
step. Each lookup tells you the next key to use. Compute each step in your head.
|
||||
State only the final integer."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe62a9c8197914677ea9a2424a0","encrypted_content":"gAAAAABp-r_nDEFD_pLd6MEzoOCG8OKekq-Ev8vnbPTkSie4RovCRsX1uArYsm0yKKyAAEYh8tRzZVKUfeKd6HTYEqGrwUeI9Nd7K14uFr0VC8_AEFOX5_RmizhJInjfMLB5xr26SgIbSkBYpELGYzo4H-nlAJVwn_Tgni-Ka5vNjJZsh4-w5wUXQ3NaK4j0bXJYDwtKFlQ5lLzPa4KgzkWJk60HxK847Z5ZVk7WsH9SKDDPSLEsctmXxY5quJk9XUSGN5kR0TPKGscIxTXtzDPHcudCiAjdRGv0g_TIJlIUQ6ofDXWQl8bdpYoqQIbEy3Khqh3ksKoSG71n0UsUw-6d0IeBMQm04VcGiCUhDHaKa3EhFxh5MNgmuVEHDSp4ByweVwK7G4GniKd8NvkauFyROQ2Omqnj_6MKaWccIaQVm5Yw9WU0_PIPFDOKyse7cms2tlyGpFxeDgTgSARsxLaZEZMIWciZT1DazA9LrrD13aCD4ePHWCRJbpbkDH7FM3T2TrT16-G0kCip_OXm9FZZxJvAgUXMbnVEs1SugWBC2nTCfYrLRwXjyzZk35ekzv6LIy7Zigh0t-_RTOWrBkNxA-7jvgCP_YF51cG1no4hTFEHhhcuwYSmMSMMAkcWi0Dh-FXby7k9tjUGMKTjs1RN9NI8tc2lSwmAEdhZdk_JpEnVlfpDEsjRFRIE6RfghF-SRn2gFAUfRczeHKplmtWM0eDKiaK6XKHTPQats0mz1B16alRJOEeFbwMPQA1Myz7atKb8Z5I_IrlGBh1Ho9buaY_RFfCpGemPcO9eRrGoZ7QLwSQH6pw1MrDusZ_pt4vKYC3qDu3rEZ_6OHwFYnRPVj_Qu4bt9f5arKPTKAp2hmEsYKCDvaWeBp2lgEOb66sv34qH6qs-ooCAOG7rmtjSLCVu8F0R11NiFciwvmlmECOwT9d6emgKjn9T4_2CBaUmo9rdEiNfq-cmhMG7a26jGazsIl9Iz-prux7hV8871iYlNDhn_Mr86kkCmknKL53tGb-29XcK-jTUnlLGrHmhnBl06Cf0f-cE7sT6D2tjacDeh9P6gtu0enBfPbuJH75oA240HiKU7gRBd-kYnw6hvhCMKUkjb88Uj51OFOxB0vDLnF4_6y8rswyWoPVeeuRMWHJto_ersRSiuWnzVjJAo4dK9MLfBKRhIth6WDBu6ReL2575QfzUYEFTGXmGPX33OVw0-mgm","summary":[]},{"type":"function_call","call_id":"call_I670mAzR6AYszdoqaI96qg0k","name":"db_lookup","arguments":"{\"key\":
|
||||
\"start\"}"},{"type":"function_call_output","call_id":"call_I670mAzR6AYszdoqaI96qg0k","output":"Begin
|
||||
with the value 7."}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look
|
||||
up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '2127'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAA/31XWbOiyBJ+719xwudmApR1IuZBERBRERG36QmigGKTtQABO85/v6B91NPL5cGg
|
||||
8qvMyq3Mj+9f3t4GgTP4+22AYJGZODnkbMK1Xcq2XDCicILmXGC5kOFYhmUJjuFI3GHhELcIAto4
|
||||
6Qy+9iZSK4R2+WEmTQp4l9sIghI6JugxgmFYnMRZnLlhRQnKquh17DTOItjtuytZwD57KK2S3i8X
|
||||
RAW8i4MoChKvk33vlp0gAy1Evb4DLzBKs27RAe/3gz9Mfj6awG8oRCjtNZMqim4CF8G8gondmhlM
|
||||
QFS2HYj/dd8cJB/GTAeWIIiKV80gKUpU2WXQBf0qj0FjplWZVaVZpmf4K1imaWTaIPpsLk4dGPUx
|
||||
eVmJUX9R2BAf0hhOYsPR4LEBgf64V7X7SZ3k31tq7gl6lrb4c2HZEcneCsv2DwDAtYbAItx7LW5G
|
||||
yjaD99KCIk36EjygLmWozfo022lSwuTWAt64fyYZhswcKDNWwsY4yyrr2Bwr5517viLBrLlQCpxl
|
||||
vvBH7iEFNSdb5VjnprVvmCqf+F4ocuV0pNB0E6Md6e0u0wMVu2FYiOSYRYgrnOO6XNbENrbrZVJO
|
||||
dsplXpyVMAtPkC4op6iTK3+tjGsl0YaqZJssPWVJzMuwpvQYO6XNEV+H5eEYMubRuZ6vh9hil0Ri
|
||||
iW0be3KUgilAMDoihhCxui1acyI367ErVpHN8HBrhtKRzrchdgAHIVzBSzY+bxqtGOVTsXCJy3w+
|
||||
15cbo6IuM1HFzrFmXHiCZ5KVflawsdaIpOc3Q9m0pflKn1VFEko5JYMRx8xMnJ+oWrh1iNlcrkbG
|
||||
XmK9aqmHMy4ewmixFI3MFZqLtSEDBYswWPP2PpNbd984iz2ajuR9HmuU7LCJIM9FhWHjkNzFi3XN
|
||||
ztX1eYwSW7e9Q75V6KqLWzgMZ2WxEvhG3FA1AXcnnWiDRqBVvVOjrth0GjVNg2/TwBoTcy63C0pT
|
||||
NhhuLBpzoZhrz+XGUGHN01DMZ6tE9KlE2e2dVj4emm1WoUwkciFRJrzO10BjDldjHs2xhDBE71Ti
|
||||
xka0Cis/jg6Fz5MVQpe8qNc5sV5Mcl+zT83aKg75UrmKVb2l5zQvREMJGSkfzqzdNJd9IqQjm5pM
|
||||
jwED1PExYSyh1mSz9N1YswJGQGB3WsaeJSSX47m5XOGaYcc6Tx6DY+3JV6kyNOVa5xtFsZoM0inc
|
||||
CQwn8vDgC5ayHZ7wsz056UYaN67OB4tyteTxVcDtRddQK46MNuV27axnbLarMG0iwmvASm6jHmar
|
||||
fFWJ+k42d1p3M+R0TQaAgMfGkod+Vo+j86hiHUPnL/NWdxaLmnPW+5XW2ClFXKcgbrWdAhMrEayQ
|
||||
POWnjRJhw9JMoISP7QOVRwGfT8dknMcLhmwDhOJjSXNnWdRoXeLXGEuf4xGIvbGcqIU62emagmEz
|
||||
oF4mhru1MX6/UNVV7oZDcDCCJa9c9OUl5TBarDySL+SDY3tNrmpHybUpNbMhG28NB+lh7FvEPliq
|
||||
dHTGFzqa5bhsFLLPThPuQFl+V8WEdXiqjpgjYCap23pDYSJtU1K3mQBL3OHEFQ/MOd/GqrGRABmc
|
||||
IOuvsiptS283jaywjVx81RqYJOBeXVmiF1v0NE8Yw5lDRzADcqrIdlllQZZBO64zT+V2RBJE2+iY
|
||||
ro4hklAbbjcneiyEbArgZrgZSSRRG9lydqEroz1uZSUioqE9dmhbwj1qzAq8x1maKxv2eGxrq5Hg
|
||||
uKmcqgdVyPklFvNCGK65WrMb8lquPNxMze2W5vOTt1am1+uepYzOda/cS75B0qS+Foz02MwRn+2D
|
||||
JtdMdzteOSJZJOuRv5FIpgWVDBPRUUfEtQKuOkd2LkxbW3Gj6ewypo6GGihEoKZThQSW5p8mcB1n
|
||||
+1YVXTaq3MJYSCks82Ahkbsl5TtBKOdnWd0HTqDok2VzYA8MZY3hlqr0UcLndKLj4WaxVebMYsHl
|
||||
MxqPoYNxkntdNsohWc4CvIW1t4s11asDRKKCTYdVGRa+hVQ4vZLYCSMzvjjXp9wR8iOu/fPPcwQU
|
||||
VRwD1A/Mf/+7yd6//m4GufafZxAYWjbZzyBAUCTrOC5OugRLctSvM8itktusvU3OFyf+wCVuIEBe
|
||||
FXez6YZ//zY4w/bb4O9vA+bb4P25qzdo3n29vRohhdOBoIST0x7N1vmQH0nT0kNPjQTEN58cy4zS
|
||||
9Fxlg3v83e9/t6mcAdRZgtHnWd/RhTuhyTqu1A1Q+BvO0UGXIK0K84NO3T17zPsMdWGWnUnbh2YX
|
||||
zx8xBPupfGcLgyHp30nEc5I/yBR03RTdxrcfeP6PIF+K25t/sKsCuLBsO596224AP3GpAqJL0EVV
|
||||
Bh/szAVVVA5+kL4UwU+0roRx1jOa6iYnfiSghE35dK5zLQbP9Us/3Pa9tt3gApGVFsEtnYOu0YMq
|
||||
ftLCeyH8tPOvh0FVpoMHUPzKn35uu2fxHVjYKMg+UrvoGuCtyt7A2wVEFXyz2reuLG9B8lb68C2r
|
||||
rtcIvjmgBBYo4F//r4ceUN89cVc+VLwEfq9vl7Auu5/lHXJvhFfRSwgdTe0L/gK+P97fvz7N90w4
|
||||
QNB5JOPF+GP934vCxwE/vgFeEOA4QZ8gEK1ffb7V/stPJ9/8u31C9Pfjp4tUplmXHq+L3OoN4A9h
|
||||
drszHHsXoK5KH9R44AQFsKKPb4qqAB58NlSQfCLmBEN8/RV4ofvfn/8S3a1ynpr4p977mfATBPs7
|
||||
5HeGH1fyqc2xn4yXaQmiJzpkuUdbV8XnK9i1DeibrT/g/cv7/wBdVsXR5g0AAA==
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f7527072b8a0044-ORD
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 04:13:30 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '2766'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=DRZquxDLKhKw.kobZpdPpYdyAORTxb8d3bk6OV6L_3g-1778040807.5506942-1.0.1.1-C63sdNCp_e3AcsWa9mPoKJLTMlDCSogmc69aY3uzn3tTS0hpqMcaswl4Vlx9jLN79I4aGoL9PKji0tX10HxiuZxf4oEWmMjN_QopXLRhPxlCyZuOq26Bg0WQvU0arTlt;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
04:43:30 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_af2b96dc465344fbaecb33d287edd1c1
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve
|
||||
this puzzle: call db_lookup(''start''), then follow each instruction step by
|
||||
step. Each lookup tells you the next key to use. Compute each step in your head.
|
||||
State only the final integer."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe62a9c8197914677ea9a2424a0","encrypted_content":"gAAAAABp-r_nDEFD_pLd6MEzoOCG8OKekq-Ev8vnbPTkSie4RovCRsX1uArYsm0yKKyAAEYh8tRzZVKUfeKd6HTYEqGrwUeI9Nd7K14uFr0VC8_AEFOX5_RmizhJInjfMLB5xr26SgIbSkBYpELGYzo4H-nlAJVwn_Tgni-Ka5vNjJZsh4-w5wUXQ3NaK4j0bXJYDwtKFlQ5lLzPa4KgzkWJk60HxK847Z5ZVk7WsH9SKDDPSLEsctmXxY5quJk9XUSGN5kR0TPKGscIxTXtzDPHcudCiAjdRGv0g_TIJlIUQ6ofDXWQl8bdpYoqQIbEy3Khqh3ksKoSG71n0UsUw-6d0IeBMQm04VcGiCUhDHaKa3EhFxh5MNgmuVEHDSp4ByweVwK7G4GniKd8NvkauFyROQ2Omqnj_6MKaWccIaQVm5Yw9WU0_PIPFDOKyse7cms2tlyGpFxeDgTgSARsxLaZEZMIWciZT1DazA9LrrD13aCD4ePHWCRJbpbkDH7FM3T2TrT16-G0kCip_OXm9FZZxJvAgUXMbnVEs1SugWBC2nTCfYrLRwXjyzZk35ekzv6LIy7Zigh0t-_RTOWrBkNxA-7jvgCP_YF51cG1no4hTFEHhhcuwYSmMSMMAkcWi0Dh-FXby7k9tjUGMKTjs1RN9NI8tc2lSwmAEdhZdk_JpEnVlfpDEsjRFRIE6RfghF-SRn2gFAUfRczeHKplmtWM0eDKiaK6XKHTPQats0mz1B16alRJOEeFbwMPQA1Myz7atKb8Z5I_IrlGBh1Ho9buaY_RFfCpGemPcO9eRrGoZ7QLwSQH6pw1MrDusZ_pt4vKYC3qDu3rEZ_6OHwFYnRPVj_Qu4bt9f5arKPTKAp2hmEsYKCDvaWeBp2lgEOb66sv34qH6qs-ooCAOG7rmtjSLCVu8F0R11NiFciwvmlmECOwT9d6emgKjn9T4_2CBaUmo9rdEiNfq-cmhMG7a26jGazsIl9Iz-prux7hV8871iYlNDhn_Mr86kkCmknKL53tGb-29XcK-jTUnlLGrHmhnBl06Cf0f-cE7sT6D2tjacDeh9P6gtu0enBfPbuJH75oA240HiKU7gRBd-kYnw6hvhCMKUkjb88Uj51OFOxB0vDLnF4_6y8rswyWoPVeeuRMWHJto_ersRSiuWnzVjJAo4dK9MLfBKRhIth6WDBu6ReL2575QfzUYEFTGXmGPX33OVw0-mgm","summary":[]},{"type":"function_call","call_id":"call_I670mAzR6AYszdoqaI96qg0k","name":"db_lookup","arguments":"{\"key\":
|
||||
\"start\"}"},{"type":"function_call_output","call_id":"call_I670mAzR6AYszdoqaI96qg0k","output":"Begin
|
||||
with the value 7."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe83488819788888aaafb2ab1fd","encrypted_content":"gAAAAABp-r_qaKH8G-A088KPm_AKkVfkzrE_w9jGidMqLh3fXoaw9IbtAS9DwhU_OCnhgjF9tD3K66xmrV4gVvDX5mfjjsF4A8rr9sdYPtMw1TmcwMntBVKvJskKjpjZe6s5dswnzCzuUzuG6UOKpRpoZpnmCIew5Sm-ZoxY0PjtXYj7_YdzkzXmb8M1nbFyymgIloaDarelYr71F-wysy_BIxPAfFulc7CeT_jGY6qTj-XaXEjNevpAkRxQs3qDFsf1vJJJSMRUu5vHFO-kmQUvC1C7nNSkK-AQxF4ghx2I_cGJNSHusnjGq5Ia397H_0CBOQjTd1HJIu3UWG8guMSjH9m2elLMFUpfExvbR4iK-l-ewCcWpIyfWxdLWrD3IWqmQ5Id8nEIJFK78mj4VmLPw8JOPkArncScgXqTK6uzkzEX2HtsNECxFR5w1eVZS1yixE6OSj4V5z-DDlxxx0ToibA1J9qcs5QKR-0ULx_LK_Pgf9AeK8_Z2FqHNnFh5nKVWdyIYXxTpurpF1qEnKBCSCwaQ7XzUJlJ-n1UFgZt0URFbsbqY3XshC4urrvqswPq1PLBqhQcZxPbsXqMKzFuwT6J6CEl2GrUoCjHbVDqIh1j6lc5BDYi7aOAYn7bEwQI_thfmQbi7EraVZMmgbEnvYkxvzeP78ASC4YiYwgIzGuUQKzwqRKKbxpe6oeVE79FCeXhEbKT2Z0kcBZSUomxfSCiLtNMC0Ni9WFfUOu94lRtTPdPH8pVu-QBFezi8GfxOXHNqNuFSVI_VQtenIoP4ia1eYxbI2hpwAlk3u8dUSCvJySdLLw9dPWNQxco51zDamyQVKenbnEbj4ZqZRKl-2t_neG0AcX5qliCqDA4mqmL74yirrmYt69kIFQ6SGCP-86km3amgAInOsOBVSQK--HaOvBUfTc-CWLOONqfj2aXUiMCKvSMvo9-6Fug4CsIXdcgxqOQYGfc5Opce8mTUdrSjmhb1WiMO6lk0LSrHq0IUsIh8Dn9X5bhXshn8dC5wl7Ya7Bofyg2EBGTo4Sc7i-nf2BfFX7kqTmOURGa4iZe8hNpuoytgVDlbjylf0NyU-GE0gwubFgmb6Dqn7UdJedE_i4DKIctupippecmwpgO9V1nilTlYoNYjrGryjTRZ6AEj8oaeR2R3G41wUpMHv6uUyYTIKl1l2cAd6cG0g5A8ECg9bQfIUcAAcQN3EdfoIoOXOEqCM-mCEjjP9wQcx4ztNg0_o_TT6CqZgPKDzzW85UgVDgtWGhU464SPEUoYxJrCpWixqQ_fTANdF4snP3hRG47yauIenFdO31zuafOJrcqEDycKflDHvA5YUOiK1iOoDK4abQhZBePmpWyOFf8lufsULGoetqiLG4VM5hdijIqkIOWidiKSBMxX8X75bAeT5uS3nCq6nS0jRLTKJ7LL9qH60med-9GfzMxKXnMHi0yewgVmQOgwir4rs8o2utjshbrOeDz4-Z-4pCskwZqdEqY0Q==","summary":[]},{"type":"function_call","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","name":"db_lookup","arguments":"{\"key\":
|
||||
\"7\"}"},{"type":"function_call_output","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","output":"unknown
|
||||
key"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look
|
||||
up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '3967'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAA/31XWbOiyBJ+n19x4jwPE4CsE3EfVAQERRQQ8M4EwVIg+1Yg0NH//aLO2bp7rg8G
|
||||
lV9lVm4FX3777eXlNQ5e/3x5bUBbOSiBsz4W+iHpe6G7IFGMYkPXC4HLhB7BYCztoQuMoFHCoxkG
|
||||
xQPq9fe7idJLgA/fzJRFC55yvwEuBIHj3jGMphmUQBkMfWAtdGHX3nX8Mq8yMO97Knmun0ZN2RV3
|
||||
v0I3a8FTHGdZXESz7Nu8nAWVO4Lmrh+AHmRlNS9m4Pvz4DeTPxxNPlDQNOVds+iy7CEIG1B3oPBH
|
||||
pwKFm8FxBtE/nn7GxZsxJwDQjbP2s2ZctLDpfBjPQX+W5+7glB2sOujAMgU/g7AsM8d3s6/m8jIA
|
||||
2T2mqIII+QeJ4ChOISiB4IvX9w2Nez/us9rzpFny30dqngn6KG3774X1SIp9FJah0YUbMiS5cFF6
|
||||
4bGP4x5G4FiBZ2ndtizuJXiH5pQ1Y3VPs18WEBSPFoiW99+qQhqn5wVfM48cyiwmUmpril7nMss6
|
||||
l7oubmumoBIiPYfmVRFJia2kGwno7owduyYIEtHGbvQq8M6E2IR6OhGaIcYl5fJA3BuNdzx4XHy6
|
||||
qUwqWrik2Yt8cbNca7kqSKeWYobcJfSaoyx6kUx1cKw2QDgi5LRbSO1wQFPS6C4eikFy002dMN0c
|
||||
dHfC9CxGDelGLBo5Wm9qtxH1rl1k1CLAmMrqZPK0X52QxcUZtuy8B9yGJtVdEeNW0QCzAXLkhQiR
|
||||
AEVOXihghNbXyFFb3NowIq2mYEhZEm8hM0okuadJfDmslk3vxHV7xuxOvixvIz/oqBjvGu5w2zqC
|
||||
cxztkENQAxHbswA22lpDIiK5cSpntRB3JVU/jFeodyjnVdZoOEbqNTC7tcThcOmaDDHpXd9bmSkp
|
||||
AmqaCZVXa19szP3UeUJ61ESG9hHB2mhiuQqG4QQsocy5lojqwlvRgV1FyknpgT3tCul0DDg7DWxR
|
||||
l7Z0tNqcFX51I282Olmnqw3j/uoIhNLtcN72dbVhJr4d6q1hjTtTwhGWVNCk1xy3Ik6o6GSkbTPw
|
||||
Jvc60UJ7uWB1XJUhOndHxqLU2SpdKJmJyY7hLimwcAvwfGE2xNm6pqaR0z29JAfySBsHL77CZYGQ
|
||||
yrU02JWM8f5+Yzuoket+Ziz3iMafm92CgyErwNIudyKjGh6hBdWVHVGO8tyzphSragzOGJOxFoya
|
||||
ThPJCoWsD4bZsWnSl/neY7uEn5eeU5sXZ8VrfCGVNJ3Z6zKWTaCAzT5b2KVJQKvOyzao6kzjK1Fu
|
||||
3AGxBbDnsMKNlPrIbwttByJ/o9VbJkOEsZ5LZS3OwmA6iSpIUXJ1bpeledTlfSz70F8iS5WUDpU0
|
||||
uOp0ia6nvVAFW3pRHA762UmO7OGcyrIJEx7su6Djr+YOxrF/bU1TqxnrJnGGrJ9G29nZq2FSeX7Y
|
||||
Xoe0PuK05xSBpO76kdPZpd4rDEnlgybZIJPRTkqGOsEWB/JibtcVWgWGEG92EYoeMdLYruos4449
|
||||
jTk7WKROqW2qxjplXhzy7Y1ndOXMe714JS+84ybAUjwt2oetU2uQFbsLh8PRa7XWFus8hpXboF3g
|
||||
R4rcj7Eqq5brLdJbrNrbxKGL3s/8en0xjCkI9iMZLDEmaaeNb5MKQQHj7CjwIhzo6xG7kFfNbHJV
|
||||
A7oh8nLE4McwNxkHJB46snWolsPR10PGms4BHbkKlyZauV8jJN97k+NEgKvSk6fUmtqygN+QJ5nW
|
||||
69Rf0aqC14qC5HRSDJHu7kNy6vwlaiW4JiWl7nNrRcPxwKnjdCqRZDp0scTR2mUKNZ3oIikLSW9v
|
||||
CNv4sIUcAZzZ/3i4XkRzwwiiGCR8UWVu221x3bglhib0kdlQ1GZn5tXt6HKjS3kGDp1rIx7Naxq7
|
||||
0JLsKZ7G7SIzhClEKAVzYtAVm03d7Zrq7AbEjtsVtrAlS48/GpLrNRwCjltECuWTLGjZaQUja23B
|
||||
WJcimiBSqSfPUhv58pgSQ1KCcXkzlzssv6HKfCWtA5B2gSlss5J2zOZc8QrUisgu0iVHj9ZotqXq
|
||||
+dWqSp1kgds66V4oRQejTiaiY3F+h1Q1lfJsuw1U6jSitSag1mac5tdGlRveDu3L2zQqplrKVtCa
|
||||
9nlnLX1iyyMbZ9k4+dTupXTlLI5u5uGtjge5SRupax2cTYKr1ZJ1FROp/TNRky1ksNo7FqYjib0b
|
||||
o+Ykb/g4rzYhjZYuw1zo5QkMauzH01LoEGwQ9PVlzwh1AG9ds4uFQ8GYBVsmS62gJ2PDO4N+Yw0/
|
||||
WbFEcrz5jiqLNrVSVxmQZdXWL+awlG7TDrILWrqByZ8EUQ43XLB3uP2Z2zq9RIlILpFFx1yXkkwZ
|
||||
IOcQYUOcNaCpc+sKF8uKC9AHOZ4yokIjskgh03X+oo376UYoq/kLuQ6GvhtSPbjCCqHYzGYYH3Fy
|
||||
ahj7tKm41OBge+RS6pCs7BpQHktAuThe2S3Ll+eDiiEVcVMkhRtkfJms9XZz2U4cEmG+r+3wCwcV
|
||||
dp231jIEfkac5CkYpagXLS/kJ2x14t0uaLKUZtdkeq17FzF3DR8OaUaNrLFBMHzNq5Cm2arLqhJo
|
||||
lneiVBO1nfxiVXwPVZFSCK0Z40Vsz99qOExRcA2otRI0VtkfW2gau6lbaTS1sUdePf7ng1a0XZ67
|
||||
zZ2E/ffvh+z777/iNaH/77wG+PjCv/MalgnDmc9QIYr7LoszP/OasCse/O3Bxj458S/89AG6TdTl
|
||||
M9954N/+ek3B+Nfrn3/NSqDC/nr9/rHzbtR5+vt4PMcpm6dKFjNyW2ieeNCubhXkwodG4eYPvwLP
|
||||
ycoy7arXZw7m/78fbG9+a86WQPaVQ8409EmUq5mDz8QM/ILLzlAfl13rvNH0p2fvPLJq5lDhbNK/
|
||||
AmeO6V+xBtzZ3pOFvuLE9UlOPxjiO0kHYVg2D1p4jaPrP0F+KvDd/Dtrb90QwHH26W47jMEXjt6C
|
||||
po/nqGD8xvpDt8vg6z/DRNmAL+MCBHl1Z8rdQ479kwAIBvjh3Oxa7n6sP/XEY9/n1nvtQeOVbfxI
|
||||
52sOgrjLP8aNZyGu5ezfHXY7WL6+A+3PvPzH1vsofgBav4mrt9Tu5gZ46aoX96V3sw68eOPLXJaX
|
||||
uHiBV/BSddOUgZfAha7ntuCP/9dD79C9e/K5fE37KfBnfeeEzdn9Kp+RZyN8Fn0KYR5/7gX/BH5/
|
||||
f/7++4f5+4QVNyB4T8Yn4+/rvz8pvB3wz2z5CXGDIL4nyM3Uzz4/av/bDyc//HuMpvf78cNFgmU1
|
||||
pyeaI/fuBtB3YfW4MyzzFDRzld5Grtcgbl0ve5tVu9aNwEdDxcWXgW+B4r//DHwaI799vCXmWxV8
|
||||
aKJfeu/HQRLH6F8hvzL8fiU/tDGW+mIdltDNPmASY9/7umu/3sG5b9x7t91P+P7b9/8Bf9t5rz8Q
|
||||
AAA=
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f752719988393b1-ORD
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 04:13:35 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '4711'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=ADJyoHaKUvEeC3nK2_sSJ7dr9CEJ2NNMqc9USLXclT4-1778040810.495933-1.0.1.1-uG8A.s4fFg.igFwTcj1WtTLUqjC4PfZvnuYaIcoE7CBZKb_G.0rMV93DAq7s2AeJruAKwYTEjGzqEdIZMlC3NnwS6wHlmWG9sXbbAroBBCjQg2JoSGoYK1dMCFoUIdsl;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
04:43:35 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_c6ffb3ca01ef49e7b02c5b09f6dacbd1
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Solve
|
||||
this puzzle: call db_lookup(''start''), then follow each instruction step by
|
||||
step. Each lookup tells you the next key to use. Compute each step in your head.
|
||||
State only the final integer."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe62a9c8197914677ea9a2424a0","encrypted_content":"gAAAAABp-r_nDEFD_pLd6MEzoOCG8OKekq-Ev8vnbPTkSie4RovCRsX1uArYsm0yKKyAAEYh8tRzZVKUfeKd6HTYEqGrwUeI9Nd7K14uFr0VC8_AEFOX5_RmizhJInjfMLB5xr26SgIbSkBYpELGYzo4H-nlAJVwn_Tgni-Ka5vNjJZsh4-w5wUXQ3NaK4j0bXJYDwtKFlQ5lLzPa4KgzkWJk60HxK847Z5ZVk7WsH9SKDDPSLEsctmXxY5quJk9XUSGN5kR0TPKGscIxTXtzDPHcudCiAjdRGv0g_TIJlIUQ6ofDXWQl8bdpYoqQIbEy3Khqh3ksKoSG71n0UsUw-6d0IeBMQm04VcGiCUhDHaKa3EhFxh5MNgmuVEHDSp4ByweVwK7G4GniKd8NvkauFyROQ2Omqnj_6MKaWccIaQVm5Yw9WU0_PIPFDOKyse7cms2tlyGpFxeDgTgSARsxLaZEZMIWciZT1DazA9LrrD13aCD4ePHWCRJbpbkDH7FM3T2TrT16-G0kCip_OXm9FZZxJvAgUXMbnVEs1SugWBC2nTCfYrLRwXjyzZk35ekzv6LIy7Zigh0t-_RTOWrBkNxA-7jvgCP_YF51cG1no4hTFEHhhcuwYSmMSMMAkcWi0Dh-FXby7k9tjUGMKTjs1RN9NI8tc2lSwmAEdhZdk_JpEnVlfpDEsjRFRIE6RfghF-SRn2gFAUfRczeHKplmtWM0eDKiaK6XKHTPQats0mz1B16alRJOEeFbwMPQA1Myz7atKb8Z5I_IrlGBh1Ho9buaY_RFfCpGemPcO9eRrGoZ7QLwSQH6pw1MrDusZ_pt4vKYC3qDu3rEZ_6OHwFYnRPVj_Qu4bt9f5arKPTKAp2hmEsYKCDvaWeBp2lgEOb66sv34qH6qs-ooCAOG7rmtjSLCVu8F0R11NiFciwvmlmECOwT9d6emgKjn9T4_2CBaUmo9rdEiNfq-cmhMG7a26jGazsIl9Iz-prux7hV8871iYlNDhn_Mr86kkCmknKL53tGb-29XcK-jTUnlLGrHmhnBl06Cf0f-cE7sT6D2tjacDeh9P6gtu0enBfPbuJH75oA240HiKU7gRBd-kYnw6hvhCMKUkjb88Uj51OFOxB0vDLnF4_6y8rswyWoPVeeuRMWHJto_ersRSiuWnzVjJAo4dK9MLfBKRhIth6WDBu6ReL2575QfzUYEFTGXmGPX33OVw0-mgm","summary":[]},{"type":"function_call","call_id":"call_I670mAzR6AYszdoqaI96qg0k","name":"db_lookup","arguments":"{\"key\":
|
||||
\"start\"}"},{"type":"function_call_output","call_id":"call_I670mAzR6AYszdoqaI96qg0k","output":"Begin
|
||||
with the value 7."},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfe83488819788888aaafb2ab1fd","encrypted_content":"gAAAAABp-r_qaKH8G-A088KPm_AKkVfkzrE_w9jGidMqLh3fXoaw9IbtAS9DwhU_OCnhgjF9tD3K66xmrV4gVvDX5mfjjsF4A8rr9sdYPtMw1TmcwMntBVKvJskKjpjZe6s5dswnzCzuUzuG6UOKpRpoZpnmCIew5Sm-ZoxY0PjtXYj7_YdzkzXmb8M1nbFyymgIloaDarelYr71F-wysy_BIxPAfFulc7CeT_jGY6qTj-XaXEjNevpAkRxQs3qDFsf1vJJJSMRUu5vHFO-kmQUvC1C7nNSkK-AQxF4ghx2I_cGJNSHusnjGq5Ia397H_0CBOQjTd1HJIu3UWG8guMSjH9m2elLMFUpfExvbR4iK-l-ewCcWpIyfWxdLWrD3IWqmQ5Id8nEIJFK78mj4VmLPw8JOPkArncScgXqTK6uzkzEX2HtsNECxFR5w1eVZS1yixE6OSj4V5z-DDlxxx0ToibA1J9qcs5QKR-0ULx_LK_Pgf9AeK8_Z2FqHNnFh5nKVWdyIYXxTpurpF1qEnKBCSCwaQ7XzUJlJ-n1UFgZt0URFbsbqY3XshC4urrvqswPq1PLBqhQcZxPbsXqMKzFuwT6J6CEl2GrUoCjHbVDqIh1j6lc5BDYi7aOAYn7bEwQI_thfmQbi7EraVZMmgbEnvYkxvzeP78ASC4YiYwgIzGuUQKzwqRKKbxpe6oeVE79FCeXhEbKT2Z0kcBZSUomxfSCiLtNMC0Ni9WFfUOu94lRtTPdPH8pVu-QBFezi8GfxOXHNqNuFSVI_VQtenIoP4ia1eYxbI2hpwAlk3u8dUSCvJySdLLw9dPWNQxco51zDamyQVKenbnEbj4ZqZRKl-2t_neG0AcX5qliCqDA4mqmL74yirrmYt69kIFQ6SGCP-86km3amgAInOsOBVSQK--HaOvBUfTc-CWLOONqfj2aXUiMCKvSMvo9-6Fug4CsIXdcgxqOQYGfc5Opce8mTUdrSjmhb1WiMO6lk0LSrHq0IUsIh8Dn9X5bhXshn8dC5wl7Ya7Bofyg2EBGTo4Sc7i-nf2BfFX7kqTmOURGa4iZe8hNpuoytgVDlbjylf0NyU-GE0gwubFgmb6Dqn7UdJedE_i4DKIctupippecmwpgO9V1nilTlYoNYjrGryjTRZ6AEj8oaeR2R3G41wUpMHv6uUyYTIKl1l2cAd6cG0g5A8ECg9bQfIUcAAcQN3EdfoIoOXOEqCM-mCEjjP9wQcx4ztNg0_o_TT6CqZgPKDzzW85UgVDgtWGhU464SPEUoYxJrCpWixqQ_fTANdF4snP3hRG47yauIenFdO31zuafOJrcqEDycKflDHvA5YUOiK1iOoDK4abQhZBePmpWyOFf8lufsULGoetqiLG4VM5hdijIqkIOWidiKSBMxX8X75bAeT5uS3nCq6nS0jRLTKJ7LL9qH60med-9GfzMxKXnMHi0yewgVmQOgwir4rs8o2utjshbrOeDz4-Z-4pCskwZqdEqY0Q==","summary":[]},{"type":"function_call","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","name":"db_lookup","arguments":"{\"key\":
|
||||
\"7\"}"},{"type":"function_call_output","call_id":"call_Uj506iEKjBZWrHPq2C3GDtgr","output":"unknown
|
||||
key"},{"type":"reasoning","id":"rs_0429c1fcf5cbfa350169fabfeb569481978703af8553a073b9","encrypted_content":"gAAAAABp-r_vFGcSWQD083z5Jsq67CmK99_ZqqnwC8n6j4kVfWhNH5J9pJw5e7uV1QurddjHY1w7BdbV4HrfTkz4SUHio6aFeHMUrbQObDiRwP8kHX2JSY3m3wXaXABn5_qJi85Lj7CD6X73jzqdQpEeGQ-5zL3JsxO0k5UuZb01t5EuzuGzw_0LR1Tli0UJw43rKgCEqarHTus3l63d18pXuK5RMBR-3Z_xI93rKewxrkTaH1DBgxtlxtD5Z4f-d0-RbfG14Svq-QS3wsfg5Xrn85KJHwf8yJ55M752AxBArv_iqsV1YuKZAwyFxT0HiLrDOwI_G_QyYfD-0U-HsVGeESCS-g4jwDPDXst2aJPTOyhtTu0DbpXyU_Ukbrtlws4OOZurl-W7LvvXlWJNG0WWj6mpCcHrWMzubGkQSH87c-GXESHoBdxxReXGomDs4gqnbB7dYpgNRNveYzLnJRQdDYkdYHTJI7gBEVNFBw5wY0zXRhYtivh_G4NuL2FYcTPr8zFsxqIUXyLWJ2-95N0jvS_ap4R0H_l5YY8twKvT4stYA39T2PKt067Cl906VXoatJWjW9yfLjn1fIe2m3Wr4VXhkWUm7v7A5x5Q7UObihtAn-5NhoU9BK1FcMEY_0UmTclUAM-SFVrL3Dtf9GtoYoLH8PUb4Sdph9y0D6baVSNnBpydV18l9XtgruSH5p0t9cexYA3zzTAmMb9ujFxYAb_qWZ_BFSFnJo77lYCoiKWeNeEMl3YoW4tXqmosdpqlSFpHKrax-YGeMD1nagNqQFInSLegcESqI8l-Gyqws4X3VGxW_jPGJgjh_wZAWQTKMiKctcA-AP5JOpJxaPzZghRMGpdI73nOOTV_jQ9OVkKKWtjFeMuduFhWLtiichsWWSq8XwJDUKTRyY_LYBxzPFFxIhxkqQ27b_ndJPLvyDT9ATvN856mxSJYelK0uJjxqj13O5ZWICp0pdUGiELg00Q15UIBqllDQv71_Ltnk_oSEprXRlbifFswF8TNVFbvHh5ZF_ajeXNbSgMfs_qSt9HuZD2tybsSsYHqmitpar0udcgNKvyiPKPXab3kwiPYIj_7nvclcqCZUUzddMy5dA18jszEcY5N46eUV_NtZGO7hQ1Z5hSWrmPSeTUHFKg82QfmW8_ejb0y9qfPoxQcTf8XzVd7gaNDkjSoMC-5Fvbz__geDpkRbNqSPs9eFE5RK7TqkcB7PN2qNN-m7jnxgTaMf5zucA0Xj2SJjoTcDCNS22d_qikzo-jzOuiJD7SZzfST4ugJlf5bMUGIiOItD4e_jszixhZHWE8GHHdjFnplasuI2TUwjUSGvgWr66ELWmpwQaDya6bU2t_hrHQWhkiatXJYzizyI3lUGzf-6N1_ieunEEquLrpVad4LDLnYGI5obFQUJabrD-eQI-JfKRKGSlRBtgXCXtiTJg744kJv5VJsgcKyk4xjoeyAwWAL1mw0NKt0XOeJLdWGIlo7_WrVpFNtSngYnkAD7yXyWsoPbcpBpk_j32YT5aZ6NTeyT5jH_XDcu-pq6kF9sIdP6Ry0qSG0XEyz0zXpmUbL0vowzyNWPoKXdsWYVLXAc4IF-E_Ar_mzsMJkB_3Qalb2sT2dmW7UkaXO_Ej2PpA9aNW-qcV4q5st81qbQnW_JHvai0WzKEFimpEf70oa88Z7ARexPicizAGu-1xGTCZM8GqdtwurLiGOn8Wn9ojASn7zUEF_xTw9UcjB94jQwc_PKHY6BPBleKKPYTZWxAJwzLt937JwezczGHKfEDdM_DMVDI_vJ6H-mJ5nu8hAJK6UemD-GE4VSeSP_NtGZXXinevdm2k8HN7-KH6-zhwP8yMzw4NBnwCCdxvuxkTdhtp-69lY88c-_m6xyvkrpDkUDtsQDk6OjBYqe6b94tKnQh9I9FoVOP1-p4wNJNDxK2AjCTsEZIzD-g1ccSL2ZDtN9CmsXAfecl4RKzdyJgvHXbfFz1BRFaudrlk79C5khqva-WLrFfxkl6y9UE-12CFPt779pulpoeSXbR6PW0Y_mZXpFvtPH6N4Sryi3iYObDtxzgdhd6CNdrXovQstWULzuBS76EYyFPQ=","summary":[]},{"type":"function_call","call_id":"call_Vik9mkNli8KsnSbHOShapdmG","name":"db_lookup","arguments":"{\"key\":
|
||||
\"step1\"}"},{"type":"function_call_output","call_id":"call_Vik9mkNli8KsnSbHOShapdmG","output":"unknown
|
||||
key"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"db_lookup","description":"Look
|
||||
up a value by key in the puzzle database.","parameters":{"properties":{"key":{"type":"string"}},"required":["key"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '6407'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAA/7VXWY+byhJ+z6+w/BxH7MuR7oONd7BjG29wEqGmafbN7Caa/34BZ2xPltHVlWYe
|
||||
RqaKqq7+auGrH596vb5j9P/p9ROUxhpGETzETWjSUDcBSWM4w5tAN5HJMDjG4TwLOMqkSBIwDNJx
|
||||
CsL+59ZFpLsIZq9uojBFNzlMEMiQoYFWh7Msh1GNF7rTpRnI8rS1gVEQ+6h572akA+hZSZSHbVwm
|
||||
8FN0Ezu+74RWI/vRPDaCGFxR0tobqEB+FDcPjeLldvCry1+OZjstSpKotQxz3+8EZoIuOQrhVYtR
|
||||
CPzs2iixL1inc8JXZ5qBMuD46bOlE6ZZksPMaS79LA9ApUV5FueZlkUe+l2ZRZGvQeC/dRdEBvLb
|
||||
O1lxNqC/0AMCI5gBRg0Isn9/IQHtcc9mt5Mayb8dNDeAHqlN30msybG3xPIkADrCCJo1MAbQ3XGd
|
||||
k+wao1tqQRqFbQruqgay5Bq3MMMozFDYlYA1bP9G8SDRqmCIHSplk5W25lKYvwLbQBAuziWdNDmj
|
||||
vXg1on0RwAk6bVezwep4Hg/jhLVGMuPO1dV0PoiTNXvNRTgKx5zmE9VptZueRkeQVIPSkfFJ8HW9
|
||||
s5ZRKa+Xu5QUrQ03xwJ5dZlqijG18+mR2YOpcbamJm2iyW5EoulyI2yDFbPEw+RgR5PhCtF4eE2C
|
||||
mObkmosJcWhuZ6Gqzom5V1nn0UUAZcYsM5uChSmnx42lXtyRGJKqlpbw6gWKEegbejY6wYXnRNHc
|
||||
kJfqWYsDGRvUvspvKn2yGB/WsrQnYnG8Z3CqRvlFGcCcjmrdY1VKOCx5t9rUB+WcmwUQ1l6lMMyU
|
||||
5mCoZdh5W8acJx2yo30aDddidNnALbaXY++g6xeQz3OZzm3PmB3kwtIz63w4qfYQDGOg18l0NKOk
|
||||
6VlCcyxW1qeKitNks6cPXLCeHpWcXR4UHJVHiXd1e5f6Kn1ZDeURWU0p5utYwdwolFYckgZT+QLx
|
||||
C1Geh5qxGe1ktNZ4MjxuvWNQm2LuKIEYFVitnA6B6IhSgfnzen0dOrCYhYUzPJMGPpsnTq5FohCv
|
||||
dbkSJNbRgOBV+m6QoEKZzpOgWFUlT9hcAt1FMtsdancoT7jscNgPHaygLz7NHufBaO4UFEXklOib
|
||||
9jkWJ/MQSu5xuCk346OtZzhb4yJuX/e26ij78AICT+FOpDjYLCR5qs2AbJJHfOCWy9FmNxLgboFd
|
||||
pXx+srV9GlkupSNF59fFhrC8oqykpWdNdqY0XM3MAbnFF6sa5ppURO5lRwrEAZKSRM9qM9Uoa2Kp
|
||||
B9/kLtXBsOcInKpgyw3m/rYQ9v7MHawLd5IzYYIUcgJlLJZ0Fk0FPjB4b7rNFJUe1y7ciOaZZvF0
|
||||
eD3EWlAHy8lkKm3nllidwvJUjq76/CCpZooBIO7H2KR2YP7VqFUPKSZeHQTXdD3JH9i7VVktzDW1
|
||||
UIxtUMbCUZgGPlosBmKd5Bk/3qmhBxxv5wzT83aO02pQUu5xcXHxGb8/sV45qqvYTN1MbSbC5Dw4
|
||||
uXt/d6UENr5iuzGbqNnpxEjjNVAxvlj4yn8eYyHNgwAk7RD993sne/n8p7lkwr/OJROnWZ5r5xJP
|
||||
cibOAQ4ShgkgwH+fS2YedvO3m6ZPQfzl+9IpQWLlQTOvOv2Pb30PXb/1//nW15HlhN/6L483W6fa
|
||||
Ld7u50k/lWJJLZnJecvKIU+aVpCriHxYhCDo4jJ0zY8iL4/7/z8GALYY6BgweArAVtx8PumPxaAA
|
||||
fo7exQBlpJ2uIe7ErmDmo6sfDMuLN7l+DAY61WEAyLYMaBLqBKnDD8Ygs1Gvw6HHvovEkkrijYxt
|
||||
JzUlLIz0OBMn4ozxPqga9K4aeJpjSZMmWd6EkGbMj0Vi1HZEr3Qyu/cEypd3UbEMAioKg4veQpCt
|
||||
EXNZG6wqMMzHoAK7+uChTnK0qeuESeE8gX0sKo1RkhHvghAuQnEYGOpXXYkvkPy6I09CA8n/UBrN
|
||||
/+8drYtB0nhC/luy2PDNGyOOG7LdMDD0B9LaqAonylPtlY/fIrsTxjhp7po1LqGNtOZSf9UlqKV1
|
||||
N7rZJyj7xkIfVPDOxhsmGSUd/7Mdy/55yacvQev+Ts9TYKLs2sTU+jYd9IaMpygpnOZWmfNK702Q
|
||||
+1n/59YQJejNXpChIG4pcd7J8Z8AZKjKHsE1oQXg8fxUFN17z7XXL1CiR6nTwdkPkOHkwWOvuCXC
|
||||
jpr4WjXIs6h/V6S/E/Bfa++RfAOlMHHiV2ilpgB6edwDPztMv/aatPSa1mu7Ls7r2kc9A2RAByn6
|
||||
8l4N3VVt9QRN+pL06eK3/DaANei+lTeaWyE8i56u0Ow5bcKflC/33y+fH+7bVcpJkHEH48n5/fn7
|
||||
k8HrAT+XyCcNMAynBQj4m+eYu9x/+uXkLr5uB23745dGyqK4gcdqbq63DrC7MO56hudugqTJ0utu
|
||||
1TecFOj+61Kap8BCj4JywjebHU0Sn39XPO2LPx5Toukq42GJvam9XzdGHOf/pPmT43tLPlmzb5xn
|
||||
UQb8h5ah8XtZ5+nbFmzKBrTF1h7w8unlvzbc0z8nEAAA
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f752737fd3c0044-ORD
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 04:13:37 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '2162'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=_EXrqe7IjNWn6uSESYNRV6be_iYv5vfdiSH8V3vdNgs-1778040815.3529754-1.0.1.1-qv4Kk81buc06G3o0AUYL254nHY_tKoM.r4ToKt1atzCBnU7vDOrMSgZt87H9a__SsX0twv_Dqbwnv35sedBqkSVJ5tcTI3rPQI4rOXZ2o7bFUr43uDG7zWB_uR4Rv0xw;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
04:43:37 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_83b594dde5454daa8a186b3a26e8df84
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Pick
|
||||
a clever country name, look up its population, then check whether it can have
|
||||
dragons. Be brief."}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"lookup_population","description":"Returns
|
||||
the current population of the specified fictional country.","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}},{"type":"function","name":"can_have_dragons","description":"Returns
|
||||
True if the specified population can have dragons.","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '722'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAA/71XWZOiyhJ+n1/R4fNwApD1vLmhICIqrmcmiAIKRLZiKRUm+r9fwG61e3oi7hJx
|
||||
fTAks3Kp/NL8kl/fXl46vtP5+6WTwRyZpMsIpGjRPMORXZZySIoTXWDRFO9YXUagRM4SBbLbpSyW
|
||||
Z1yGtbnO98ZFYp2gXby7SeIc3uR2BkEBHRM0Oorna1ue5qlWlxegwHljYycRCmF97mZkATvwsgTH
|
||||
TV4uCHN4E/th6MdeLftVP9YCBEqYNfYOPMMwQfVDrXi9BX53+Sk002phliWNZYzDsBW4GUwxjO3S
|
||||
RDAGYVHWSvIvstX58bsz04EF8MP82dKP8yLDduHXl36WR+BqJrhAuDCLJIC/K4skCU0bhB/dRYkD
|
||||
w+ZOHioI9i+WoEmaI0iGoLud+4EMNOGezW6Rask/bWluBXpAm/8ZWEHkHbsBVuRIKHAC6Vo8DQTh
|
||||
hkXrpCgRvEEL8iRuILir6pJlJWrKbCdxAeO2Bbxe8+kjIpOBBrltEBuzWB8YW06R9+t5ni0sbX02
|
||||
gpEZbcg1QKP1hdFJlkulrrgBVzQ4GWMJDa04PRwZbFu5gayRvL86Z4IgWG8ZBZUHT0LXYUdld1EE
|
||||
qrQnSo4ieR0rZ6wOh/mih1fq3jP8rjII0zxBWlAxhaAMVuVAFzyDG9mZWAWrfDTTJolSTg8CvZwY
|
||||
ispr5arAqevK/mouKT6xEbeStcgnV2QtSWuUrOH4pKygI3CcaOpbKVIFo/R6R0nDVzI7L3ejSCMT
|
||||
cb2wgZkJfkokRmH39+RMt/gySgvqgCfunqeWWo514miJs3nXlLtyts1ExdDYfLmQVYK9+Ny+HCn0
|
||||
DBnVGYPteoO7YD4OxghWrBsEG2dJc0FxusrustiQZ6WSNgItCodetNEjOln23TzzTAvtmaWo2Zy6
|
||||
mSJigLfm8mrsaLHf3RCTqZoMp9HZj7dLyByG4s53Np7pLqI9POdaJqiL7kTDO1kdokWyE2Lblvub
|
||||
FR6a9my4WJUWkwuUvhghJw4de7qiMNCwtpH2/aB/1aeYy6mxxpKJdLog3fLMgKTigb5zLrsZmSpL
|
||||
ja+jKMZQVK14zc7OkDxq+lwxB2Sw5emUkg/eKZ76cRisnD2pDFfkvqf0qjklnibqfOcUVUAX/Q07
|
||||
73Nj06MDow8inouW2mYwG1aD3Yq4CvZoA9lpNOyFmryohHBb9WVH2gUkWfbpmW1soozmablUgqWm
|
||||
pjTAWJLmYbDexfR6xubdPvLn+X4Es52Ap5fJsOfLe9mE3rXq2ds5F7NuNZISdLACZ76YHqzrwoWX
|
||||
/gEcJ0MmWJ8YYcLBKEkm5algRHarbk0C9wqXmCribNdjexsc9qsNbywoiSt3l9UIYEtaoQpNQaFH
|
||||
PdHr9tM4To18wkCR7LOKTKuXPOPZ9TCaKWhywaPpXBHOVSz6UARzW3Tko3GaHt3JKPFywvL7Vjo1
|
||||
lHhZDbB2PBxT54DFrehmg+mxIhmtOqPo4DuiOMp54cRN5olzFgZ6PAvKmDXjA9EbTqTzAfqAzRNp
|
||||
oxKMsL+sp0XFBCk3JSR5jC7d5YUcRr1uFYKRM+aOy3kGrEDXS0/qn2bEXPZFchGd8DUsNoeejPJq
|
||||
e1pn6Ulxk5gliMw38DwS0xOp48swJR01NQp/tIl5wmGAkgnmVFxkzhipO3xSK10NUDaGe2kMpGPB
|
||||
cpQ2WZbcmgtjIXaP3bMvkAfE296unpj97nAUT2RtRB/Y6JypYhBg+xJoFwsqRDqX9cE+jHS+2oVD
|
||||
0Ys4czO7hhREU1Zbn8qBHYmz8XhGq7RCQdeezi6LvqtpHEJHTE3plXdxPVJCepTq3o4I+2eKTmfO
|
||||
cjCbBnNjT/bUuSyfWE0Xd9UBu+yWG85PScmp14Ww6suDbjq8rIEgY1afCqMg8LRgi6Gh6BUy9scT
|
||||
WKZ4b/aFx6jNcRSBrCGmf362stfvX8161/7zrAddxiVbEqcYm7YcngI0bTsc+H3WuzhuOa1lqKck
|
||||
/sDZrRJkHo5qDmj1v37UR3BcZOWPzt8/OnrN5kkGfnReH+cb1+Yt6/YnLvnC1TZJIGu8dpIkLuDn
|
||||
hVo+hY9B1GYXJkmAkYkShMMbF94qUn//bPkQgaz2CMOPLFsT9W2VQPWWUlMX/ILta9XZT3Buvi8y
|
||||
twzvTIuy+uJF7dI+QjOA5R91GWz48MbTHZo53uj7waH3NQa6bpK1xHn0vePbZZ/gbtzf95ocuLAo
|
||||
65wa364PP2wxOczOfn2rwn/fi1yAw6Lztm4lGfywUBUwQs0ugVs59VaAAl6LR3J1ahF4PD91SHvu
|
||||
uRE7Z5hZSe635exE0PFx9FjIbkAckzq/Rg1wkXTuivz3zeVzIz6awIG5nfnovbRLWF8gzl+KI3yx
|
||||
cZbVhXl5NMZL4raaHEG7qZfz4vqtPxC+vPXnX/9Of92PNJ0V1dBm+VNRbtjXxawr/1HerqJtmE/i
|
||||
pyvWC2TTEE/K1/vv1++PEM2O6mfQuRfrU4C77OeT0XuQtw39SQMcx79VQn/Ove2Pb5+itzm2C37z
|
||||
H/py/PzHgBm1pxf/MzxP0NkgfjmCM3xxMuDVf8UvgKqPmM0R8+3I/4LTE9p/hMqvl1zv9p7x32H1
|
||||
eWT9/+C6z8YiQWaYeHUhrMYBeReidgyKwk2Q1Ti+V6Pj+DmwwvcXNJwDDz5mhB9/eMsRme+/y59e
|
||||
nX495n89J52HIflhmnx+eRLYrxRf+b3P2Icx1/3gu0gKED60FC/e5xTOP87UuoeAAwrQBHj99vov
|
||||
v/3TYzEPAAA=
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f74d0b45806549d-ORD
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 03:14:34 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '2715'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=JaAjlN.SsTHssMfEA38GX9HitAbY0eJ2RFPOYXHE.dM-1778037271.7367537-1.0.1.1-ITxNEO9g5Dnpe0xysdsQx6Pb4UfPCLUH47.8gv2jcppI9YSzDXqxNV1wVH1S0DOjPhkjvEoameT18PZ3jubWSiQeRsWVcFTBf8rKD.riIQ5F_txQnQ76olVhtIu0ZumI;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
03:44:34 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_83a358e390da43f0af03fb3186d6e121
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Pick
|
||||
a clever country name, look up its population, then check whether it can have
|
||||
dragons. Be brief."},{"type":"reasoning","id":"rs_0f4809b27460351d0169fab21897dc8196960e8680fb72a88d","encrypted_content":"gAAAAABp-rIaNe6WknTMnPCTW6JIYUOsrQbNUvTkE_mV0UapEUw4P056qF39VaxpCjTGFpDbnqZh4ucbsTpbEIYxdv---5gRmkzgej83d5Ey3QtkLFY-y6107PuJvuLDDsQAuSLYgTi3JClqsopNkz4t8JCSyCP8gT6Ecr9zkSsEMNHoJyKZ82RHTJL7NyStuqffIiSOFJi-V9WFbQsHxpbR0bEoUeGjJSed8669_PWFmL8TygAhFNux0rvRXEmN0o9UQca_r8iq-oTtcBY0MPb7ymqt1ZuHfY71RNsuP-hb9MO3_I3IrWr9JTN5sRQIL-5wi6YyEJ2MpTzvuaWUVu3aOGkGpez5fkkVdR26ktjxIfRtV0vJzFV8298ZAmVPm2oRBfsrg_bpY4R9Nc6LVKp-CuW_RxTX29B3V-HKLoDKmvinWRe4ZD9XidVg_fQmYevsNr8LQ3HNuXILDpQoX8nccIBVSuD_cMDQSyb4s81PQEpdnldcKS1uaNuNVFYBkBxPKu6s1GN50oFjwpPbg_k01nCPXdwXM0qJRN7evsJTD9LbnU5Mve0hNPOJ_C0kW72q1IZgjnKinlkSdY0JDS0YAJAzO19jHLOXdtzk2tBV5OB6G_g2kTBam76mRNVCMDzCXS-x8cEVe5KmDAlNIQz8lWzBIdFXk00yB2McTVmr272IyJkRNLq2auuFFOlkUXn2UM5s3BpiOsYEerX8uKwHDAiIYI_egxzAcWO6n5fzEFopZbkdOQKZbxQfewBZahHD4kUj48H6emooHyjt495WLW_-uAtf-KJ9MXA5AVulBzV7TQ1F6yXwSEaubFSpzpKatPmA9g3BqnnqTsH4e90B5JI2Lwsr75UDmMJpHwuEKOJ8vzn9ie9aOc9dIhTjKhfHEogs-biBbqKTJnRzCuNhZhqdZu9W9frCKhz04NzvpmZid99Es78j6HOodv8CPnMkyn5_nZ-ADHFvZeia5soFVL-48YwUKtz4kq6K-FIGpw3Rw0DmA3zlaEdG6hROrabkPPygFBjM-OIi90QmjuxltVZAIpszWjUrqjJfon5--riTuOm9qj0PuwDq0dLqTtiEVn7-d4aJr8_K9QrdGpLXujLzPLkprGeYFGaFht561NHRy6U6ln8nfh3vi80Zp7cgX4-2B3DEnHINE2Z5mvrL9kkucwkNwbeJ-qOIPCYlmP7zXlD9gm6_VMxl1epK5NUjyCcm9MGGM2L2J1efcKMwQBfNN6pphu1K2Sgwfg0FpPmqPgX-lBv12qMdRCMKkOTY0ALOIIj5NP9XzZuf5W6DOjoy6LxQ8SBIC3qDwUa8Iu5PK8EkkgNkWueTJPzpTYhjaRquY_B8","summary":[]},{"type":"function_call","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","name":"lookup_population","arguments":"{\"country\":
|
||||
\"Pundora\"}"},{"type":"function_call_output","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","output":"123124"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"lookup_population","description":"Returns
|
||||
the current population of the specified fictional country.","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}},{"type":"function","name":"can_have_dragons","description":"Returns
|
||||
True if the specified population can have dragons.","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '2423'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAA/71VS4/jNgy+z68wfJ4Z2M7Te+t5L9ttgW7bXRiyRTuayJKqR7rBIP+9lBS/MhOg
|
||||
2AK9BDE/8SPFj6ReH5IkZTT9kKQajKqydr3PyrrYrbfZapPTLN+WLamLnLTbItvn5bZe53S/o7SE
|
||||
ksKmLdNHTyHrF2jsQCOFgWhvNBALtCIey3e7fbbaIXnAjCXWGe/TyF5xwHPRqSbNsdPSCZ9XS7iB
|
||||
aGacM9Gh7RU/0aDIGbT3p3ACLhV+IHCJgQfKm9CbgILW0nsKx3kwtBr+ciCac6VAEG7PCGbPWcCY
|
||||
GMgqCpYwbuaeTBirXWMZXnpu78n3SjqrnK2sPMJb0ErJq4bwJV0vKXB/p07Zp83z5qnIiu1Ttn4q
|
||||
Vul4QBMfbu4WI6Hlz1CaWKBR2ra5L2zdwH4fhM3qVVm3BdB8vVuvorCBxJ4VBBonwj1D1hN8T8cA
|
||||
Et25HoQN+OvXVEnleEj/a/ohL1Z5sb5Mpz1xFXMOf1/+/n1/5Py3n4yQv3yxL1/++Hnz8fOWTR6C
|
||||
9BCPi+pATiiRJp2XIhy44O+3UCBFNBICX5YdlYu9pbBtUX54R36ETkw6Uw2dHRMcS6803toiZXOA
|
||||
6gjnu5jGyoircGmxPkQ9cUCMFIu+hraVOgzTgXWH611T4/qe6IF+bHRDWrBnzMlztwwWbW1Anxje
|
||||
yrJhUFriuE2v8yc1LCbMQq98c7lgz68FsPDdTslhaj2ZvmftEc7Fsl9TPoGupWGhnGkPlLl+mtAo
|
||||
xEFifh4mzsp0BMzbVr7twqkHKJhGMzWU9jPgBYRJ7AGSxmmNhUmmtktkGxCjoPH1oknLAh/hSYM7
|
||||
x+rz89v24lIenaommumI76wepdVmVpSoPRYTK7+0h90UwtyYZ1fEjeIbYgZexv+XxymEX1pMAx2L
|
||||
dRNgtH2bOQ1Brit7hhBKWazEp3nuoT8ebqKHHMPG9zM0V/3HBfsVmRJ2K89MOhzyxA95ch3y53+x
|
||||
B/6DTjO170rFhIUuPjw/ptUsyP8s17gbrVQVlx0WovYE2WhUYQ2W+2jQqONQjZQyQ2o+vNjOkA6m
|
||||
HcHE4tnLy+LxLTB7TF+n/Y+Lkk6e2WKd3D6nRf4e8B7vuGTvUVtpCZ8zr8Y95cxyp2IPEUos8fyX
|
||||
h8s/3J7RqkIJAAA=
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f74d0c81c4d2b4b-ORD
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 03:14:36 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '1077'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=Ff7ZRSpcLAJPDQ.asF9RF4bqWtg97VpC0E51yPz81.U-1778037274.8993971-1.0.1.1-lFBHWH1sK41g3Of_MJ292u9sow8m9djH6vBQSla3pNCvWDtdVwVDdW.ko4QJQaLNA6RhV21md8copjjSWmsVnZJzyKsQgS.IPimoDzj3d9Lt6eQuV2eXHZenOh25fKxo;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
03:44:36 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_477366fddd32402daba00618a4c16efd
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"Pick
|
||||
a clever country name, look up its population, then check whether it can have
|
||||
dragons. Be brief."},{"type":"reasoning","id":"rs_0f4809b27460351d0169fab21897dc8196960e8680fb72a88d","encrypted_content":"gAAAAABp-rIaNe6WknTMnPCTW6JIYUOsrQbNUvTkE_mV0UapEUw4P056qF39VaxpCjTGFpDbnqZh4ucbsTpbEIYxdv---5gRmkzgej83d5Ey3QtkLFY-y6107PuJvuLDDsQAuSLYgTi3JClqsopNkz4t8JCSyCP8gT6Ecr9zkSsEMNHoJyKZ82RHTJL7NyStuqffIiSOFJi-V9WFbQsHxpbR0bEoUeGjJSed8669_PWFmL8TygAhFNux0rvRXEmN0o9UQca_r8iq-oTtcBY0MPb7ymqt1ZuHfY71RNsuP-hb9MO3_I3IrWr9JTN5sRQIL-5wi6YyEJ2MpTzvuaWUVu3aOGkGpez5fkkVdR26ktjxIfRtV0vJzFV8298ZAmVPm2oRBfsrg_bpY4R9Nc6LVKp-CuW_RxTX29B3V-HKLoDKmvinWRe4ZD9XidVg_fQmYevsNr8LQ3HNuXILDpQoX8nccIBVSuD_cMDQSyb4s81PQEpdnldcKS1uaNuNVFYBkBxPKu6s1GN50oFjwpPbg_k01nCPXdwXM0qJRN7evsJTD9LbnU5Mve0hNPOJ_C0kW72q1IZgjnKinlkSdY0JDS0YAJAzO19jHLOXdtzk2tBV5OB6G_g2kTBam76mRNVCMDzCXS-x8cEVe5KmDAlNIQz8lWzBIdFXk00yB2McTVmr272IyJkRNLq2auuFFOlkUXn2UM5s3BpiOsYEerX8uKwHDAiIYI_egxzAcWO6n5fzEFopZbkdOQKZbxQfewBZahHD4kUj48H6emooHyjt495WLW_-uAtf-KJ9MXA5AVulBzV7TQ1F6yXwSEaubFSpzpKatPmA9g3BqnnqTsH4e90B5JI2Lwsr75UDmMJpHwuEKOJ8vzn9ie9aOc9dIhTjKhfHEogs-biBbqKTJnRzCuNhZhqdZu9W9frCKhz04NzvpmZid99Es78j6HOodv8CPnMkyn5_nZ-ADHFvZeia5soFVL-48YwUKtz4kq6K-FIGpw3Rw0DmA3zlaEdG6hROrabkPPygFBjM-OIi90QmjuxltVZAIpszWjUrqjJfon5--riTuOm9qj0PuwDq0dLqTtiEVn7-d4aJr8_K9QrdGpLXujLzPLkprGeYFGaFht561NHRy6U6ln8nfh3vi80Zp7cgX4-2B3DEnHINE2Z5mvrL9kkucwkNwbeJ-qOIPCYlmP7zXlD9gm6_VMxl1epK5NUjyCcm9MGGM2L2J1efcKMwQBfNN6pphu1K2Sgwfg0FpPmqPgX-lBv12qMdRCMKkOTY0ALOIIj5NP9XzZuf5W6DOjoy6LxQ8SBIC3qDwUa8Iu5PK8EkkgNkWueTJPzpTYhjaRquY_B8","summary":[]},{"type":"function_call","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","name":"lookup_population","arguments":"{\"country\":
|
||||
\"Pundora\"}"},{"type":"function_call_output","call_id":"call_uy7tfNVokIN7NjFF6k7OtLyl","output":"123124"},{"type":"function_call","call_id":"call_jwY8kllWAsnoSXtjXZQ5KR6i","name":"can_have_dragons","arguments":"{\"population\":
|
||||
123124}"},{"type":"function_call_output","call_id":"call_jwY8kllWAsnoSXtjXZQ5KR6i","output":"true"}],"model":"gpt-5.5","reasoning":{"effort":"high"},"store":false,"stream":false,"tools":[{"type":"function","name":"lookup_population","description":"Returns
|
||||
the current population of the specified fictional country.","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}},{"type":"function","name":"can_have_dragons","description":"Returns
|
||||
True if the specified population can have dragons.","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '2645'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAA/71Wy47bOBC8z1cQOs8MJFm27P2KINhbNhAosWVzhyIZPpwYgf89TcqSKHtmDwmw
|
||||
N6uLrCa7qpv++URIxln2F8kMWN3kfbXPD21ZV7t8sy1YXuwOPW3LoisZq/bFYXfYtvuS9mULXdHW
|
||||
fZ49BwrV/gudm2iUtDDGOwPUAWtowIq63uebuqx3EbOOOm/Dnk4NWgCuGze1tHs7GuVlOFdPhYUx
|
||||
zIXg8oixn/iJAU0vYMJ+BmcQSuMHAtcx8UR5l7qOKBijwk7phYiB3sA3D7K7NBokFe6CYP6aR4zL
|
||||
iaxh4CgXNt3JpXXGd47jpdP4QH80yjvtXePUGzyCTinRdFSs6QbFQIQ7HbV72b5uX8q83L3k1Uu5
|
||||
yeYFhoZ06bYxE0a+xNKMBZqlHezxP5Rl9SYPyrZ1Ve9hAzsGHc1bGvNFFnfREHnAWnqEBfhIwgh2
|
||||
SjqQy6HSg61opzLBDzfvjguolMrRqbRfvq5AoY7aqPYdJBIh7yd0kDKUnKgllGilvYhkRPWkKDfP
|
||||
RVmRf3yZFxWhkpEL2GfCHemoxC1nIMzQI6Z+zWby6+3XnC/TSB4v0XP0TUOl/Y42nGGjRESptRyL
|
||||
hdV4mngiB3rYoAVArO2Ajho9r7Gd0Jbwji0ROnPlbTN1XBPFni2BxRm0Q8ruBM0bXD7EDASZRkNl
|
||||
ZXUafYaNa5Vc9Rv0vTKxsid+PN0umVk/DNRM9HMDWtqDu+CZAnfPYdVuFsyZ460cnxq4p16M6qOp
|
||||
lIFV5zsYdDC9j/HiVoCbzLfD4dEGunwn9orrxrLfjnwG0yrLYznR1Iz7YZkcoxAnheeLynmnshmw
|
||||
jy02pem9jFNgEZ+B7QzXU2k/A15AWuJOQDpvDBbmzpQBsRq6UC9Geh75qCAdzkJnLq8LtaRDTCqU
|
||||
evO6WWiWJcFZA0prbFKUUXssJlZ+HY8NG9PchZMr4qQLhkjA69IaSwtmYZhyA2zV+kmCOZb07TIN
|
||||
xqckQShjfKzEp/Ts0R9Pd9njGeNLFHooVf33BfsbmQi/lyeR7nFiPAiFS5qwpLkt+ROdErU/lIrj
|
||||
7D2OD+LvaZUk+Z/lmmejU7pJpnw+B3Ucg4f9GDCo41SNjHFLWzH9k/DxuZpnBJer57gs6+dHIHnk
|
||||
Z8vEQcmWnflqnNw/82X1HvAe7zxkP6J2+P6JhHlbzHPK2/VMRQ9RRh0N/Nen6y8ML4eS2gkAAA==
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f74d0cfcf8b2b4b-ORD
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 03:14:37 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '1306'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=QSHgROUcoFXh6zyyvGCnOLGXX2c4b7S8nJowB4RJhtI-1778037276.1270735-1.0.1.1-m3I6gMTwPPLlpeHgduEBRKmVIQGDDZcoItTTaVkwIUQXMOTcCG5_fl8MQIwjyawnDJPGxUthuGFfjTpsPX1OSBOXGmAUziyOhJ04qWwyUzM7fMtNEWtTmOgvh4au55D0;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
03:44:37 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_e20848e56750417e9dd0940f7c4fc82e
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,216 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What
|
||||
is 1231 * 2331? Use the multiply tool."}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false,"tools":[{"type":"function","name":"multiply","description":"Multiply
|
||||
two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '400'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAA/5VVS0/bQBC+8yuQz4BsB8cO115aaKVKPaGCrLF3HBbW3u0+gAjlv3d2Hb8CVGoO
|
||||
UXa+nW/n8c3k7eT0NOIsujqNNBpVxlkdJyxjNVaX66TO4mS9aaBKqirDOi6SzbrKcVU0RRrnAJgy
|
||||
Fp15Clk9Ym0HGtkZ7O21RrDISvBYkudFvMqTfBMwY8E6431q2SqBdK93qqB+2mrpOh9XA8Jgb+ZC
|
||||
8G5Ltjc6kkHBDrX3Z/iMQio6ELDvHx4ol08XcUBRa+k9OydEMDQa/zjs6l2psANhdwTGF/1l3g1k
|
||||
JUMLXJi5J++M1a62nJKe21t4LaWzytnSyid8D1opRVmDWNK1kqHwOW2VPc8usvM0Ttfn8eV5uorG
|
||||
Cxr8c3O3/iWy/A6l6Qs0trapP29svYY0NBaKJt9keZauWYLNZhOeCyR2pzDQuC7kGaKe4M/6GEDQ
|
||||
W9diZwP+dhfBXXSVpKvk7C6q6Ge6WiX76bYnLvuYw8/uuv1yk/+6xhU8sttbJb5VX+Ob7ffJo4M2
|
||||
xNY6YbkSuygAe/q+D4VRoIkIxbLc1LFeU4rkSm3HD9pO0DOXzpSDovvAxpIrTdlaoqwfsHzC3aeY
|
||||
pop0h4ZF6eVD30caDCO7hZ6xaaQOQyTkyyHFyLi2BT2wj/o20KDdUUieuuG4ULNB/cwpKcuH+WiA
|
||||
6hMdxk5qXAyWxVZ5TblgTw75W3y1U2wUWQvTeaaKcK+v+iHkZ9SVNDxUM2qRcddOg9n34UFSfB4G
|
||||
Z2U0Aua9go/FN7Weoak1V0NlfxwUcGpfJJWirVCbi38IZUS8RFrqkTaz9PomUlmohku7F/WRYRYm
|
||||
7yxu+0U0fPZnc9/q/3xPPmCJ/LLiGtlYrTGuo6fG0/3Me3jusLNnCDDGfTlB/JznHpRychQG6Ujz
|
||||
sPL9MB1NnZWqFHJLFaw8QTwaVRiwTdEbNPV0WGUR4wYqMfwHOANbnOTHu8UizYqz9/bZdn6bFgpN
|
||||
IJsc44VQj/dzuvoI+Ih3nN7PqK20ICawSMYBcGY5rCQ9YGCDpvYn+79G2nD3kgcAAA==
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f74ce7299e4cb75-DFW
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 03:13:00 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '1326'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=zXi8aTY7cDmdwne_.I.3Hf.KrkQjzxMBHFbO2WWZqFg-1778037179.2921715-1.0.1.1-l3RZkU_etG9t6aSFPSTnlBiIypr5vSKjMdMV9iEuWrDk59KcVyvnaNk5TDFILQjTF1bFbvNJ5NmInvZIg_tp2BdSmbNPLyyqdw3RXJ9W9MUEfl9QCXSFs4E6vlaRqupr;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
03:43:00 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_e6f3256822534dd38be1cf139234e596
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What
|
||||
is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_nJmCK7SJe3ajdYYplIbH0KgL","name":"multiply","arguments":"{\"a\":
|
||||
1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_nJmCK7SJe3ajdYYplIbH0KgL","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":false,"tools":[{"type":"function","name":"multiply","description":"Multiply
|
||||
two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '619'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAA/5VVTY/aMBC98ytQjhWskgAhVOpPqNT7qoomyQTcdWzXdthFK/57x3YSEhYOPUFm
|
||||
PM/z8d74c7FcRqyOvi8jjUYV8b7aplBleZlWh+0miZPs0ECZlBVmeZwnsMkPmMEO4sMW8jrLMFo5
|
||||
CFn+wcoOMFKY3l5pBIt1Ac6X7Pd5vNkneex9xoLtjIupZKs40rkQVEL1dtSyEy6vBrjBYGacM3Ek
|
||||
2yd9kkHBBbWLr/GMXCr6IMc1XDxA3l2deC9qLV2k6Dj3hkbj3w5FdSkUCuD2Qs74JeTJxABW1GiB
|
||||
cTONZMJY3VWWUdFTewsfheys6mxh5Rt+dVopeVEBn8O1skbuajoqu9697NZpnGbreLtON9F4QIO7
|
||||
bhoWbiLLq29NaNA42tYcn0+2dhY3WdjRSHebJq92+30FYYIexV4Uehw0Bo4Tx7MRemclhUVxS2qa
|
||||
2Ax2aBN+2DHaHwAhpIWhta+/Z04uj0rL8oHHAxFukm6S5bdluqGfH8t0lWeH1TZLovHstf83hkfq
|
||||
BMbn1DCiQQHCvBOrRreW3HvBGEa1U3GLAcdjECU1TRT5fLpEkEBhReogluEDlpHrzGRnikFAhZ/d
|
||||
OGGqtVWWIKsTFm94eerT6Loe+BGl21OgDenQSDGTDzaN1L5RXL73NUama1vQA/ooJwMN2gul5KAb
|
||||
hjPxGNRnRkVZNsixgY6HWRJFpMaZji22ylG48/akr78fWp8bZdbC7XtCFn8udL1P+Yy6lIb5bhJF
|
||||
a9a1tz0Q5nCSlJ8fXGdlNDrMV8EM1zSd8Jq+zb5GU2mmhs7+pAqZ4pelfZfUirZEbV5upwW0QTL9
|
||||
sZvHUaSlGWkzKS8MkdpCPZzbnQzuDJM0GUnsGPbeSOqZFsr/i108QIncbmQa65mSfV53V41fEz3e
|
||||
VB6eiIkH6pq5dgL/Na3dM2VxlwbxSDP/wjgx3anOSlVM1kE8GpUX2CEPBk0zHTZnVDMDJR+enM7v
|
||||
tZF+TMz29mG7+mqfPAYje7wC61tgPCPq/XOQ7B85HuGO6n0GbWlP8glykowK6MxcrcQ9qMF6Ul0X
|
||||
1381ZAUOAggAAA==
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f74ce7c3b71cb75-DFW
|
||||
content-encoding:
|
||||
- gzip
|
||||
content-type:
|
||||
- application/json
|
||||
date:
|
||||
- Wed, 06 May 2026 03:13:02 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '1106'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=yqi2SzRPEQIj_22_ctCcbWfMKVpSPEKe83ZGD3EBaLY-1778037180.8329482-1.0.1.1-enl4RmONGIjXc08ZO75dW7K7g.J5p7W_GNXjJSYbhPlgeJK0.2h4kJfDALyQFDJUpAFXFOhfg.OC3VA.I7QfOELuUPnL1HR2iM7vf7Giaqal.KxFdQJDyFBy5H6GRQcu;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
03:43:02 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_3a3a8ea3224a44ef8a1ccfa281e21c3a
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,393 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What
|
||||
is 1231 * 2331? Use the multiply tool."}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true,"tools":[{"type":"function","name":"multiply","description":"Multiply
|
||||
two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '399'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: 'event: response.created
|
||||
|
||||
data: {"type":"response.created","response":{"id":"resp_00d64fa806f333310169fab1be69d081a08f8285661855594c","object":"response","created_at":1778037182,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply
|
||||
two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0}
|
||||
|
||||
|
||||
event: response.in_progress
|
||||
|
||||
data: {"type":"response.in_progress","response":{"id":"resp_00d64fa806f333310169fab1be69d081a08f8285661855594c","object":"response","created_at":1778037182,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply
|
||||
two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1}
|
||||
|
||||
|
||||
event: response.output_item.added
|
||||
|
||||
data: {"type":"response.output_item.added","item":{"id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","type":"function_call","status":"in_progress","arguments":"","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply"},"output_index":0,"sequence_number":2}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"ReLAoBL6oNRsXA","output_index":0,"sequence_number":3}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"a","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"oZA1PK3x65X0w51","output_index":0,"sequence_number":4}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"WFzXP9CAvnsSdE","output_index":0,"sequence_number":5}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"123","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"3bk9yszS1TEHx","output_index":0,"sequence_number":6}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"ckqcfb2NtXm34XT","output_index":0,"sequence_number":7}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":",\"","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"cspustnrzHoiSV","output_index":0,"sequence_number":8}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"b","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"HhO4w36skjtIxaU","output_index":0,"sequence_number":9}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"SnDoVlFiXfYxtl","output_index":0,"sequence_number":10}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"233","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"arziGMMxMM5qo","output_index":0,"sequence_number":11}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"1","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"obhmaTA5ZilaAQc","output_index":0,"sequence_number":12}
|
||||
|
||||
|
||||
event: response.function_call_arguments.delta
|
||||
|
||||
data: {"type":"response.function_call_arguments.delta","delta":"}","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","obfuscation":"Mezf0KdDroRLDUI","output_index":0,"sequence_number":13}
|
||||
|
||||
|
||||
event: response.function_call_arguments.done
|
||||
|
||||
data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":1231,\"b\":2331}","item_id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","output_index":0,"sequence_number":14}
|
||||
|
||||
|
||||
event: response.output_item.done
|
||||
|
||||
data: {"type":"response.output_item.done","item":{"id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply"},"output_index":0,"sequence_number":15}
|
||||
|
||||
|
||||
event: response.completed
|
||||
|
||||
data: {"type":"response.completed","response":{"id":"resp_00d64fa806f333310169fab1be69d081a08f8285661855594c","object":"response","created_at":1778037182,"status":"completed","background":false,"completed_at":1778037183,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5-2026-04-23","moderation":null,"output":[{"id":"fc_00d64fa806f333310169fab1bf324481a0852cda3180856290","type":"function_call","status":"completed","arguments":"{\"a\":1231,\"b\":2331}","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"24h","reasoning":{"effort":"low","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Multiply
|
||||
two numbers.","name":"multiply","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":58,"input_tokens_details":{"cached_tokens":0},"output_tokens":23,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":81},"user":null,"metadata":{}},"sequence_number":16}
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f74ce85bdf0cb75-DFW
|
||||
content-type:
|
||||
- text/event-stream; charset=utf-8
|
||||
date:
|
||||
- Wed, 06 May 2026 03:13:02 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '266'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=dOVwLeU7r7aShRQaxTKiYcu2Y1q8ZJiD8OFt516IQkk-1778037182.353376-1.0.1.1-kDgOFNW1kiTQunazntKDBwpmZh4ai7MGVZI2ZIDLzRXP4MRBfrPevoYSr41Ul_ozGR_VudKgENypl8pRCVaMoqfP66IEVu.Oz7Jz1Le7WzsMCL78NWXm.AbJCZy_KKVD;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
03:43:02 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '39999493'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_996657ef97514aeb93f449d57303514d
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"include":["reasoning.encrypted_content"],"input":[{"role":"user","content":"What
|
||||
is 1231 * 2331? Use the multiply tool."},{"type":"function_call","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","name":"multiply","arguments":"{\"a\":
|
||||
1231, \"b\": 2331}"},{"type":"function_call_output","call_id":"call_sVidsfFJ6zlzRpelrPkTPlpd","output":"2869461"}],"model":"gpt-5.5","reasoning":{"effort":"low"},"store":false,"stream":true,"tools":[{"type":"function","name":"multiply","description":"Multiply
|
||||
two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}]}'
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '618'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Host:
|
||||
- api.openai.com
|
||||
User-Agent:
|
||||
- OpenAI/Python 2.34.0
|
||||
X-Stainless-Arch:
|
||||
- x64
|
||||
X-Stainless-Async:
|
||||
- 'false'
|
||||
X-Stainless-Lang:
|
||||
- python
|
||||
X-Stainless-OS:
|
||||
- Linux
|
||||
X-Stainless-Package-Version:
|
||||
- 2.34.0
|
||||
X-Stainless-Runtime:
|
||||
- CPython
|
||||
X-Stainless-Runtime-Version:
|
||||
- 3.11.15
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/responses
|
||||
response:
|
||||
body:
|
||||
string: "event: response.created\ndata: {\"type\":\"response.created\",\"response\"\
|
||||
:{\"id\":\"resp_0dacb603de1c9e6b0169fab1c2314081a3b1df3cc5c09e0c60\",\"object\"\
|
||||
:\"response\",\"created_at\":1778037186,\"status\":\"in_progress\",\"background\"\
|
||||
:false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\"\
|
||||
:null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\"\
|
||||
:null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"\
|
||||
parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\"\
|
||||
:null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\"\
|
||||
:{\"effort\":\"low\",\"summary\":null},\"safety_identifier\":null,\"service_tier\"\
|
||||
:\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\"\
|
||||
:\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"\
|
||||
type\":\"function\",\"description\":\"Multiply two numbers.\",\"name\":\"\
|
||||
multiply\",\"parameters\":{\"properties\":{\"a\":{\"type\":\"integer\"},\"\
|
||||
b\":{\"type\":\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\"\
|
||||
,\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\"\
|
||||
:0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\"\
|
||||
:{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\"\
|
||||
:\"response.in_progress\",\"response\":{\"id\":\"resp_0dacb603de1c9e6b0169fab1c2314081a3b1df3cc5c09e0c60\"\
|
||||
,\"object\":\"response\",\"created_at\":1778037186,\"status\":\"in_progress\"\
|
||||
,\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\"\
|
||||
:0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\"\
|
||||
:null,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\"\
|
||||
:null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"\
|
||||
previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\"\
|
||||
:\"24h\",\"reasoning\":{\"effort\":\"low\",\"summary\":null},\"safety_identifier\"\
|
||||
:null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\"\
|
||||
:{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\"\
|
||||
:\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Multiply two\
|
||||
\ numbers.\",\"name\":\"multiply\",\"parameters\":{\"properties\":{\"a\":{\"\
|
||||
type\":\"integer\"},\"b\":{\"type\":\"integer\"}},\"required\":[\"a\",\"b\"\
|
||||
],\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"\
|
||||
top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"\
|
||||
user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\n\
|
||||
data: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\"\
|
||||
:\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\"\
|
||||
:2}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\"\
|
||||
,\"content_index\":0,\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"\
|
||||
logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: response.output_text.delta\n\
|
||||
data: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\"\
|
||||
:\"123\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"obfuscation\":\"yJXGKTBmb2UOz\",\"output_index\":0,\"sequence_number\"\
|
||||
:4}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\
|
||||
,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"obfuscation\":\"0bPnrPoF6d7HddV\",\"output_index\":0,\"\
|
||||
sequence_number\":5}\n\nevent: response.output_text.delta\ndata: {\"type\"\
|
||||
:\"response.output_text.delta\",\"content_index\":0,\"delta\":\" \xD7\",\"\
|
||||
item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\"\
|
||||
:[],\"obfuscation\":\"ft7PFRXyUN31Dr\",\"output_index\":0,\"sequence_number\"\
|
||||
:6}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\
|
||||
,\"content_index\":0,\"delta\":\" \",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"obfuscation\":\"HWsKMbb1JWpQBGo\",\"output_index\":0,\"\
|
||||
sequence_number\":7}\n\nevent: response.output_text.delta\ndata: {\"type\"\
|
||||
:\"response.output_text.delta\",\"content_index\":0,\"delta\":\"233\",\"item_id\"\
|
||||
:\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\
|
||||
obfuscation\":\"OyardqgvtiISP\",\"output_index\":0,\"sequence_number\":8}\n\
|
||||
\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\
|
||||
,\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"obfuscation\":\"M5u2aOAAbIF9HaL\",\"output_index\":0,\"\
|
||||
sequence_number\":9}\n\nevent: response.output_text.delta\ndata: {\"type\"\
|
||||
:\"response.output_text.delta\",\"content_index\":0,\"delta\":\" =\",\"item_id\"\
|
||||
:\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\
|
||||
obfuscation\":\"DblLLi7wbRu7Yj\",\"output_index\":0,\"sequence_number\":10}\n\
|
||||
\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\
|
||||
,\"content_index\":0,\"delta\":\" **\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"obfuscation\":\"iiojCebBdYGkO\",\"output_index\":0,\"sequence_number\"\
|
||||
:11}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\
|
||||
,\"content_index\":0,\"delta\":\"2\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"obfuscation\":\"eJPGA9ScIqJTAvj\",\"output_index\":0,\"\
|
||||
sequence_number\":12}\n\nevent: response.output_text.delta\ndata: {\"type\"\
|
||||
:\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\"\
|
||||
:\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\
|
||||
obfuscation\":\"pkBo5DcWhNmQVlb\",\"output_index\":0,\"sequence_number\":13}\n\
|
||||
\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\
|
||||
,\"content_index\":0,\"delta\":\"869\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"obfuscation\":\"6Z8oUB1RMn3DD\",\"output_index\":0,\"sequence_number\"\
|
||||
:14}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\
|
||||
,\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"obfuscation\":\"Bf7uSiaON5ThxbH\",\"output_index\":0,\"\
|
||||
sequence_number\":15}\n\nevent: response.output_text.delta\ndata: {\"type\"\
|
||||
:\"response.output_text.delta\",\"content_index\":0,\"delta\":\"461\",\"item_id\"\
|
||||
:\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"logprobs\":[],\"\
|
||||
obfuscation\":\"qqwxt4JuKvQU6\",\"output_index\":0,\"sequence_number\":16}\n\
|
||||
\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\"\
|
||||
,\"content_index\":0,\"delta\":\"**\",\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"obfuscation\":\"XM0rK6J7j52uLE\",\"output_index\":0,\"\
|
||||
sequence_number\":17}\n\nevent: response.output_text.done\ndata: {\"type\"\
|
||||
:\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"logprobs\":[],\"output_index\":0,\"sequence_number\":18,\"text\":\"1231\
|
||||
\ \xD7 2331 = **2,869,461**\"}\n\nevent: response.content_part.done\ndata:\
|
||||
\ {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\"\
|
||||
:\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\",\"output_index\"\
|
||||
:0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"\
|
||||
text\":\"1231 \xD7 2331 = **2,869,461**\"},\"sequence_number\":19}\n\nevent:\
|
||||
\ response.output_item.done\ndata: {\"type\":\"response.output_item.done\"\
|
||||
,\"item\":{\"id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\"\
|
||||
,\"annotations\":[],\"logprobs\":[],\"text\":\"1231 \xD7 2331 = **2,869,461**\"\
|
||||
}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"\
|
||||
sequence_number\":20}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\"\
|
||||
,\"response\":{\"id\":\"resp_0dacb603de1c9e6b0169fab1c2314081a3b1df3cc5c09e0c60\"\
|
||||
,\"object\":\"response\",\"created_at\":1778037186,\"status\":\"completed\"\
|
||||
,\"background\":false,\"completed_at\":1778037187,\"error\":null,\"frequency_penalty\"\
|
||||
:0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\"\
|
||||
:null,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\"\
|
||||
:null,\"output\":[{\"id\":\"msg_0dacb603de1c9e6b0169fab1c2e40081a3857ad788023cb7ea\"\
|
||||
,\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\"\
|
||||
,\"annotations\":[],\"logprobs\":[],\"text\":\"1231 \xD7 2331 = **2,869,461**\"\
|
||||
}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\"\
|
||||
:true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\"\
|
||||
:null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"low\"\
|
||||
,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\"\
|
||||
,\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"\
|
||||
},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\"\
|
||||
:\"function\",\"description\":\"Multiply two numbers.\",\"name\":\"multiply\"\
|
||||
,\"parameters\":{\"properties\":{\"a\":{\"type\":\"integer\"},\"b\":{\"type\"\
|
||||
:\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\",\"additionalProperties\"\
|
||||
:false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\"\
|
||||
:\"disabled\",\"usage\":{\"input_tokens\":94,\"input_tokens_details\":{\"\
|
||||
cached_tokens\":0},\"output_tokens\":18,\"output_tokens_details\":{\"reasoning_tokens\"\
|
||||
:0},\"total_tokens\":112},\"user\":null,\"metadata\":{}},\"sequence_number\"\
|
||||
:21}\n\n"
|
||||
headers:
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
cf-ray:
|
||||
- 9f74ce9d5cec11fd-ORD
|
||||
content-type:
|
||||
- text/event-stream; charset=utf-8
|
||||
date:
|
||||
- Wed, 06 May 2026 03:13:06 GMT
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '353'
|
||||
openai-project:
|
||||
- proj_2Y4H7T2XPShczTQx5fqMXdNp
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
server:
|
||||
- cloudflare
|
||||
set-cookie:
|
||||
- __cf_bm=YuSyQqrRg_ripxVBGX9ZnLTFASaREsd8RjcORfPvRSE-1778037186.1373405-1.0.1.1-OTt6GIIb.vqG7KdooMAgro_13UWD_y3xibuJnSKYjofIRRPgkkgl3ivm9KWDf85p3Bq6X.FzVkbHHpoqVRfiGNNvdSjs_gxjjGTIEXa8..L8k11d2g6DXPHMJLp1bXFs;
|
||||
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 06 May 2026
|
||||
03:43:06 GMT
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
transfer-encoding:
|
||||
- chunked
|
||||
x-content-type-options:
|
||||
- nosniff
|
||||
x-ratelimit-limit-requests:
|
||||
- '15000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '40000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '14999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '39999457'
|
||||
x-ratelimit-reset-requests:
|
||||
- 4ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_808d2a0863014e248d080b21301d5d58
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,333 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is 1231 * 2331?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"multiply","description":"Multiply
|
||||
two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '351'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.78.0
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.13.3
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: 'data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_1EYWDzueHEp8OsB8jJSEp7WB","type":"function","function":{"name":"multiply","arguments":""}}],"refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"a"}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"123"}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"1"}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":",\""}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"b"}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"233"}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"1"}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJBDk2xe66hjff60joVYpXi1hh4","object":"chat.completion.chunk","created":1747148049,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_dbaca60df0","choices":[],"usage":{"prompt_tokens":54,"completion_tokens":20,"total_tokens":74,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 93f2fd4c4ce5238d-SJC
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 13 May 2025 14:54:09 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=ys1VX4Q4znOtsubzjx.nHCe9_hPEK_9fLmKeIYZd9LE-1747148049-1.0.1.1-c_hjWtmrr1A3GwMaBXzfhWwaX3EZ2E5Iz_5j.KkgJ3qqPA8vdd2tTpYVL1KRgkOWgWKSUHvYx9I62zt4yf.e9GO3PWHu60ji3ZEjh81.uNc;
|
||||
path=/; expires=Tue, 13-May-25 15:24:09 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=hnWsKFQeNuxEVe5VU69rl9nk7g.ahx0f.wEzyB.f7Kk-1747148049996-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '547'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-envoy-upstream-service-time:
|
||||
- '552'
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999993'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_c3e995e7a86953713a6dc1b17e399fd5
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is 1231 * 2331?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"call_1EYWDzueHEp8OsB8jJSEp7WB","function":{"name":"multiply","arguments":"{\"a\":
|
||||
1231, \"b\": 2331}"}}]},{"role":"tool","tool_call_id":"call_1EYWDzueHEp8OsB8jJSEp7WB","content":"2869461"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"multiply","description":"Multiply
|
||||
two numbers.","parameters":{"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"],"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '633'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.78.0
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.13.3
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: 'data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
result"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
of"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
\\("},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"123"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"1"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
\\"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"times"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"233"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"1"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
\\"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":")"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
is"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
\\("},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"2"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"869"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"461"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":"
|
||||
\\"},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{"content":")."},"logprobs":null,"finish_reason":null}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null}
|
||||
|
||||
|
||||
data: {"id":"chatcmpl-BWlJCN7VZTtSHROczp0AbrjFGhRMA","object":"chat.completion.chunk","created":1747148050,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_0392822090","choices":[],"usage":{"prompt_tokens":87,"completion_tokens":26,"total_tokens":113,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 93f2fd522938eb20-SJC
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 13 May 2025 14:54:10 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=fdLfMrUkpH5wVR0YRkrP2U10JK7jFW._HUYimqZukgg-1747148050-1.0.1.1-aQIjrwFbyIcTr_HW09RscO7okcLFLbvptmCQBFweX4SskJ3FKciprVe5ffCuvcWn03rURb.wLkcTAzQzoZIdOv6OBYcJu5vMutdjPs9t0EI;
|
||||
path=/; expires=Tue, 13-May-25 15:24:10 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=X620Mz_MZZuz8JBE23JWZUpAD7vTnI.UtEcQjGZZPBA-1747148050599-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '221'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-envoy-upstream-service-time:
|
||||
- '225'
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999987'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_51f3397f64a0302e34a4d78ea85e0585
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,330 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"Can the country of Crumpet have
|
||||
dragons? Answer with only YES or NO"}],"model":"gpt-4o-mini","stream":false,"tools":[{"type":"function","function":{"name":"lookup_population","description":"Returns
|
||||
the current population of the specified fictional country","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}}},{"type":"function","function":{"name":"can_have_dragons","description":"Returns
|
||||
True if the specified population can have dragons, False otherwise","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '650'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.78.0
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.13.3
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jFPBjtowEL3nK6w5kyrJ0gI5slXppWzbZbdqyyoyziS4OLZrO1sQ4t+r
|
||||
GEjCLpWaQ2TNm/fmzYy9DwgBnkNKgK2pY5UW4fSbns1n+ee7+eLP9uPmx+N29Zu5L/fT99VCwqBh
|
||||
qNUvZO7MesNUpQU6rk4wM0gdNqrxaDiK390kb2MPVCpH0dBK7cKhCisueZhEyTCMRmE8PrHXijO0
|
||||
kJKfASGE7P2/8Slz3EJKosE5UqG1tERI2yRCwCjRRIBay62j0sGgA5mSDmVjXdZC9ACnlMgYFaIr
|
||||
fPz2vXM3LCpEtlh8Hz98mKuRmd/Su+nD3Imv98+fZr16R+md9oaKWrJ2SD28jacvihECklaeK5Ta
|
||||
1DrTSteCXhEhBKgp6wqlaxqA/RKYqqUzuyWkS7g1daXRLeEAF7RDcO381JuLwaK2VLweGJVSOW/F
|
||||
T+zphBza5QhVaqNW9gUVCi65XWcGqfU990cfnI14C1BfbBe0UZV2mVMb9EUnyVEUugvYgfHoBDrl
|
||||
qOjFo8ngilyWo6Pcb7+9cIyyNeYdtbt4tM656gFBr/XXbq5pH9vnsvwf+Q5gDLXDPNMGc84uO+7S
|
||||
DDbv819p7ZC9YbBonjnDzHE0zTpyLGgtjq8G7M46rLKCyxKNNtw/HSh0Ft1MknGSRJMIgkPwFwAA
|
||||
//8DALof6VxIBAAA
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 93f47072dde6f88d-IAD
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 13 May 2025 19:07:32 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=vfHkbLfwVTTGPkFT0I4U0xn5CHQZYIpOutDV4z7NRlA-1747163252-1.0.1.1-kj_JiiyNxn9AWCWisV6.pYNShKVqqT0Foicji2.ZLNaAkHm5VEwac0QjxVhCiWQs9Xp_wvkeTzrgVxmD8bkzDwTPn96U.81YERXZda3_m18;
|
||||
path=/; expires=Tue, 13-May-25 19:37:32 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=SQgXKMy2qkeOsbwwTl62blvuirTS_TkZSvEOztbYIlI-1747163252293-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '574'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-envoy-upstream-service-time:
|
||||
- '591'
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999981'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_1e7dabaf1f0dba1ec89a134d3bde8476
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"Can the country of Crumpet have
|
||||
dragons? Answer with only YES or NO"},{"role":"assistant","tool_calls":[{"type":"function","id":"call_TTY8UFNo7rNCaOBUNtlRSvMG","function":{"name":"lookup_population","arguments":"{\"country\":
|
||||
\"Crumpet\"}"}}]},{"role":"tool","tool_call_id":"call_TTY8UFNo7rNCaOBUNtlRSvMG","content":"123124"}],"model":"gpt-4o-mini","stream":false,"tools":[{"type":"function","function":{"name":"lookup_population","description":"Returns
|
||||
the current population of the specified fictional country","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}}},{"type":"function","function":{"name":"can_have_dragons","description":"Returns
|
||||
True if the specified population can have dragons, False otherwise","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '906'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.78.0
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.13.3
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAA4xTTYvbMBC9+1eIOcfFH2k+fNyWlEIPLaWkm+5itNLY0UaWVEkOzYb892J7YzvZ
|
||||
FOqDEfPmvXkzIx0DQkBwyAiwLfWsMjK8W5tP39a7es+i1YZvNs9fXxb4Jf7A/R1bwaRh6KdnZP7M
|
||||
esd0ZSR6oVUHM4vUY6Maz6fzeJYm76ctUGmOsqGVxodTHVZCiTCJkmkYzcN48creasHQQUZ+BYQQ
|
||||
cmz/jU/F8Q9kJJqcIxU6R0uErE8iBKyWTQSoc8J5qjxMBpBp5VE11lUt5QjwWsucUSmHwt13HJ2H
|
||||
YVEpc/p7+eMgvq92Lz9n68U9Z2n6UX9e3o/qddIH0xoqasX6IY3wPp5dFSMEFK2wK6jyLd1jzi0t
|
||||
tXJXGoQAtWVdofKNfzg+gNGmlrTRfYAsTtI4mZ7ggnQKbp0fR0OxWNSOyrfTokpp34q343p8RU79
|
||||
ZqQujdVP7ooKhVDCbXOL1LUNj+cenI20FqC+WC0Yqyvjc6932BaN40WnCsP1G6Fn0GtP5SieziY3
|
||||
9HKOnop29/11Y5RtkQ/U4drRmgs9AoJR72/d3NLu+heq/B/5AWAMjUeeG4tcsMuOhzSLzev8V1o/
|
||||
5dYwOLR7wTD3Am2zD44FrWX3ZsAdnMcqL4Qq0Ror2ocDhcmjdJkskiRaRhCcgr8AAAD//wMAmw02
|
||||
QkYEAAA=
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 93f47082ba71d640-IAD
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 13 May 2025 19:07:35 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=LL6YtOWVW4fA687_GIMcuJC7CM2I.uKx1vGaNkjFTgo-1747163255-1.0.1.1-qML6IsLM49e2bg7zp0uGqn3.JTJP5KlFYfb8o3v9LzyLb.cYoFBXn5te83Wxl5kVjDiXU2vH.QTFQu953KNx87LwsMkI2ZxTvH58oZWAawg;
|
||||
path=/; expires=Tue, 13-May-25 19:37:35 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=QOa3sx0F4_nAYKtjmx9ux7qfIsyipGZq94AL_SWd2ac-1747163255176-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '575'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-envoy-upstream-service-time:
|
||||
- '587'
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999976'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_66cc3b2bbe3be82a37d29fba7672d82b
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"Can the country of Crumpet have
|
||||
dragons? Answer with only YES or NO"},{"role":"assistant","tool_calls":[{"type":"function","id":"call_TTY8UFNo7rNCaOBUNtlRSvMG","function":{"name":"lookup_population","arguments":"{\"country\":
|
||||
\"Crumpet\"}"}}]},{"role":"tool","tool_call_id":"call_TTY8UFNo7rNCaOBUNtlRSvMG","content":"123124"},{"role":"assistant","tool_calls":[{"type":"function","id":"call_aq9UyiSFkzX6W8Ydc33DoI9Y","function":{"name":"can_have_dragons","arguments":"{\"population\":
|
||||
123124}"}}]},{"role":"tool","tool_call_id":"call_aq9UyiSFkzX6W8Ydc33DoI9Y","content":"true"}],"model":"gpt-4o-mini","stream":false,"tools":[{"type":"function","function":{"name":"lookup_population","description":"Returns
|
||||
the current population of the specified fictional country","parameters":{"properties":{"country":{"type":"string"}},"required":["country"],"type":"object"}}},{"type":"function","function":{"name":"can_have_dragons","description":"Returns
|
||||
True if the specified population can have dragons, False otherwise","parameters":{"properties":{"population":{"type":"integer"}},"required":["population"],"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1157'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
x-stainless-arch:
|
||||
- arm64
|
||||
x-stainless-async:
|
||||
- 'false'
|
||||
x-stainless-lang:
|
||||
- python
|
||||
x-stainless-os:
|
||||
- MacOS
|
||||
x-stainless-package-version:
|
||||
- 1.78.0
|
||||
x-stainless-read-timeout:
|
||||
- '600'
|
||||
x-stainless-retry-count:
|
||||
- '0'
|
||||
x-stainless-runtime:
|
||||
- CPython
|
||||
x-stainless-runtime-version:
|
||||
- 3.13.3
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAAAwAAAP//jJJBb9swDIXv/hUCz/HgOGmd5NYW2447bNjQDIWhSLSjThYFiS42FPnv
|
||||
g+w0drcO2EUHfXzUexSfMyHAaNgJUEfJqvM2v/3mP37Z31ebq69xb/zdp+Jw8/Sh2lf8qG9gkRR0
|
||||
eETFL6p3ijpvkQ25EauAkjF1XVbranm9Kq+qAXSk0SZZ6zlfU94ZZ/KyKNd5UeXLzVl9JKMwwk58
|
||||
z4QQ4nk4k0+n8SfsRLF4uekwRtki7C5FQkAgm25AxmgiS8ewmKAix+gG6/fvP89JwKaPMrlzvbUz
|
||||
IJ0jlind4OnhTE4XF5ZaH+gQ/5BCY5yJxzqgjOTSi5HJw0BPmRAPQ9r+VQDwgTrPNdMPHJ5brq/H
|
||||
fjANeaKrM2Niaeei7eKNdrVGlsbG2bhASXVEPUmn2cpeG5qBbBb6bzNv9R6DG9f+T/sJKIWeUdc+
|
||||
oDbqdeCpLGBawX+VXYY8GIaI4ckorNlgSB+hsZG9HRcD4q/I2NWNcS0GH8y4HY2vi9W23JRlsS0g
|
||||
O2W/AQAA//8DAFbEZUIrAwAA
|
||||
headers:
|
||||
CF-RAY:
|
||||
- 93f47096cf15d6e9-IAD
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Tue, 13 May 2025 19:07:37 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Set-Cookie:
|
||||
- __cf_bm=EDR.bZeRmrWVNTWef5aAJ2C5NT7yIBHq_6NzNGXNlX0-1747163257-1.0.1.1-YuS4Hj.Ncp4eOrYNT5L7AncdqT5Xn8a2DTxCka1HKKBGKdT8k70yvNTA3wMlQyVPxGD3HSCysY0a1n1zCkNs._TQe9hWOuoIDG9LtD9MBr4;
|
||||
path=/; expires=Tue, 13-May-25 19:37:37 GMT; domain=.api.openai.com; HttpOnly;
|
||||
Secure; SameSite=None
|
||||
- _cfuvid=3Xqq8l5nvU4mfyEz4.llgkHC3jY.IBLFTJrD76P7UsY-1747163257692-0.0.1.1-604800000;
|
||||
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
X-Content-Type-Options:
|
||||
- nosniff
|
||||
access-control-expose-headers:
|
||||
- X-Request-ID
|
||||
alt-svc:
|
||||
- h3=":443"; ma=86400
|
||||
cf-cache-status:
|
||||
- DYNAMIC
|
||||
openai-organization:
|
||||
- user-r3e61fpak04cbaokp5buoae4
|
||||
openai-processing-ms:
|
||||
- '222'
|
||||
openai-version:
|
||||
- '2020-10-01'
|
||||
strict-transport-security:
|
||||
- max-age=31536000; includeSubDomains; preload
|
||||
x-envoy-upstream-service-time:
|
||||
- '227'
|
||||
x-ratelimit-limit-requests:
|
||||
- '30000'
|
||||
x-ratelimit-limit-tokens:
|
||||
- '150000000'
|
||||
x-ratelimit-remaining-requests:
|
||||
- '29999'
|
||||
x-ratelimit-remaining-tokens:
|
||||
- '149999974'
|
||||
x-ratelimit-reset-requests:
|
||||
- 2ms
|
||||
x-ratelimit-reset-tokens:
|
||||
- 0s
|
||||
x-request-id:
|
||||
- req_d157a5a0f4b64776bc387ccab624e664
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,154 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is the current llm version?"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return
|
||||
the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '315'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: 'data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"llm_version","arguments":""}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"llm_version","arguments":"{}"}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":57,"completion_tokens":17,"total_tokens":74,"cost":0.00007159,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 23 Jul 2025 14:54:09 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is the current llm version?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"0","function":{"name":"llm_version","arguments":"{}"}}]},{"role":"tool","tool_call_id":"0","content":"0.fixed-version"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return
|
||||
the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '517'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: 'data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"The"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
current"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
of"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"ll"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"m"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
is"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
**"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"0"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"fixed-version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"**."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop","native_finish_reason":"stop","logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":107,"completion_tokens":15,"total_tokens":122,"cost":0.0001017,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 23 Jul 2025 14:54:10 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,151 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is the current llm version?"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return
|
||||
the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '315'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: 'data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"llm_version","arguments":"{}"}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":57,"completion_tokens":17,"total_tokens":74,"cost":0.00007159,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 23 Jul 2025 14:54:09 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is the current llm version?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"0","function":{"name":"llm_version","arguments":"{}"}}]},{"role":"tool","tool_call_id":"0","content":"0.fixed-version"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return
|
||||
the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '517'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: 'data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"The"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
current"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
of"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"ll"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"m"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
is"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
**"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"0"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"fixed-version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"**."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop","native_finish_reason":"stop","logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":107,"completion_tokens":15,"total_tokens":122,"cost":0.0001017,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 23 Jul 2025 14:54:10 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,149 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is the current llm version?"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return
|
||||
the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '315'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: '
|
||||
data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"llm_version:0","type":"function","function":{"name":"llm_version"}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"function":{"arguments":"{}"},"type":"function"}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"tool_calls","native_finish_reason":"tool_calls","logprobs":null}],"system_fingerprint":"fpv0_170758dd"}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":56,"completion_tokens":12,"total_tokens":68,"cost":0.00005952,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 23 Jul 2025 14:54:09 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is the current llm version?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"0","function":{"name":"llm_version","arguments":"{}"}}]},{"role":"tool","tool_call_id":"0","content":"0.fixed-version"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return
|
||||
the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '517'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: 'data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"The installed"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" of"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" LL"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"M"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" on"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" this"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" system"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" is"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" "},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"0"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"fixed-version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop","native_finish_reason":"stop","logprobs":null}]}
|
||||
|
||||
|
||||
data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":105,"completion_tokens":16,"total_tokens":121,"cost":0.000103,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 23 Jul 2025 14:54:10 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,145 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is the current llm version?"}],"model":"muse-spark-1.1","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return
|
||||
the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '315'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: 'data: {"id":"gen-1753242299-DdArgsNullVariantD00","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242299-DdArgsNullVariantD00","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"llm_version","arguments":null}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242299-DdArgsNullVariantD00","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"tool_calls","native_finish_reason":"tool_calls","logprobs":null}],"usage":{"prompt_tokens":57,"completion_tokens":17,"total_tokens":74,"cost":0.00007159,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 23 Jul 2025 14:54:09 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"messages":[{"role":"user","content":"What is the current llm version?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"0","function":{"name":"llm_version","arguments":"{}"}}]},{"role":"tool","tool_call_id":"0","content":"0.fixed-version"}],"model":"muse-spark-1.1","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return
|
||||
the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}'
|
||||
headers:
|
||||
accept:
|
||||
- application/json
|
||||
accept-encoding:
|
||||
- gzip, deflate
|
||||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '517'
|
||||
content-type:
|
||||
- application/json
|
||||
host:
|
||||
- api.openai.com
|
||||
user-agent:
|
||||
- OpenAI/Python 1.78.0
|
||||
method: POST
|
||||
uri: https://api.openai.com/v1/chat/completions
|
||||
response:
|
||||
body:
|
||||
string: 'data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"The"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
current"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
of"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"ll"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"m"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
is"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"
|
||||
**"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"0"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"fixed-version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"**."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop","native_finish_reason":"stop","logprobs":null}],"system_fingerprint":""}
|
||||
|
||||
|
||||
data: {"id":"gen-1753242300-DdArgsNullVariantD01","provider":"Meta","model":"muse-spark-1.1","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":107,"completion_tokens":15,"total_tokens":122,"cost":0.0001017,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}}
|
||||
|
||||
|
||||
data: [DONE]
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Type:
|
||||
- text/event-stream; charset=utf-8
|
||||
Date:
|
||||
- Tue, 23 Jul 2025 14:54:10 GMT
|
||||
Server:
|
||||
- cloudflare
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,502 @@
|
||||
import importlib.metadata
|
||||
import pytest
|
||||
import sqlite_utils
|
||||
import json
|
||||
import sqlite3
|
||||
import llm
|
||||
import llm_echo
|
||||
from llm.plugins import pm
|
||||
from pydantic import Field
|
||||
from pytest_httpx import IteratorStream
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
import sys
|
||||
|
||||
sys._called_from_test = True
|
||||
|
||||
|
||||
def pytest_report_header(config):
|
||||
conn = sqlite3.connect(":memory:")
|
||||
version = conn.execute("select sqlite_version()").fetchone()[0]
|
||||
conn.close()
|
||||
sqlite_utils_version = importlib.metadata.version("sqlite-utils")
|
||||
return [
|
||||
"SQLite: {}".format(version),
|
||||
"sqlite-utils: {}".format(sqlite_utils_version),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def user_path(tmpdir):
|
||||
dir = tmpdir / "llm.datasette.io"
|
||||
dir.mkdir()
|
||||
return dir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def logs_db(user_path):
|
||||
return sqlite_utils.Database(str(user_path / "logs.db"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def user_path_with_embeddings(user_path):
|
||||
path = str(user_path / "embeddings.db")
|
||||
db = sqlite_utils.Database(path)
|
||||
collection = llm.Collection("demo", db, model_id="embed-demo")
|
||||
collection.embed("1", "hello world", store=True)
|
||||
collection.embed("2", "goodbye world", store=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def templates_path(user_path):
|
||||
dir = user_path / "templates"
|
||||
dir.mkdir()
|
||||
return dir
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def env_setup(monkeypatch, user_path):
|
||||
monkeypatch.setenv("LLM_USER_PATH", str(user_path))
|
||||
|
||||
|
||||
class MockModel(llm.Model):
|
||||
model_id = "mock"
|
||||
attachment_types = {"image/png", "audio/wav"}
|
||||
can_stream = True
|
||||
supports_schema = True
|
||||
supports_tools = True
|
||||
|
||||
class Options(llm.Options):
|
||||
max_tokens: Optional[int] = Field(
|
||||
description="Maximum number of tokens to generate.", default=None
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
self.history = []
|
||||
self._queue = []
|
||||
self.resolved_model_name = None
|
||||
|
||||
def enqueue(self, messages):
|
||||
assert isinstance(messages, list)
|
||||
self._queue.append(messages)
|
||||
|
||||
def execute(self, prompt, stream, response, conversation):
|
||||
self.history.append((prompt, stream, response, conversation))
|
||||
gathered = []
|
||||
while True:
|
||||
try:
|
||||
messages = self._queue.pop(0)
|
||||
for message in messages:
|
||||
gathered.append(message)
|
||||
yield message
|
||||
break
|
||||
except IndexError:
|
||||
break
|
||||
response.set_usage(
|
||||
input=len((prompt.prompt or "").split()), output=len(gathered)
|
||||
)
|
||||
if self.resolved_model_name is not None:
|
||||
response.set_resolved_model(self.resolved_model_name)
|
||||
|
||||
|
||||
class MockKeyModel(llm.KeyModel):
|
||||
model_id = "mock_key"
|
||||
needs_key = "mock"
|
||||
|
||||
def execute(self, prompt, stream, response, conversation, key):
|
||||
return [f"key: {key}"]
|
||||
|
||||
|
||||
class MockAsyncKeyModel(llm.AsyncKeyModel):
|
||||
model_id = "mock_key"
|
||||
needs_key = "mock"
|
||||
|
||||
async def execute(self, prompt, stream, response, conversation, key):
|
||||
yield f"async, key: {key}"
|
||||
|
||||
|
||||
class AsyncMockModel(llm.AsyncModel):
|
||||
model_id = "mock"
|
||||
can_stream = True
|
||||
supports_schema = True
|
||||
|
||||
def __init__(self):
|
||||
self.history = []
|
||||
self._queue = []
|
||||
self.resolved_model_name = None
|
||||
|
||||
def enqueue(self, messages):
|
||||
assert isinstance(messages, list)
|
||||
self._queue.append(messages)
|
||||
|
||||
async def execute(self, prompt, stream, response, conversation):
|
||||
self.history.append((prompt, stream, response, conversation))
|
||||
gathered = []
|
||||
while True:
|
||||
try:
|
||||
messages = self._queue.pop(0)
|
||||
for message in messages:
|
||||
gathered.append(message)
|
||||
yield message
|
||||
break
|
||||
except IndexError:
|
||||
break
|
||||
response.set_usage(
|
||||
input=len((prompt.prompt or "").split()), output=len(gathered)
|
||||
)
|
||||
if self.resolved_model_name is not None:
|
||||
response.set_resolved_model(self.resolved_model_name)
|
||||
|
||||
|
||||
class EmbedDemo(llm.EmbeddingModel):
|
||||
model_id = "embed-demo"
|
||||
batch_size = 10
|
||||
supports_binary = True
|
||||
|
||||
def __init__(self):
|
||||
self.embedded_content = []
|
||||
|
||||
def embed_batch(self, texts):
|
||||
if not hasattr(self, "batch_count"):
|
||||
self.batch_count = 0
|
||||
self.batch_count += 1
|
||||
for text in texts:
|
||||
self.embedded_content.append(text)
|
||||
words = text.split()[:16]
|
||||
embedding = [len(word) for word in words]
|
||||
# Pad with 0 up to 16 words
|
||||
embedding += [0] * (16 - len(embedding))
|
||||
yield embedding
|
||||
|
||||
|
||||
class EmbedBinaryOnly(EmbedDemo):
|
||||
model_id = "embed-binary-only"
|
||||
supports_text = False
|
||||
supports_binary = True
|
||||
|
||||
|
||||
class EmbedTextOnly(EmbedDemo):
|
||||
model_id = "embed-text-only"
|
||||
supports_text = True
|
||||
supports_binary = False
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def embed_demo():
|
||||
return EmbedDemo()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_model():
|
||||
return MockModel()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def async_mock_model():
|
||||
return AsyncMockModel()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_key_model():
|
||||
return MockKeyModel()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_async_key_model():
|
||||
return MockAsyncKeyModel()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def register_embed_demo_model(embed_demo, mock_model, async_mock_model):
|
||||
class MockModelsPlugin:
|
||||
__name__ = "MockModelsPlugin"
|
||||
|
||||
@llm.hookimpl
|
||||
def register_embedding_models(self, register):
|
||||
register(embed_demo)
|
||||
register(EmbedBinaryOnly())
|
||||
register(EmbedTextOnly())
|
||||
|
||||
@llm.hookimpl
|
||||
def register_models(self, register):
|
||||
register(mock_model, async_model=async_mock_model)
|
||||
|
||||
pm.register(MockModelsPlugin(), name="undo-mock-models-plugin")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
pm.unregister(name="undo-mock-models-plugin")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def register_echo_model():
|
||||
class EchoModelPlugin:
|
||||
__name__ = "EchoModelPlugin"
|
||||
|
||||
@llm.hookimpl
|
||||
def register_models(self, register):
|
||||
register(llm_echo.Echo(), llm_echo.EchoAsync())
|
||||
|
||||
pm.register(EchoModelPlugin(), name="undo-EchoModelPlugin")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
pm.unregister(name="undo-EchoModelPlugin")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocked_openai_chat(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/chat/completions",
|
||||
json={
|
||||
"model": "gpt-4o-mini",
|
||||
"usage": {},
|
||||
"choices": [{"message": {"content": "Bob, Alice, Eve"}}],
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
return httpx_mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocked_openai_chat_returning_fenced_code(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/chat/completions",
|
||||
json={
|
||||
"model": "gpt-4o-mini",
|
||||
"usage": {},
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "Code:\n\n````javascript\nfunction foo() {\n return 'bar';\n}\n````\nDone.",
|
||||
}
|
||||
}
|
||||
],
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
return httpx_mock
|
||||
|
||||
|
||||
def stream_events():
|
||||
for delta, finish_reason in (
|
||||
({"role": "assistant", "content": ""}, None),
|
||||
({"content": "Hi"}, None),
|
||||
({"content": "."}, None),
|
||||
({}, "stop"),
|
||||
):
|
||||
yield "data: {}\n\n".format(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "chat-1",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1695096940,
|
||||
"model": "gpt-3.5-turbo-0613",
|
||||
"choices": [
|
||||
{"index": 0, "delta": delta, "finish_reason": finish_reason}
|
||||
],
|
||||
}
|
||||
)
|
||||
).encode("utf-8")
|
||||
yield "data: [DONE]\n\n".encode("utf-8")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocked_openai_chat_stream(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/chat/completions",
|
||||
stream=IteratorStream(stream_events()),
|
||||
headers={"Content-Type": "text/event-stream"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocked_openai_completion(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/completions",
|
||||
json={
|
||||
"id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7",
|
||||
"object": "text_completion",
|
||||
"created": 1589478378,
|
||||
"model": "gpt-3.5-turbo-instruct",
|
||||
"choices": [
|
||||
{
|
||||
"text": "\n\nThis is indeed a test",
|
||||
"index": 0,
|
||||
"logprobs": None,
|
||||
"finish_reason": "length",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12},
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
return httpx_mock
|
||||
|
||||
|
||||
def stream_completion_events():
|
||||
choices_chunks = [
|
||||
[
|
||||
{
|
||||
"text": "\n\n",
|
||||
"index": 0,
|
||||
"logprobs": {
|
||||
"tokens": ["\n\n"],
|
||||
"token_logprobs": [-0.6],
|
||||
"top_logprobs": [{"\n\n": -0.6, "\n": -1.9}],
|
||||
"text_offset": [16],
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"text": "Hi",
|
||||
"index": 0,
|
||||
"logprobs": {
|
||||
"tokens": ["Hi"],
|
||||
"token_logprobs": [-1.1],
|
||||
"top_logprobs": [{"Hi": -1.1, "Hello": -0.7}],
|
||||
"text_offset": [18],
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"text": ".",
|
||||
"index": 0,
|
||||
"logprobs": {
|
||||
"tokens": ["."],
|
||||
"token_logprobs": [-1.1],
|
||||
"top_logprobs": [{".": -1.1, "!": -0.9}],
|
||||
"text_offset": [20],
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"text": "",
|
||||
"index": 0,
|
||||
"logprobs": {
|
||||
"tokens": [],
|
||||
"token_logprobs": [],
|
||||
"top_logprobs": [],
|
||||
"text_offset": [],
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
]
|
||||
|
||||
for choices in choices_chunks:
|
||||
yield "data: {}\n\n".format(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "cmpl-80MdSaou7NnPuff5ZyRMysWBmgSPS",
|
||||
"object": "text_completion",
|
||||
"created": 1695097702,
|
||||
"choices": choices,
|
||||
"model": "gpt-3.5-turbo-instruct",
|
||||
}
|
||||
)
|
||||
).encode("utf-8")
|
||||
yield "data: [DONE]\n\n".encode("utf-8")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocked_openai_completion_logprobs_stream(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/completions",
|
||||
stream=IteratorStream(stream_completion_events()),
|
||||
headers={"Content-Type": "text/event-stream"},
|
||||
)
|
||||
return httpx_mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocked_openai_completion_logprobs(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/completions",
|
||||
json={
|
||||
"id": "cmpl-80MeBfKJutM0uMNJkRrebJLeP3bxL",
|
||||
"object": "text_completion",
|
||||
"created": 1695097747,
|
||||
"model": "gpt-3.5-turbo-instruct",
|
||||
"choices": [
|
||||
{
|
||||
"text": "\n\nHi.",
|
||||
"index": 0,
|
||||
"logprobs": {
|
||||
"tokens": ["\n\n", "Hi", "1"],
|
||||
"token_logprobs": [-0.6, -1.1, -0.9],
|
||||
"top_logprobs": [
|
||||
{"\n\n": -0.6, "\n": -1.9},
|
||||
{"Hi": -1.1, "Hello": -0.7},
|
||||
{".": -0.9, "!": -1.1},
|
||||
],
|
||||
"text_offset": [16, 18, 20],
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
return httpx_mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocked_localai(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="http://localai.localhost/chat/completions",
|
||||
json={
|
||||
"model": "orca",
|
||||
"usage": {},
|
||||
"choices": [{"message": {"content": "Bob, Alice, Eve"}}],
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="http://localai.localhost/completions",
|
||||
json={
|
||||
"model": "completion-babbage",
|
||||
"usage": {},
|
||||
"choices": [{"text": "Hello"}],
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
return httpx_mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collection():
|
||||
collection = llm.Collection("test", model_id="embed-demo")
|
||||
collection.embed(1, "hello world")
|
||||
collection.embed(2, "goodbye world")
|
||||
return collection
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vcr_config():
|
||||
return {"filter_headers": ["Authorization"]}
|
||||
|
||||
|
||||
def extract_braces(s):
|
||||
first = s.find("{")
|
||||
last = s.rfind("}")
|
||||
if first != -1 and last != -1 and first < last:
|
||||
return s[first : last + 1]
|
||||
return None
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
# This should only run in environments where both
|
||||
# llm-cluster and llm-mistral are installed
|
||||
|
||||
PLUGINS=$(llm plugins)
|
||||
echo "$PLUGINS" | jq 'any(.[]; .name == "llm-mistral")' | \
|
||||
grep -q true || ( \
|
||||
echo "Test failed: llm-mistral not found" && \
|
||||
exit 1 \
|
||||
)
|
||||
# With the LLM_LOAD_PLUGINS we should not see that
|
||||
PLUGINS2=$(LLM_LOAD_PLUGINS=llm-cluster llm plugins)
|
||||
echo "$PLUGINS2" | jq 'any(.[]; .name == "llm-mistral")' | \
|
||||
grep -q false || ( \
|
||||
echo "Test failed: llm-mistral should not have been loaded" && \
|
||||
exit 1 \
|
||||
)
|
||||
echo "$PLUGINS2" | jq 'any(.[]; .name == "llm-cluster")' | \
|
||||
grep -q true || ( \
|
||||
echo "Test llm-cluster should have been loaded" && \
|
||||
exit 1 \
|
||||
)
|
||||
# With LLM_LOAD_PLUGINS='' we should see no plugins
|
||||
PLUGINS3=$(LLM_LOAD_PLUGINS='' llm plugins)
|
||||
echo "$PLUGINS3"| \
|
||||
grep -q '\[\]' || ( \
|
||||
echo "Test failed: plugins should have returned []" && \
|
||||
exit 1 \
|
||||
)
|
||||
@@ -0,0 +1,128 @@
|
||||
from click.testing import CliRunner
|
||||
from llm.cli import cli
|
||||
import llm
|
||||
import json
|
||||
import pytest
|
||||
import re
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id_or_alias", ("gpt-3.5-turbo", "chatgpt"))
|
||||
def test_set_alias(model_id_or_alias):
|
||||
with pytest.raises(llm.UnknownModelError):
|
||||
llm.get_model("this-is-a-new-alias")
|
||||
llm.set_alias("this-is-a-new-alias", model_id_or_alias)
|
||||
assert llm.get_model("this-is-a-new-alias").model_id == "gpt-3.5-turbo"
|
||||
|
||||
|
||||
def test_remove_alias():
|
||||
with pytest.raises(KeyError):
|
||||
llm.remove_alias("some-other-alias")
|
||||
llm.set_alias("some-other-alias", "gpt-3.5-turbo")
|
||||
assert llm.get_model("some-other-alias").model_id == "gpt-3.5-turbo"
|
||||
llm.remove_alias("some-other-alias")
|
||||
with pytest.raises(llm.UnknownModelError):
|
||||
llm.get_model("some-other-alias")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args", (["aliases", "list"], ["aliases"]))
|
||||
def test_cli_aliases_list(args):
|
||||
llm.set_alias("e-demo", "embed-demo")
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, args)
|
||||
assert result.exit_code == 0
|
||||
for line in (
|
||||
"3.5 : gpt-3.5-turbo\n"
|
||||
"chatgpt : gpt-3.5-turbo\n"
|
||||
"chatgpt-16k : gpt-3.5-turbo-16k\n"
|
||||
"3.5-16k : gpt-3.5-turbo-16k\n"
|
||||
"4 : gpt-4\n"
|
||||
"gpt4 : gpt-4\n"
|
||||
"4-32k : gpt-4-32k\n"
|
||||
"e-demo : embed-demo (embedding)\n"
|
||||
"ada : text-embedding-ada-002 (embedding)\n"
|
||||
).split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
# Turn the whitespace into a regex
|
||||
regex = r"\s+".join(re.escape(part) for part in line.split())
|
||||
assert re.search(regex, result.output)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args", (["aliases", "list"], ["aliases"]))
|
||||
def test_cli_aliases_list_json(args):
|
||||
llm.set_alias("e-demo", "embed-demo")
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, args + ["--json"])
|
||||
assert result.exit_code == 0
|
||||
assert (
|
||||
json.loads(result.output).items()
|
||||
>= {
|
||||
"3.5": "gpt-3.5-turbo",
|
||||
"chatgpt": "gpt-3.5-turbo",
|
||||
"chatgpt-16k": "gpt-3.5-turbo-16k",
|
||||
"3.5-16k": "gpt-3.5-turbo-16k",
|
||||
"4": "gpt-4",
|
||||
"gpt4": "gpt-4",
|
||||
"4-32k": "gpt-4-32k",
|
||||
"ada": "text-embedding-ada-002",
|
||||
"e-demo": "embed-demo",
|
||||
}.items()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args,expected,expected_error",
|
||||
(
|
||||
(["foo", "bar"], {"foo": "bar"}, None),
|
||||
(["foo", "-q", "mo"], {"foo": "mock"}, None),
|
||||
(["foo", "-q", "mog"], None, "No model found matching query: mog"),
|
||||
),
|
||||
)
|
||||
def test_cli_aliases_set(user_path, args, expected, expected_error):
|
||||
# Should be not aliases.json at start
|
||||
assert not (user_path / "aliases.json").exists()
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["aliases", "set"] + args)
|
||||
if not expected_error:
|
||||
assert result.exit_code == 0
|
||||
assert (user_path / "aliases.json").exists()
|
||||
assert json.loads((user_path / "aliases.json").read_text("utf-8")) == expected
|
||||
else:
|
||||
assert result.exit_code == 1
|
||||
assert result.output.strip() == f"Error: {expected_error}"
|
||||
|
||||
|
||||
def test_cli_aliases_path(user_path):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["aliases", "path"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output.strip() == str(user_path / "aliases.json")
|
||||
|
||||
|
||||
def test_cli_aliases_remove(user_path):
|
||||
(user_path / "aliases.json").write_text(json.dumps({"foo": "bar"}), "utf-8")
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["aliases", "remove", "foo"])
|
||||
assert result.exit_code == 0
|
||||
assert json.loads((user_path / "aliases.json").read_text("utf-8")) == {}
|
||||
|
||||
|
||||
def test_cli_aliases_remove_invalid(user_path):
|
||||
(user_path / "aliases.json").write_text(json.dumps({"foo": "bar"}), "utf-8")
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["aliases", "remove", "invalid"])
|
||||
assert result.exit_code == 1
|
||||
assert result.output == "Error: No such alias: invalid\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args", (["models"], ["models", "list"]))
|
||||
def test_cli_aliases_are_registered(user_path, args):
|
||||
(user_path / "aliases.json").write_text(
|
||||
json.dumps({"foo": "bar", "turbo": "gpt-3.5-turbo"}), "utf-8"
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, args)
|
||||
assert result.exit_code == 0
|
||||
# Check for model line only, without keys, as --options is not used
|
||||
assert "gpt-3.5-turbo (aliases: 3.5, chatgpt, turbo)" in result.output
|
||||
@@ -0,0 +1,61 @@
|
||||
import llm
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_model(async_mock_model):
|
||||
gathered = []
|
||||
async_mock_model.enqueue(["hello world"])
|
||||
async for chunk in async_mock_model.prompt("hello"):
|
||||
gathered.append(chunk)
|
||||
assert gathered == ["hello world"]
|
||||
# Not as an iterator
|
||||
async_mock_model.enqueue(["hello world"])
|
||||
response = await async_mock_model.prompt("hello")
|
||||
text = await response.text()
|
||||
assert text == "hello world"
|
||||
assert isinstance(response, llm.AsyncResponse)
|
||||
usage = await response.usage()
|
||||
assert usage.input == 1
|
||||
assert usage.output == 1
|
||||
assert usage.details is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_model_conversation(async_mock_model):
|
||||
async_mock_model.enqueue(["joke 1"])
|
||||
conversation = async_mock_model.conversation()
|
||||
response = await conversation.prompt("joke")
|
||||
text = await response.text()
|
||||
assert text == "joke 1"
|
||||
async_mock_model.enqueue(["joke 2"])
|
||||
response2 = await conversation.prompt("again")
|
||||
text2 = await response2.text()
|
||||
assert text2 == "joke 2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_on_done(async_mock_model):
|
||||
async_mock_model.enqueue(["hello world"])
|
||||
response = await async_mock_model.prompt(prompt="hello")
|
||||
caught = []
|
||||
|
||||
def done(response):
|
||||
caught.append(response)
|
||||
|
||||
assert len(caught) == 0
|
||||
await response.on_done(done)
|
||||
await response.text()
|
||||
assert response._done
|
||||
assert len(caught) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_conversation(async_mock_model):
|
||||
async_mock_model.enqueue(["one"])
|
||||
conversation = async_mock_model.conversation()
|
||||
response1 = await conversation.prompt("hi").text()
|
||||
async_mock_model.enqueue(["two"])
|
||||
response2 = await conversation.prompt("hi").text()
|
||||
assert response1 == "one"
|
||||
assert response2 == "two"
|
||||
@@ -0,0 +1,379 @@
|
||||
"""Async parity: every sync API must work the same
|
||||
way on AsyncResponse and AsyncConversation.
|
||||
|
||||
Uses the llm-echo plugin (sync ``Echo`` + async ``EchoAsync``) so both
|
||||
paths exercise real registered models with identical behaviour.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import llm
|
||||
import pytest
|
||||
|
||||
# ---- basic sanity: both variants are registered --------------------
|
||||
|
||||
|
||||
def test_echo_registered_for_both():
|
||||
assert isinstance(llm.get_model("echo"), llm.Model)
|
||||
assert isinstance(llm.get_async_model("echo"), llm.AsyncModel)
|
||||
|
||||
|
||||
# ---- AsyncResponse.to_dict / from_dict -----------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_to_dict_captures_chain_and_output():
|
||||
model = llm.get_async_model("echo")
|
||||
r = model.prompt("hello")
|
||||
await r.text()
|
||||
|
||||
d = r.to_dict()
|
||||
assert d["model"] == "echo"
|
||||
assert d["prompt"]["messages"] == [llm.user("hello").to_dict()]
|
||||
# Echo's output is JSON describing the input; it's the assistant's text.
|
||||
assert len(d["messages"]) == 1
|
||||
assert d["messages"][0]["role"] == "assistant"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_to_dict_raises_before_awaited():
|
||||
model = llm.get_async_model("echo")
|
||||
r = model.prompt("hello")
|
||||
with pytest.raises(ValueError):
|
||||
r.to_dict()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_from_dict_rehydrates():
|
||||
model = llm.get_async_model("echo")
|
||||
r = model.prompt("hello")
|
||||
await r.text()
|
||||
|
||||
payload = json.dumps(r.to_dict())
|
||||
restored = llm.AsyncResponse.from_dict(json.loads(payload))
|
||||
|
||||
assert restored._done
|
||||
# text_or_raise should match (same text as original)
|
||||
assert restored.text_or_raise() == r.text_or_raise()
|
||||
# messages structure preserved
|
||||
assert await restored.messages() == await r.messages()
|
||||
# prompt.messages (the chain that was sent) preserved
|
||||
assert restored.prompt.messages == r.prompt.messages
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_from_dict_then_reply_continues():
|
||||
"""Persist an async response across process
|
||||
boundary (via JSON), rehydrate, continue with reply()."""
|
||||
model = llm.get_async_model("echo")
|
||||
r1 = model.prompt("q1")
|
||||
await r1.text()
|
||||
|
||||
payload = json.dumps(r1.to_dict())
|
||||
restored = llm.AsyncResponse.from_dict(json.loads(payload))
|
||||
|
||||
r2 = await restored.reply("q2")
|
||||
await r2.text()
|
||||
|
||||
# r2 was sent the full chain including r1's output.
|
||||
chain_roles = [m.role for m in r2.prompt.messages]
|
||||
assert chain_roles == ["user", "assistant", "user"]
|
||||
assert r2.prompt.messages[0].parts[0].text == "q1"
|
||||
assert r2.prompt.messages[-1].parts[0].text == "q2"
|
||||
|
||||
|
||||
# ---- AsyncResponse rehydrated via from_row (SQLite path) -----------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_from_row_response_messages_synthesized(tmp_path):
|
||||
"""SQLite rehydrate for async responses must populate
|
||||
response.messages from _chunks+_tool_calls so follow-up chains
|
||||
don't silently drop the assistant turn."""
|
||||
import sqlite_utils
|
||||
from llm.migrations import migrate
|
||||
|
||||
model = llm.get_async_model("echo")
|
||||
r = model.prompt("hello")
|
||||
await r.text()
|
||||
|
||||
db = sqlite_utils.Database(str(tmp_path / "logs.db"))
|
||||
migrate(db)
|
||||
# to_sync_response is what log_to_db uses for async.
|
||||
sync_r = await r.to_sync_response()
|
||||
sync_r.log_to_db(db)
|
||||
|
||||
row = next(db["responses"].rows)
|
||||
rehydrated = llm.AsyncResponse.from_row(db, row)
|
||||
|
||||
assert rehydrated._stream_events == []
|
||||
# response.messages falls back to _chunks — must not be empty.
|
||||
msgs = await rehydrated.messages()
|
||||
assert len(msgs) == 1
|
||||
assert msgs[0].role == "assistant"
|
||||
assert isinstance(msgs[0].parts[0], llm.parts.TextPart)
|
||||
|
||||
|
||||
# ---- AsyncConversation follow-up via load_conversation -------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_load_conversation_follow_up_preserves_chain(tmp_path):
|
||||
"""Async equivalent of the llm -c regression: after log_to_db +
|
||||
load_conversation, a follow-up turn's prompt.messages is the full
|
||||
[user, assistant, user] chain — not missing the assistant."""
|
||||
import sqlite_utils
|
||||
from llm.cli import load_conversation
|
||||
from llm.migrations import migrate
|
||||
|
||||
model = llm.get_async_model("echo")
|
||||
r1 = model.prompt("q1")
|
||||
await r1.text()
|
||||
|
||||
db_path = tmp_path / "logs.db"
|
||||
db = sqlite_utils.Database(str(db_path))
|
||||
migrate(db)
|
||||
(await r1.to_sync_response()).log_to_db(db)
|
||||
|
||||
conv = load_conversation(None, async_=True, database=str(db_path))
|
||||
r2 = conv.prompt("q2")
|
||||
await r2.text()
|
||||
|
||||
chain = r2.prompt.messages
|
||||
assert [m.role for m in chain] == ["user", "assistant", "user"]
|
||||
assert chain[0].parts[0].text == "q1"
|
||||
assert chain[-1].parts[0].text == "q2"
|
||||
|
||||
|
||||
# ---- Sync/async semantic parity for reply()+to_dict() --------------
|
||||
|
||||
|
||||
def _capture_sync(model):
|
||||
r1 = model.prompt("ping")
|
||||
r1.text()
|
||||
payload1 = json.dumps(r1.to_dict())
|
||||
restored = llm.Response.from_dict(json.loads(payload1))
|
||||
r2 = restored.reply("pong")
|
||||
r2.text()
|
||||
return r2.prompt.messages
|
||||
|
||||
|
||||
async def _capture_async(model):
|
||||
r1 = model.prompt("ping")
|
||||
await r1.text()
|
||||
payload1 = json.dumps(r1.to_dict())
|
||||
restored = llm.AsyncResponse.from_dict(json.loads(payload1))
|
||||
r2 = await restored.reply("pong")
|
||||
await r2.text()
|
||||
return r2.prompt.messages
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_and_async_produce_identical_chain():
|
||||
"""Run the full save → restore → reply loop against sync Echo and
|
||||
async EchoAsync. The chain sent on the second turn must be
|
||||
structurally identical."""
|
||||
sync_chain = _capture_sync(llm.get_model("echo"))
|
||||
async_chain = await _capture_async(llm.get_async_model("echo"))
|
||||
|
||||
# Echo's assistant output differs between invocations only in
|
||||
# the "previous" field — but for the first turn both see empty
|
||||
# previous, so outputs match.
|
||||
sync_dicts = [m.to_dict() for m in sync_chain]
|
||||
async_dicts = [m.to_dict() for m in async_chain]
|
||||
assert sync_dicts == async_dicts
|
||||
|
||||
|
||||
# ---- AsyncChainResponse tool-result turn pre-bakes chain -----------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_chain_tool_result_turn_has_full_chain():
|
||||
"""AsyncChainResponse must pre-bake the full chain on tool-result
|
||||
turns, same as sync ChainResponse."""
|
||||
|
||||
async def my_tool(x: int) -> int:
|
||||
"Double the input."
|
||||
return x * 2
|
||||
|
||||
model = llm.get_async_model("echo")
|
||||
# Drive a one-iteration chain by asking echo to emit a tool call
|
||||
# (echo's JSON-prompt syntax).
|
||||
chain = model.chain(
|
||||
json.dumps(
|
||||
{
|
||||
"tool_calls": [{"name": "my_tool", "arguments": {"x": 5}}],
|
||||
"prompt": "prompt",
|
||||
}
|
||||
),
|
||||
tools=[llm.Tool.function(my_tool, name="my_tool")],
|
||||
)
|
||||
|
||||
responses = []
|
||||
async for response in chain.responses():
|
||||
responses.append(response)
|
||||
|
||||
# Two responses: the tool-call turn and the tool-result turn.
|
||||
assert len(responses) == 2
|
||||
second = responses[1]
|
||||
# Second turn's prompt.messages includes the prior turn (user +
|
||||
# assistant with tool call) plus a tool-role message with the result.
|
||||
chain_roles = [m.role for m in second.prompt.messages]
|
||||
assert "tool" in chain_roles
|
||||
assert chain_roles[0] == "user"
|
||||
|
||||
|
||||
# ---- astream_events() parity with stream_events() ------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_astream_events_matches_stream_events_for_text_only():
|
||||
"""Echo yields plain str (legacy plugin). Both sync and async
|
||||
paths should wrap those into StreamEvent(type='text') with the
|
||||
same shape."""
|
||||
sync_model = llm.get_model("echo")
|
||||
async_model = llm.get_async_model("echo")
|
||||
|
||||
sync_r = sync_model.prompt("hello")
|
||||
sync_events = list(sync_r.stream_events())
|
||||
|
||||
async_r = async_model.prompt("hello")
|
||||
async_events = []
|
||||
async for ev in async_r.astream_events():
|
||||
async_events.append(ev)
|
||||
|
||||
# Same event types, same payload.
|
||||
assert [e.type for e in sync_events] == [e.type for e in async_events]
|
||||
assert all(e.type == "text" for e in sync_events)
|
||||
assert "".join(e.chunk for e in sync_events) == "".join(
|
||||
e.chunk for e in async_events
|
||||
)
|
||||
|
||||
|
||||
# ---- Async reply chaining --------------------------------------------
|
||||
|
||||
|
||||
# ---- Additional edge cases ----------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_from_dict_model_override():
|
||||
model = llm.get_async_model("echo")
|
||||
r = model.prompt("hi")
|
||||
await r.text()
|
||||
payload = json.dumps(r.to_dict())
|
||||
|
||||
# Pass model explicitly to override whatever's in the payload.
|
||||
alt = llm.get_async_model("echo")
|
||||
restored = llm.AsyncResponse.from_dict(json.loads(payload), model=alt)
|
||||
assert restored.model is alt
|
||||
|
||||
|
||||
def test_sync_from_dict_model_override():
|
||||
model = llm.get_model("echo")
|
||||
r = model.prompt("hi")
|
||||
r.text()
|
||||
payload = json.dumps(r.to_dict())
|
||||
|
||||
alt = llm.get_model("echo")
|
||||
restored = llm.Response.from_dict(json.loads(payload), model=alt)
|
||||
assert restored.model is alt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_to_dict_preserves_datetime():
|
||||
model = llm.get_async_model("echo")
|
||||
r = model.prompt("hi")
|
||||
await r.text()
|
||||
d = r.to_dict()
|
||||
assert "datetime_utc" in d
|
||||
assert isinstance(d["datetime_utc"], str)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_to_dict_preserves_usage_when_set(async_mock_model):
|
||||
"""When a plugin calls response.set_usage, to_dict captures it.
|
||||
async_mock_model does set usage; llm-echo's async variant doesn't."""
|
||||
async_mock_model.enqueue(["ok"])
|
||||
r = async_mock_model.prompt("hi")
|
||||
await r.text()
|
||||
d = r.to_dict()
|
||||
assert "usage" in d
|
||||
assert d["usage"]["input"] is not None
|
||||
assert d["usage"]["output"] is not None
|
||||
|
||||
# And it round-trips.
|
||||
restored = llm.AsyncResponse.from_dict(d, model=async_mock_model)
|
||||
assert restored.input_tokens == d["usage"]["input"]
|
||||
assert restored.output_tokens == d["usage"]["output"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_reply_messages_kwarg_appends():
|
||||
"""AsyncResponse.reply(messages=[...]) appends extra messages onto
|
||||
the chain in place of a trailing user string (mirrors sync test)."""
|
||||
model = llm.get_async_model("echo")
|
||||
r1 = model.prompt("q1")
|
||||
await r1.text()
|
||||
r2 = await r1.reply(messages=[llm.user("extra")])
|
||||
await r2.text()
|
||||
assert [m.role for m in r2.prompt.messages] == ["user", "assistant", "user"]
|
||||
assert r2.prompt.messages[-1].parts[0].text == "extra"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_full_chain_to_dict_round_trip_three_turns():
|
||||
"""Serialize on turn 3 — chain must include q1, a1, q2, a2, q3 on
|
||||
round-trip."""
|
||||
model = llm.get_async_model("echo")
|
||||
r1 = model.prompt("q1")
|
||||
await r1.text()
|
||||
r2 = await r1.reply("q2")
|
||||
await r2.text()
|
||||
r3 = await r2.reply("q3")
|
||||
await r3.text()
|
||||
|
||||
payload = json.dumps(r3.to_dict())
|
||||
restored = llm.AsyncResponse.from_dict(json.loads(payload))
|
||||
assert [m.role for m in restored.prompt.messages] == [
|
||||
"user",
|
||||
"assistant",
|
||||
"user",
|
||||
"assistant",
|
||||
"user",
|
||||
]
|
||||
texts = [m.parts[0].text for m in restored.prompt.messages if m.parts]
|
||||
assert texts[0] == "q1"
|
||||
assert texts[2] == "q2"
|
||||
assert texts[4] == "q3"
|
||||
|
||||
# And continuing from the restored response extends the chain.
|
||||
r4 = await restored.reply("q4")
|
||||
await r4.text()
|
||||
assert [m.role for m in r4.prompt.messages] == [
|
||||
"user",
|
||||
"assistant",
|
||||
"user",
|
||||
"assistant",
|
||||
"user",
|
||||
"assistant",
|
||||
"user",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_reply_chains_three_turns():
|
||||
model = llm.get_async_model("echo")
|
||||
r1 = model.prompt("q1")
|
||||
await r1.text()
|
||||
r2 = await r1.reply("q2")
|
||||
await r2.text()
|
||||
r3 = await r2.reply("q3")
|
||||
await r3.text()
|
||||
|
||||
chain = r3.prompt.messages
|
||||
assert [m.role for m in chain] == ["user", "assistant", "user", "assistant", "user"]
|
||||
texts = [m.parts[0].text for m in chain if m.parts]
|
||||
assert texts[0] == "q1"
|
||||
assert texts[2] == "q2"
|
||||
assert texts[4] == "q3"
|
||||
@@ -0,0 +1,98 @@
|
||||
from click.testing import CliRunner
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import ANY
|
||||
import llm
|
||||
from llm import cli
|
||||
import pytest
|
||||
|
||||
TINY_PNG = (
|
||||
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xa6\x00\x00\x01\x1a"
|
||||
b"\x02\x03\x00\x00\x00\xe6\x99\xc4^\x00\x00\x00\tPLTE\xff\xff\xff"
|
||||
b"\x00\xff\x00\xfe\x01\x00\x12t\x01J\x00\x00\x00GIDATx\xda\xed\xd81\x11"
|
||||
b"\x000\x08\xc0\xc0.]\xea\xaf&Q\x89\x04V\xe0>\xf3+\xc8\x91Z\xf4\xa2\x08EQ\x14E"
|
||||
b"Q\x14EQ\x14EQ\xd4B\x91$I3\xbb\xbf\x08EQ\x14EQ\x14EQ\x14E\xd1\xa5"
|
||||
b"\xd4\x17\x91\xc6\x95\x05\x15\x0f\x9f\xc5\t\x9f\xa4\x00\x00\x00\x00IEND\xaeB`"
|
||||
b"\x82"
|
||||
)
|
||||
|
||||
TINY_WAV = b"RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00D\xac\x00\x00"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attachment_type,attachment_content",
|
||||
[
|
||||
("image/png", TINY_PNG),
|
||||
("audio/wav", TINY_WAV),
|
||||
],
|
||||
)
|
||||
def test_prompt_attachment(mock_model, logs_db, attachment_type, attachment_content):
|
||||
runner = CliRunner()
|
||||
mock_model.enqueue(["two boxes"])
|
||||
result = runner.invoke(
|
||||
cli.cli,
|
||||
["prompt", "-m", "mock", "describe file", "-a", "-"],
|
||||
input=attachment_content,
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert result.output == "two boxes\n"
|
||||
assert mock_model.history[0][0].attachments[0] == llm.Attachment(
|
||||
type=attachment_type, path=None, url=None, content=attachment_content, _id=ANY
|
||||
)
|
||||
|
||||
# Check it was logged correctly
|
||||
conversations = list(logs_db["conversations"].rows)
|
||||
assert len(conversations) == 1
|
||||
conversation = conversations[0]
|
||||
assert conversation["model"] == "mock"
|
||||
assert conversation["name"] == "describe file"
|
||||
response = list(logs_db["responses"].rows)[0]
|
||||
attachment = list(logs_db["attachments"].rows)[0]
|
||||
assert attachment == {
|
||||
"id": ANY,
|
||||
"type": attachment_type,
|
||||
"path": None,
|
||||
"url": None,
|
||||
"content": attachment_content,
|
||||
}
|
||||
prompt_attachment = list(logs_db["prompt_attachments"].rows)[0]
|
||||
assert prompt_attachment["attachment_id"] == attachment["id"]
|
||||
assert prompt_attachment["response_id"] == response["id"]
|
||||
|
||||
|
||||
def _count_open_fds():
|
||||
"""Count open file descriptors (macOS and Linux only)."""
|
||||
if sys.platform == "darwin":
|
||||
fd_dir = "/dev/fd"
|
||||
elif sys.platform == "linux":
|
||||
fd_dir = "/proc/self/fd"
|
||||
else:
|
||||
return None
|
||||
return len(os.listdir(fd_dir))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform not in ("darwin", "linux"),
|
||||
reason="File descriptor counting only supported on macOS and Linux",
|
||||
)
|
||||
def test_attachment_no_file_descriptor_leak(tmp_path):
|
||||
"""Verify reading attachments from paths doesn't leak file descriptors."""
|
||||
test_file = tmp_path / "test.bin"
|
||||
test_file.write_bytes(b"x" * 1000)
|
||||
|
||||
# Warm up - first call may open other resources
|
||||
attachment = llm.Attachment(path=str(test_file))
|
||||
_ = attachment.id()
|
||||
_ = attachment.content_bytes()
|
||||
|
||||
baseline = _count_open_fds()
|
||||
|
||||
# Create many attachments and read them
|
||||
for _ in range(100):
|
||||
a = llm.Attachment(path=str(test_file))
|
||||
_ = a.id()
|
||||
_ = a.content_bytes()
|
||||
|
||||
# File descriptor count should not have grown significantly
|
||||
assert _count_open_fds() <= baseline + 5
|
||||
@@ -0,0 +1,396 @@
|
||||
from click.testing import CliRunner
|
||||
import re
|
||||
from unittest.mock import ANY
|
||||
import json
|
||||
import llm.cli
|
||||
import pytest
|
||||
import sqlite_utils
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_chat_basic(mock_model, logs_db):
|
||||
runner = CliRunner()
|
||||
mock_model.enqueue(["one world"])
|
||||
mock_model.enqueue(["one again"])
|
||||
result = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-m", "mock"],
|
||||
input="Hi\nHi two\nquit\n",
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert result.output == (
|
||||
"Chatting with mock"
|
||||
"\nType 'exit' or 'quit' to exit"
|
||||
"\nType '!multi' to enter multiple lines, then '!end' to finish"
|
||||
"\nType '!edit' to open your default editor and modify the prompt"
|
||||
"\nType '!fragment <my_fragment> [<another_fragment> ...]' to insert one or more fragments"
|
||||
"\n> Hi"
|
||||
"\none world"
|
||||
"\n> Hi two"
|
||||
"\none again"
|
||||
"\n> quit"
|
||||
"\n"
|
||||
)
|
||||
# Should have logged
|
||||
conversations = list(logs_db["conversations"].rows)
|
||||
assert conversations[0] == {
|
||||
"id": ANY,
|
||||
"name": "Hi",
|
||||
"model": "mock",
|
||||
}
|
||||
conversation_id = conversations[0]["id"]
|
||||
responses = list(logs_db["responses"].rows)
|
||||
assert responses == [
|
||||
{
|
||||
"id": ANY,
|
||||
"model": "mock",
|
||||
"resolved_model": None,
|
||||
"prompt": "Hi",
|
||||
"system": None,
|
||||
"prompt_json": None,
|
||||
"options_json": "{}",
|
||||
"response": "one world",
|
||||
"response_json": None,
|
||||
"conversation_id": conversation_id,
|
||||
"duration_ms": ANY,
|
||||
"datetime_utc": ANY,
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"token_details": None,
|
||||
"schema_id": None,
|
||||
"reasoning": None,
|
||||
},
|
||||
{
|
||||
"id": ANY,
|
||||
"model": "mock",
|
||||
"resolved_model": None,
|
||||
"prompt": "Hi two",
|
||||
"system": None,
|
||||
"prompt_json": None,
|
||||
"options_json": "{}",
|
||||
"response": "one again",
|
||||
"response_json": None,
|
||||
"conversation_id": conversation_id,
|
||||
"duration_ms": ANY,
|
||||
"datetime_utc": ANY,
|
||||
"input_tokens": 2,
|
||||
"output_tokens": 1,
|
||||
"token_details": None,
|
||||
"schema_id": None,
|
||||
"reasoning": None,
|
||||
},
|
||||
]
|
||||
# Now continue that conversation
|
||||
mock_model.enqueue(["continued"])
|
||||
result2 = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-m", "mock", "-c"],
|
||||
input="Continue\nquit\n",
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result2.exit_code == 0
|
||||
assert result2.output == (
|
||||
"Chatting with mock"
|
||||
"\nType 'exit' or 'quit' to exit"
|
||||
"\nType '!multi' to enter multiple lines, then '!end' to finish"
|
||||
"\nType '!edit' to open your default editor and modify the prompt"
|
||||
"\nType '!fragment <my_fragment> [<another_fragment> ...]' to insert one or more fragments"
|
||||
"\n> Continue"
|
||||
"\ncontinued"
|
||||
"\n> quit"
|
||||
"\n"
|
||||
)
|
||||
new_responses = list(
|
||||
logs_db.query(
|
||||
"select * from responses where id not in ({})".format(
|
||||
", ".join("?" for _ in responses)
|
||||
),
|
||||
[r["id"] for r in responses],
|
||||
)
|
||||
)
|
||||
assert new_responses == [
|
||||
{
|
||||
"id": ANY,
|
||||
"model": "mock",
|
||||
"resolved_model": None,
|
||||
"prompt": "Continue",
|
||||
"system": None,
|
||||
"prompt_json": None,
|
||||
"options_json": "{}",
|
||||
"response": "continued",
|
||||
"response_json": None,
|
||||
"conversation_id": conversation_id,
|
||||
"duration_ms": ANY,
|
||||
"datetime_utc": ANY,
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"token_details": None,
|
||||
"schema_id": None,
|
||||
"reasoning": None,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_chat_system(mock_model, logs_db):
|
||||
runner = CliRunner()
|
||||
mock_model.enqueue(["I am mean"])
|
||||
result = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-m", "mock", "--system", "You are mean"],
|
||||
input="Hi\nquit\n",
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert result.output == (
|
||||
"Chatting with mock"
|
||||
"\nType 'exit' or 'quit' to exit"
|
||||
"\nType '!multi' to enter multiple lines, then '!end' to finish"
|
||||
"\nType '!edit' to open your default editor and modify the prompt"
|
||||
"\nType '!fragment <my_fragment> [<another_fragment> ...]' to insert one or more fragments"
|
||||
"\n> Hi"
|
||||
"\nI am mean"
|
||||
"\n> quit"
|
||||
"\n"
|
||||
)
|
||||
responses = list(logs_db["responses"].rows)
|
||||
assert responses == [
|
||||
{
|
||||
"id": ANY,
|
||||
"model": "mock",
|
||||
"resolved_model": None,
|
||||
"prompt": "Hi",
|
||||
"system": "You are mean",
|
||||
"prompt_json": None,
|
||||
"options_json": "{}",
|
||||
"response": "I am mean",
|
||||
"response_json": None,
|
||||
"conversation_id": ANY,
|
||||
"duration_ms": ANY,
|
||||
"datetime_utc": ANY,
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"token_details": None,
|
||||
"schema_id": None,
|
||||
"reasoning": None,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_chat_options(mock_model, logs_db, user_path):
|
||||
options_path = user_path / "model_options.json"
|
||||
options_path.write_text(json.dumps({"mock": {"max_tokens": "5"}}), "utf-8")
|
||||
|
||||
runner = CliRunner()
|
||||
mock_model.enqueue(["Default options response"])
|
||||
result = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-m", "mock"],
|
||||
input="Hi\nquit\n",
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
mock_model.enqueue(["Override options response"])
|
||||
result = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-m", "mock", "--option", "max_tokens", "10"],
|
||||
input="Hi with override\nquit\n",
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
responses = list(logs_db["responses"].rows)
|
||||
assert responses == [
|
||||
{
|
||||
"id": ANY,
|
||||
"model": "mock",
|
||||
"resolved_model": None,
|
||||
"prompt": "Hi",
|
||||
"system": None,
|
||||
"prompt_json": None,
|
||||
"options_json": '{"max_tokens": 5}',
|
||||
"response": "Default options response",
|
||||
"response_json": None,
|
||||
"conversation_id": ANY,
|
||||
"duration_ms": ANY,
|
||||
"datetime_utc": ANY,
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"token_details": None,
|
||||
"schema_id": None,
|
||||
"reasoning": None,
|
||||
},
|
||||
{
|
||||
"id": ANY,
|
||||
"model": "mock",
|
||||
"resolved_model": None,
|
||||
"prompt": "Hi with override",
|
||||
"system": None,
|
||||
"prompt_json": None,
|
||||
"options_json": '{"max_tokens": 10}',
|
||||
"response": "Override options response",
|
||||
"response_json": None,
|
||||
"conversation_id": ANY,
|
||||
"duration_ms": ANY,
|
||||
"datetime_utc": ANY,
|
||||
"input_tokens": 3,
|
||||
"output_tokens": 1,
|
||||
"token_details": None,
|
||||
"schema_id": None,
|
||||
"reasoning": None,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
@pytest.mark.parametrize(
|
||||
"input,expected",
|
||||
(
|
||||
(
|
||||
"Hi\n!multi\nthis is multiple lines\nuntil the !end\n!end\nquit\n",
|
||||
[
|
||||
{"prompt": "Hi", "response": "One\n"},
|
||||
{
|
||||
"prompt": "this is multiple lines\nuntil the !end",
|
||||
"response": "Two\n",
|
||||
},
|
||||
],
|
||||
),
|
||||
# quit should not work within !multi
|
||||
(
|
||||
"!multi\nthis is multiple lines\nquit\nuntil the !end\n!end\nquit\n",
|
||||
[
|
||||
{
|
||||
"prompt": "this is multiple lines\nquit\nuntil the !end",
|
||||
"response": "One\n",
|
||||
}
|
||||
],
|
||||
),
|
||||
# Try custom delimiter
|
||||
(
|
||||
"!multi abc\nCustom delimiter\n!end\n!end 123\n!end abc\nquit\n",
|
||||
[{"prompt": "Custom delimiter\n!end\n!end 123", "response": "One\n"}],
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_chat_multi(mock_model, logs_db, input, expected):
|
||||
runner = CliRunner()
|
||||
mock_model.enqueue(["One\n"])
|
||||
mock_model.enqueue(["Two\n"])
|
||||
mock_model.enqueue(["Three\n"])
|
||||
result = runner.invoke(
|
||||
llm.cli.cli, ["chat", "-m", "mock", "--option", "max_tokens", "10"], input=input
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
rows = list(logs_db["responses"].rows_where(select="prompt, response"))
|
||||
assert rows == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom_database_path", (False, True))
|
||||
def test_llm_chat_creates_log_database(tmpdir, monkeypatch, custom_database_path):
|
||||
user_path = tmpdir / "user"
|
||||
custom_db_path = tmpdir / "custom_log.db"
|
||||
monkeypatch.setenv("LLM_USER_PATH", str(user_path))
|
||||
runner = CliRunner()
|
||||
args = ["chat", "-m", "mock"]
|
||||
if custom_database_path:
|
||||
args.extend(["--database", str(custom_db_path)])
|
||||
result = runner.invoke(
|
||||
llm.cli.cli,
|
||||
args,
|
||||
catch_exceptions=False,
|
||||
input="Hi\nHi two\nquit\n",
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
# Should have created user_path and put a logs.db in it
|
||||
if custom_database_path:
|
||||
assert custom_db_path.exists()
|
||||
db_path = str(custom_db_path)
|
||||
else:
|
||||
assert (user_path / "logs.db").exists()
|
||||
db_path = str(user_path / "logs.db")
|
||||
assert sqlite_utils.Database(db_path)["responses"].count == 2
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_chat_tools(logs_db):
|
||||
runner = CliRunner()
|
||||
functions = textwrap.dedent("""
|
||||
def upper(text: str) -> str:
|
||||
"Convert text to upper case"
|
||||
return text.upper()
|
||||
""")
|
||||
result = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-m", "echo", "--functions", functions],
|
||||
input="\n".join(
|
||||
[
|
||||
json.dumps(
|
||||
{
|
||||
"prompt": "Convert hello to uppercase",
|
||||
"tool_calls": [
|
||||
{"name": "upper", "arguments": {"text": "hello"}}
|
||||
],
|
||||
}
|
||||
),
|
||||
"quit",
|
||||
]
|
||||
),
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
normalized_output = re.sub(r"tc_[0-9a-z]{26}", "tc_TCID", result.output)
|
||||
assert normalized_output == (
|
||||
"Chatting with echo\n"
|
||||
"Type 'exit' or 'quit' to exit\n"
|
||||
"Type '!multi' to enter multiple lines, then '!end' to finish\n"
|
||||
"Type '!edit' to open your default editor and modify the prompt\n"
|
||||
"Type '!fragment <my_fragment> [<another_fragment> ...]' to insert one or more fragments\n"
|
||||
'> {"prompt": "Convert hello to uppercase", "tool_calls": [{"name": "upper", '
|
||||
'"arguments": {"text": "hello"}}]}\n'
|
||||
"{\n"
|
||||
' "prompt": "Convert hello to uppercase",\n'
|
||||
' "system": "",\n'
|
||||
' "attachments": [],\n'
|
||||
' "stream": true,\n'
|
||||
' "previous": []\n'
|
||||
"}{\n"
|
||||
' "prompt": "",\n'
|
||||
' "system": "",\n'
|
||||
' "attachments": [],\n'
|
||||
' "stream": true,\n'
|
||||
' "previous": [\n'
|
||||
" {\n"
|
||||
' "prompt": "{\\"prompt\\": \\"Convert hello to uppercase\\", '
|
||||
'\\"tool_calls\\": [{\\"name\\": \\"upper\\", \\"arguments\\": {\\"text\\": '
|
||||
'\\"hello\\"}}]}"\n'
|
||||
" }\n"
|
||||
" ],\n"
|
||||
' "tool_results": [\n'
|
||||
" {\n"
|
||||
' "name": "upper",\n'
|
||||
' "output": "HELLO",\n'
|
||||
' "tool_call_id": "tc_TCID"\n'
|
||||
" }\n"
|
||||
" ]\n"
|
||||
"}\n"
|
||||
"> quit\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_chat_fragments(tmpdir):
|
||||
path1 = str(tmpdir / "frag1.txt")
|
||||
path2 = str(tmpdir / "frag2.txt")
|
||||
with open(path1, "w") as fp:
|
||||
fp.write("one")
|
||||
with open(path2, "w") as fp:
|
||||
fp.write("two")
|
||||
runner = CliRunner()
|
||||
output = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-m", "echo", "-f", path1],
|
||||
input=("hi\n!fragment {}\nquit\n".format(path2)),
|
||||
).output
|
||||
assert '"prompt": "one' in output
|
||||
assert '"prompt": "two"' in output
|
||||
@@ -0,0 +1,100 @@
|
||||
from click.testing import CliRunner
|
||||
import sys
|
||||
import llm.cli
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_chat_template_system_only_no_duplicate_prompt(
|
||||
mock_model, logs_db, templates_path
|
||||
):
|
||||
# Template that only sets a system prompt, no user prompt
|
||||
(templates_path / "wild-french.yaml").write_text(
|
||||
"system: Speak in French\n", "utf-8"
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
mock_model.enqueue(["Bonjour !"])
|
||||
result = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-m", "mock", "-t", "wild-french"],
|
||||
input="hi\nquit\n",
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Ensure the logged prompt is not duplicated (no "hi\nhi")
|
||||
rows = list(logs_db["responses"].rows)
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["prompt"] == "hi"
|
||||
assert rows[0]["system"] == "Speak in French"
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_chat_system_fragments_only_first_turn(tmpdir, mock_model, logs_db):
|
||||
# Create a system fragment file
|
||||
sys_frag_path = str(tmpdir / "sys.txt")
|
||||
with open(sys_frag_path, "w", encoding="utf-8") as fp:
|
||||
fp.write("System fragment content")
|
||||
|
||||
runner = CliRunner()
|
||||
# Two responses queued for two turns
|
||||
mock_model.enqueue(["first"])
|
||||
mock_model.enqueue(["second"])
|
||||
result = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-m", "mock", "--system-fragment", sys_frag_path],
|
||||
input="Hi\nHi two\nquit\n",
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Verify only the first response has the system fragment
|
||||
responses = list(logs_db["responses"].rows)
|
||||
assert len(responses) == 2
|
||||
first_id = responses[0]["id"]
|
||||
second_id = responses[1]["id"]
|
||||
|
||||
sys_frags = list(logs_db["system_fragments"].rows)
|
||||
# Exactly one system fragment row, attached to the first response only
|
||||
assert len(sys_frags) == 1
|
||||
assert sys_frags[0]["response_id"] == first_id
|
||||
assert sys_frags[0]["response_id"] != second_id
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_chat_template_loads_tools_into_logs(logs_db, templates_path):
|
||||
# Template that specifies tools; ensure chat picks them up
|
||||
(templates_path / "mytools.yaml").write_text(
|
||||
"model: echo\n" "tools:\n" "- llm_version\n" "- llm_time\n",
|
||||
"utf-8",
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
llm.cli.cli,
|
||||
["chat", "-t", "mytools"],
|
||||
input="hi\nquit\n",
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Verify a single response was logged for the conversation
|
||||
responses = list(logs_db["responses"].rows)
|
||||
assert len(responses) == 1
|
||||
assert responses[0]["prompt"] == "hi"
|
||||
response_id = responses[0]["id"]
|
||||
|
||||
# Tools from the template should be recorded against that response
|
||||
rows = list(
|
||||
logs_db.query(
|
||||
"""
|
||||
select tools.name from tools
|
||||
join tool_responses tr on tr.tool_id = tools.id
|
||||
where tr.response_id = ?
|
||||
order by tools.name
|
||||
""",
|
||||
[response_id],
|
||||
)
|
||||
)
|
||||
assert [r["name"] for r in rows] == ["llm_time", "llm_version"]
|
||||
@@ -0,0 +1,506 @@
|
||||
from click.testing import CliRunner
|
||||
import json
|
||||
import llm
|
||||
from llm.cli import cli
|
||||
import pytest
|
||||
import sqlite_utils
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mocked_models(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="GET",
|
||||
url="https://api.openai.com/v1/models",
|
||||
json={
|
||||
"data": [
|
||||
{
|
||||
"id": "ada:2020-05-03",
|
||||
"object": "model",
|
||||
"created": 1588537600,
|
||||
"owned_by": "openai",
|
||||
},
|
||||
{
|
||||
"id": "babbage:2020-05-03",
|
||||
"object": "model",
|
||||
"created": 1588537600,
|
||||
"owned_by": "openai",
|
||||
},
|
||||
]
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
return httpx_mock
|
||||
|
||||
|
||||
def test_openai_models(mocked_models):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["openai", "models", "--key", "x"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == (
|
||||
"id owned_by created \n"
|
||||
"ada:2020-05-03 openai 2020-05-03T20:26:40+00:00\n"
|
||||
"babbage:2020-05-03 openai 2020-05-03T20:26:40+00:00\n"
|
||||
)
|
||||
|
||||
|
||||
def test_openai_options_min_max():
|
||||
options = {
|
||||
"temperature": [0, 2],
|
||||
"top_p": [0, 1],
|
||||
"frequency_penalty": [-2, 2],
|
||||
"presence_penalty": [-2, 2],
|
||||
}
|
||||
runner = CliRunner()
|
||||
|
||||
for option, [min_val, max_val] in options.items():
|
||||
result = runner.invoke(cli, ["-m", "chatgpt", "-o", option, "-10"])
|
||||
assert result.exit_code == 1
|
||||
assert f"greater than or equal to {min_val}" in result.output
|
||||
result2 = runner.invoke(cli, ["-m", "chatgpt", "-o", option, "10"])
|
||||
assert result2.exit_code == 1
|
||||
assert f"less than or equal to {max_val}" in result2.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
(
|
||||
"gpt-5",
|
||||
"gpt-5-mini",
|
||||
"gpt-5.1",
|
||||
"gpt-5.2",
|
||||
"gpt-5.4",
|
||||
"gpt-5.5",
|
||||
),
|
||||
)
|
||||
def test_gpt5_models_support_verbosity_option(model_id):
|
||||
assert "verbosity" in llm.get_model(model_id).Options.model_fields
|
||||
assert "verbosity" in llm.get_async_model(model_id).Options.model_fields
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ("gpt-4o", "gpt-4.5-preview", "o3", "o4-mini"))
|
||||
def test_non_gpt5_openai_chat_models_do_not_support_verbosity_option(model_id):
|
||||
assert "verbosity" not in llm.get_model(model_id).Options.model_fields
|
||||
assert "verbosity" not in llm.get_async_model(model_id).Options.model_fields
|
||||
|
||||
|
||||
def test_gpt5_verbosity_option_is_sent_to_openai_chat_completions(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/chat/completions",
|
||||
json={
|
||||
"model": "gpt-5",
|
||||
"usage": {},
|
||||
"choices": [{"message": {"content": "Verbose enough"}}],
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"-m",
|
||||
"gpt-5",
|
||||
"-o",
|
||||
"chat_completions",
|
||||
"1",
|
||||
"-o",
|
||||
"verbosity",
|
||||
"high",
|
||||
"--no-stream",
|
||||
"--key",
|
||||
"x",
|
||||
"Say hi",
|
||||
],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
request_body = json.loads(httpx_mock.get_requests()[-1].content)
|
||||
assert request_body["verbosity"] == "high"
|
||||
assert "text" not in request_body
|
||||
|
||||
|
||||
def test_gpt5_verbosity_option_is_sent_to_openai_responses_by_default(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/responses",
|
||||
json={
|
||||
"id": "resp_test_1",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"model": "gpt-5",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "Verbose enough",
|
||||
"annotations": [],
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 5,
|
||||
"output_tokens": 3,
|
||||
"total_tokens": 8,
|
||||
},
|
||||
"status": "completed",
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"-m",
|
||||
"gpt-5",
|
||||
"-o",
|
||||
"verbosity",
|
||||
"high",
|
||||
"--no-stream",
|
||||
"--key",
|
||||
"x",
|
||||
"Say hi",
|
||||
],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
request_body = json.loads(httpx_mock.get_requests()[-1].content)
|
||||
assert request_body["text"]["verbosity"] == "high"
|
||||
assert request_body["include"] == ["reasoning.encrypted_content"]
|
||||
assert "verbosity" not in request_body
|
||||
|
||||
|
||||
def test_gpt5_verbosity_option_validates_allowed_values():
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "gpt-5", "-o", "verbosity", "extreme", "Say hi"],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "Input should be 'low', 'medium' or 'high'" in result.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id,expected_description",
|
||||
(
|
||||
(
|
||||
"gpt-4o",
|
||||
"Controls the detail level for image attachments. Supported values are low, high, and auto.",
|
||||
),
|
||||
(
|
||||
"gpt-5.4",
|
||||
"Controls the detail level for image attachments. Supported values are low, high, original, and auto.",
|
||||
),
|
||||
(
|
||||
"gpt-5.5",
|
||||
"Controls the detail level for image attachments. Supported values are low, high, original, and auto.",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_openai_image_detail_option_description(model_id, expected_description):
|
||||
field = llm.get_model(model_id).Options.model_fields["image_detail"]
|
||||
assert field.description == expected_description
|
||||
|
||||
|
||||
def test_openai_image_detail_option_is_sent_on_image_attachments(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/chat/completions",
|
||||
json={
|
||||
"model": "gpt-4o",
|
||||
"usage": {},
|
||||
"choices": [{"message": {"content": "Looks detailed"}}],
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"-m",
|
||||
"gpt-4o",
|
||||
"-o",
|
||||
"image_detail",
|
||||
"high",
|
||||
"--at",
|
||||
"https://example.com/image.jpg",
|
||||
"image/jpeg",
|
||||
"--no-stream",
|
||||
"--key",
|
||||
"x",
|
||||
"Describe this",
|
||||
],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
request_body = json.loads(httpx_mock.get_requests()[-1].content)
|
||||
image_part = request_body["messages"][0]["content"][1]
|
||||
assert image_part == {
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://example.com/image.jpg",
|
||||
"detail": "high",
|
||||
},
|
||||
}
|
||||
assert "image_detail" not in request_body
|
||||
|
||||
|
||||
def test_openai_image_detail_original_is_sent_for_gpt54(httpx_mock):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/chat/completions",
|
||||
json={
|
||||
"model": "gpt-5.4",
|
||||
"usage": {},
|
||||
"choices": [{"message": {"content": "Original detail"}}],
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"-m",
|
||||
"gpt-5.4",
|
||||
"-o",
|
||||
"chat_completions",
|
||||
"1",
|
||||
"-o",
|
||||
"image_detail",
|
||||
"original",
|
||||
"--at",
|
||||
"https://example.com/image.jpg",
|
||||
"image/jpeg",
|
||||
"--no-stream",
|
||||
"--key",
|
||||
"x",
|
||||
"Describe this",
|
||||
],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
request_body = json.loads(httpx_mock.get_requests()[-1].content)
|
||||
image_part = request_body["messages"][0]["content"][1]
|
||||
assert image_part["image_url"]["detail"] == "original"
|
||||
|
||||
|
||||
def test_openai_image_detail_original_is_sent_for_gpt54_responses_by_default(
|
||||
httpx_mock,
|
||||
):
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
url="https://api.openai.com/v1/responses",
|
||||
json={
|
||||
"id": "resp_test_1",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"model": "gpt-5.4",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "Original detail",
|
||||
"annotations": [],
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 5,
|
||||
"output_tokens": 3,
|
||||
"total_tokens": 8,
|
||||
},
|
||||
"status": "completed",
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"-m",
|
||||
"gpt-5.4",
|
||||
"-o",
|
||||
"image_detail",
|
||||
"original",
|
||||
"--at",
|
||||
"https://example.com/image.jpg",
|
||||
"image/jpeg",
|
||||
"--no-stream",
|
||||
"--key",
|
||||
"x",
|
||||
"Describe this",
|
||||
],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
request_body = json.loads(httpx_mock.get_requests()[-1].content)
|
||||
image_part = request_body["input"][0]["content"][1]
|
||||
assert image_part == {
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.jpg",
|
||||
"detail": "original",
|
||||
}
|
||||
assert "image_detail" not in request_body
|
||||
|
||||
|
||||
def test_openai_image_detail_original_is_rejected_for_other_models():
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "gpt-5", "-o", "image_detail", "original", "Say hi"],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "Input should be 'low', 'high' or 'auto'" in result.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("gpt-4o-mini", "gpt-4o-audio-preview"))
|
||||
@pytest.mark.parametrize("filetype", ("mp3", "wav"))
|
||||
def test_only_gpt4_audio_preview_allows_mp3_or_wav(httpx_mock, model, filetype):
|
||||
httpx_mock.add_response(
|
||||
method="HEAD",
|
||||
url=f"https://www.example.com/example.{filetype}",
|
||||
content=b"binary-data",
|
||||
headers={"Content-Type": "audio/mpeg" if filetype == "mp3" else "audio/wav"},
|
||||
)
|
||||
if model == "gpt-4o-audio-preview":
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
# chat completion request
|
||||
url="https://api.openai.com/v1/chat/completions",
|
||||
json={
|
||||
"id": "chatcmpl-AQT9a30kxEaM1bqxRPepQsPlCyGJh",
|
||||
"object": "chat.completion",
|
||||
"created": 1730871958,
|
||||
"model": "gpt-4o-audio-preview-2024-10-01",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Why did the pelican get kicked out of the restaurant?\n\nBecause he had a big bill and no way to pay it!",
|
||||
"refusal": None,
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 55,
|
||||
"completion_tokens": 25,
|
||||
"total_tokens": 80,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"audio_tokens": 44,
|
||||
"text_tokens": 11,
|
||||
"image_tokens": 0,
|
||||
},
|
||||
"completion_tokens_details": {
|
||||
"reasoning_tokens": 0,
|
||||
"audio_tokens": 0,
|
||||
"text_tokens": 25,
|
||||
"accepted_prediction_tokens": 0,
|
||||
"rejected_prediction_tokens": 0,
|
||||
},
|
||||
},
|
||||
"system_fingerprint": "fp_49254d0e9b",
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
httpx_mock.add_response(
|
||||
method="GET",
|
||||
url=f"https://www.example.com/example.{filetype}",
|
||||
content=b"binary-data",
|
||||
headers={
|
||||
"Content-Type": "audio/mpeg" if filetype == "mp3" else "audio/wav"
|
||||
},
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"-m",
|
||||
model,
|
||||
"-a",
|
||||
f"https://www.example.com/example.{filetype}",
|
||||
"--no-stream",
|
||||
"--key",
|
||||
"x",
|
||||
],
|
||||
)
|
||||
if model == "gpt-4o-audio-preview":
|
||||
assert result.exit_code == 0
|
||||
assert result.output == (
|
||||
"Why did the pelican get kicked out of the restaurant?\n\n"
|
||||
"Because he had a big bill and no way to pay it!\n"
|
||||
)
|
||||
else:
|
||||
assert result.exit_code == 1
|
||||
long = "audio/mpeg" if filetype == "mp3" else "audio/wav"
|
||||
assert (
|
||||
f"This model does not support attachments of type '{long}'" in result.output
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_", (False, True))
|
||||
@pytest.mark.parametrize("usage", (None, "-u", "--usage"))
|
||||
def test_gpt4o_mini_sync_and_async(monkeypatch, tmpdir, httpx_mock, async_, usage):
|
||||
user_path = tmpdir / "user_dir"
|
||||
log_db = user_path / "logs.db"
|
||||
monkeypatch.setenv("LLM_USER_PATH", str(user_path))
|
||||
assert not log_db.exists()
|
||||
httpx_mock.add_response(
|
||||
method="POST",
|
||||
# chat completion request
|
||||
url="https://api.openai.com/v1/chat/completions",
|
||||
json={
|
||||
"id": "chatcmpl-AQT9a30kxEaM1bqxRPepQsPlCyGJh",
|
||||
"object": "chat.completion",
|
||||
"created": 1730871958,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Ho ho ho",
|
||||
"refusal": None,
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 2000,
|
||||
"total_tokens": 12,
|
||||
},
|
||||
"system_fingerprint": "fp_49254d0e9b",
|
||||
},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
runner = CliRunner()
|
||||
args = ["-m", "gpt-4o-mini", "--key", "x", "--no-stream"]
|
||||
if usage:
|
||||
args.append(usage)
|
||||
if async_:
|
||||
args.append("--async")
|
||||
result = runner.invoke(cli, args, catch_exceptions=False)
|
||||
assert result.exit_code == 0
|
||||
assert result.stdout == "Ho ho ho\n"
|
||||
if usage:
|
||||
assert result.stderr == "Token usage: 1,000 input, 2,000 output\n"
|
||||
# Confirm it was correctly logged
|
||||
assert log_db.exists()
|
||||
db = sqlite_utils.Database(str(log_db))
|
||||
assert db["responses"].count == 1
|
||||
row = next(db["responses"].rows)
|
||||
assert row["response"] == "Ho ho ho"
|
||||
@@ -0,0 +1,138 @@
|
||||
from click.testing import CliRunner
|
||||
from llm.cli import cli
|
||||
import pytest
|
||||
import json
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args,expected_options,expected_error",
|
||||
(
|
||||
(
|
||||
["gpt-4o-mini", "temperature", "0.5"],
|
||||
{"gpt-4o-mini": {"temperature": "0.5"}},
|
||||
None,
|
||||
),
|
||||
(
|
||||
["gpt-4o-mini", "temperature", "invalid"],
|
||||
{},
|
||||
"Error: temperature\n Input should be a valid number",
|
||||
),
|
||||
(
|
||||
["gpt-4o-mini", "not-an-option", "invalid"],
|
||||
{},
|
||||
"Extra inputs are not permitted",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_set_model_default_options(user_path, args, expected_options, expected_error):
|
||||
path = user_path / "model_options.json"
|
||||
assert not path.exists()
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["models", "options", "set"] + args)
|
||||
if not expected_error:
|
||||
assert result.exit_code == 0
|
||||
assert path.exists()
|
||||
data = json.loads(path.read_text("utf-8"))
|
||||
assert data == expected_options
|
||||
else:
|
||||
assert result.exit_code == 1
|
||||
assert expected_error in result.output
|
||||
|
||||
|
||||
def test_model_options_list_and_show(user_path):
|
||||
(user_path / "model_options.json").write_text(
|
||||
json.dumps(
|
||||
{"gpt-4o-mini": {"temperature": 0.5}, "gpt-4o": {"temperature": 0.7}}
|
||||
),
|
||||
"utf-8",
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["models", "options", "list"])
|
||||
assert result.exit_code == 0
|
||||
assert (
|
||||
result.output
|
||||
== "gpt-4o-mini:\n temperature: 0.5\ngpt-4o:\n temperature: 0.7\n"
|
||||
)
|
||||
result = runner.invoke(cli, ["models", "options", "show", "gpt-4o-mini"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == "temperature: 0.5\n"
|
||||
|
||||
|
||||
def test_model_options_clear(user_path):
|
||||
path = user_path / "model_options.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"gpt-4o-mini": {"temperature": 0.5},
|
||||
"gpt-4o": {"temperature": 0.7, "top_p": 0.9},
|
||||
}
|
||||
),
|
||||
"utf-8",
|
||||
)
|
||||
assert path.exists()
|
||||
runner = CliRunner()
|
||||
# Clear all for gpt-4o-mini
|
||||
result = runner.invoke(cli, ["models", "options", "clear", "gpt-4o-mini"])
|
||||
assert result.exit_code == 0
|
||||
# Clear just top_p for gpt-4o
|
||||
result2 = runner.invoke(cli, ["models", "options", "clear", "gpt-4o", "top_p"])
|
||||
assert result2.exit_code == 0
|
||||
data = json.loads(path.read_text("utf-8"))
|
||||
assert data == {"gpt-4o": {"temperature": 0.7}}
|
||||
|
||||
|
||||
def test_prompt_uses_model_options(user_path):
|
||||
path = user_path / "model_options.json"
|
||||
path.write_text("{}", "utf-8")
|
||||
# Prompt should not use an option
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["-m", "echo", "prompt"])
|
||||
assert result.exit_code == 0
|
||||
assert json.loads(result.output) == {
|
||||
"prompt": "prompt",
|
||||
"system": "",
|
||||
"attachments": [],
|
||||
"stream": True,
|
||||
"previous": [],
|
||||
}
|
||||
|
||||
# Now set an option
|
||||
path.write_text(json.dumps({"echo": {"example_bool": True}}), "utf-8")
|
||||
|
||||
result2 = runner.invoke(cli, ["-m", "echo", "prompt"])
|
||||
assert result2.exit_code == 0
|
||||
assert json.loads(result2.output) == {
|
||||
"prompt": "prompt",
|
||||
"system": "",
|
||||
"attachments": [],
|
||||
"stream": True,
|
||||
"previous": [],
|
||||
"options": {"example_bool": True},
|
||||
}
|
||||
|
||||
# Option can be over-ridden
|
||||
result3 = runner.invoke(
|
||||
cli, ["-m", "echo", "prompt", "-o", "example_bool", "false"]
|
||||
)
|
||||
assert result3.exit_code == 0
|
||||
assert json.loads(result3.output) == {
|
||||
"prompt": "prompt",
|
||||
"system": "",
|
||||
"attachments": [],
|
||||
"stream": True,
|
||||
"previous": [],
|
||||
"options": {"example_bool": False},
|
||||
}
|
||||
# Using an alias should also pick up that option
|
||||
aliases_path = user_path / "aliases.json"
|
||||
aliases_path.write_text('{"e": "echo"}', "utf-8")
|
||||
result4 = runner.invoke(cli, ["-m", "e", "prompt"])
|
||||
assert result4.exit_code == 0
|
||||
assert json.loads(result4.output) == {
|
||||
"prompt": "prompt",
|
||||
"system": "",
|
||||
"attachments": [],
|
||||
"stream": True,
|
||||
"previous": [],
|
||||
"options": {"example_bool": True},
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Tests for CLI streaming display: reasoning → stderr (dim),
|
||||
text → stdout, -R / --hide-reasoning flag.
|
||||
"""
|
||||
|
||||
import click
|
||||
from click.testing import CliRunner
|
||||
|
||||
import llm
|
||||
from llm.cli import cli
|
||||
|
||||
|
||||
def test_text_goes_to_stdout_not_stderr(mock_model):
|
||||
mock_model.enqueue(["Hello world"])
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "mock", "hi", "--no-log"],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "Hello world" in result.stdout
|
||||
# No reasoning was emitted — stderr should be empty.
|
||||
assert result.stderr == ""
|
||||
|
||||
|
||||
def test_reasoning_goes_to_stderr_not_stdout(mock_model):
|
||||
mock_model.enqueue(
|
||||
[
|
||||
llm.parts.StreamEvent(
|
||||
type="reasoning", chunk="thinking hard", part_index=0
|
||||
),
|
||||
llm.parts.StreamEvent(type="text", chunk="answer", part_index=1),
|
||||
]
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "mock", "hi", "--no-log"],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "thinking hard" in result.stderr
|
||||
assert "thinking hard" not in result.stdout
|
||||
assert "answer" in result.stdout
|
||||
|
||||
|
||||
def test_reasoning_rendered_in_dim_style(mock_model):
|
||||
"""The click.style(..., dim=True) wrapper emits the ANSI dim code."""
|
||||
mock_model.enqueue(
|
||||
[
|
||||
llm.parts.StreamEvent(type="reasoning", chunk="t", part_index=0),
|
||||
llm.parts.StreamEvent(type="text", chunk="x", part_index=1),
|
||||
]
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "mock", "hi", "--no-log"],
|
||||
catch_exceptions=False,
|
||||
color=True,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
# ANSI dim escape sequence is \x1b[2m
|
||||
dim_start = click.style("x", dim=True).split("x", 1)[0]
|
||||
assert dim_start in result.stderr
|
||||
|
||||
|
||||
def test_hide_reasoning_flag_suppresses_reasoning(mock_model):
|
||||
mock_model.enqueue(
|
||||
[
|
||||
llm.parts.StreamEvent(
|
||||
type="reasoning", chunk="hidden thinking", part_index=0
|
||||
),
|
||||
llm.parts.StreamEvent(type="text", chunk="answer", part_index=1),
|
||||
]
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "mock", "hi", "--no-log", "--hide-reasoning"],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "hidden thinking" not in result.stderr
|
||||
assert "hidden thinking" not in result.stdout
|
||||
assert "answer" in result.stdout
|
||||
assert mock_model.history[0][0].hide_reasoning is True
|
||||
|
||||
|
||||
def test_hide_reasoning_short_flag_R(mock_model):
|
||||
mock_model.enqueue(
|
||||
[
|
||||
llm.parts.StreamEvent(type="reasoning", chunk="hidden", part_index=0),
|
||||
llm.parts.StreamEvent(type="text", chunk="x", part_index=1),
|
||||
]
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "mock", "hi", "--no-log", "-R"],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "hidden" not in result.stderr
|
||||
|
||||
|
||||
def test_newline_between_reasoning_and_text(mock_model):
|
||||
"""When reasoning ends and text begins, stderr gets a newline so the
|
||||
text on stdout starts on a fresh visual line."""
|
||||
mock_model.enqueue(
|
||||
[
|
||||
llm.parts.StreamEvent(type="reasoning", chunk="think", part_index=0),
|
||||
llm.parts.StreamEvent(type="text", chunk="answer", part_index=1),
|
||||
]
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "mock", "hi", "--no-log"],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
# Reasoning ends, then a newline is emitted on stderr.
|
||||
assert result.stderr.rstrip("\n").endswith("think") or "think\n" in result.stderr
|
||||
|
||||
|
||||
def test_async_path_reasoning_to_stderr(async_mock_model):
|
||||
async_mock_model.enqueue(
|
||||
[
|
||||
llm.parts.StreamEvent(
|
||||
type="reasoning", chunk="async thinking", part_index=0
|
||||
),
|
||||
llm.parts.StreamEvent(type="text", chunk="async answer", part_index=1),
|
||||
]
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "mock", "hi", "--async", "--no-log"],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "async thinking" in result.stderr
|
||||
assert "async answer" in result.stdout
|
||||
|
||||
|
||||
def test_plain_str_plugin_still_works(mock_model):
|
||||
"""A plugin that yields plain strings (legacy) still displays
|
||||
correctly — no reasoning branch, everything to stdout."""
|
||||
mock_model.enqueue(["plain ", "text"])
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["-m", "mock", "hi", "--no-log"],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "plain text" in result.stdout
|
||||
assert result.stderr == ""
|
||||
@@ -0,0 +1,187 @@
|
||||
import json
|
||||
import llm
|
||||
from llm.embeddings import Entry
|
||||
import pytest
|
||||
import sqlite_utils
|
||||
from unittest.mock import ANY
|
||||
|
||||
|
||||
def test_demo_plugin():
|
||||
model = llm.get_embedding_model("embed-demo")
|
||||
assert model.embed("hello world") == [5, 5] + [0] * 14
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch_size,expected_batches",
|
||||
(
|
||||
(None, 100),
|
||||
(10, 100),
|
||||
),
|
||||
)
|
||||
def test_embed_huge_list(batch_size, expected_batches):
|
||||
model = llm.get_embedding_model("embed-demo")
|
||||
huge_list = ("hello {}".format(i) for i in range(1000))
|
||||
kwargs = {}
|
||||
if batch_size:
|
||||
kwargs["batch_size"] = batch_size
|
||||
results = model.embed_multi(huge_list, **kwargs)
|
||||
assert repr(type(results)) == "<class 'generator'>"
|
||||
first_twos = {}
|
||||
for result in results:
|
||||
key = (result[0], result[1])
|
||||
first_twos[key] = first_twos.get(key, 0) + 1
|
||||
assert first_twos == {(5, 1): 10, (5, 2): 90, (5, 3): 900}
|
||||
assert model.batch_count == expected_batches
|
||||
|
||||
|
||||
def test_embed_store(collection):
|
||||
collection.embed("3", "hello world again", store=True)
|
||||
assert collection.db["embeddings"].count == 3
|
||||
assert (
|
||||
next(collection.db["embeddings"].rows_where("id = ?", ["3"]))["content"]
|
||||
== "hello world again"
|
||||
)
|
||||
|
||||
|
||||
def test_embed_metadata(collection):
|
||||
collection.embed("3", "hello yet again", metadata={"foo": "bar"}, store=True)
|
||||
assert collection.db["embeddings"].count == 3
|
||||
assert json.loads(
|
||||
next(collection.db["embeddings"].rows_where("id = ?", ["3"]))["metadata"]
|
||||
) == {"foo": "bar"}
|
||||
entry = collection.similar("hello yet again")[0]
|
||||
assert entry.id == "3"
|
||||
assert entry.metadata == {"foo": "bar"}
|
||||
assert entry.content == "hello yet again"
|
||||
|
||||
|
||||
def test_collection(collection):
|
||||
assert collection.id == 1
|
||||
assert collection.count() == 2
|
||||
# Check that the embeddings are there
|
||||
rows = list(collection.db["embeddings"].rows)
|
||||
assert rows == [
|
||||
{
|
||||
"collection_id": 1,
|
||||
"id": "1",
|
||||
"embedding": llm.encode([5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
|
||||
"content": None,
|
||||
"content_blob": None,
|
||||
"content_hash": collection.content_hash("hello world"),
|
||||
"metadata": None,
|
||||
"updated": ANY,
|
||||
},
|
||||
{
|
||||
"collection_id": 1,
|
||||
"id": "2",
|
||||
"embedding": llm.encode([7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
|
||||
"content": None,
|
||||
"content_blob": None,
|
||||
"content_hash": collection.content_hash("goodbye world"),
|
||||
"metadata": None,
|
||||
"updated": ANY,
|
||||
},
|
||||
]
|
||||
assert isinstance(rows[0]["updated"], int) and rows[0]["updated"] > 0
|
||||
|
||||
|
||||
def test_similar(collection):
|
||||
results = list(collection.similar("hello world"))
|
||||
assert results == [
|
||||
Entry(id="1", score=pytest.approx(0.9999999999999999)),
|
||||
Entry(id="2", score=pytest.approx(0.9863939238321437)),
|
||||
]
|
||||
|
||||
|
||||
def test_similar_prefixed(collection):
|
||||
results = list(collection.similar("hello world", prefix="2"))
|
||||
assert results == [
|
||||
Entry(id="2", score=pytest.approx(0.9863939238321437)),
|
||||
]
|
||||
|
||||
|
||||
def test_similar_by_id(collection):
|
||||
results = list(collection.similar_by_id("1"))
|
||||
assert results == [
|
||||
Entry(id="2", score=pytest.approx(0.9863939238321437)),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch_size,expected_batches",
|
||||
(
|
||||
(None, 100),
|
||||
(5, 200),
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize("with_metadata", (False, True))
|
||||
def test_embed_multi(with_metadata, batch_size, expected_batches):
|
||||
db = sqlite_utils.Database(memory=True)
|
||||
collection = llm.Collection("test", db, model_id="embed-demo")
|
||||
model = collection.model()
|
||||
assert getattr(model, "batch_count", 0) == 0
|
||||
ids_and_texts = ((str(i), "hello {}".format(i)) for i in range(1000))
|
||||
kwargs = {}
|
||||
if batch_size is not None:
|
||||
kwargs["batch_size"] = batch_size
|
||||
if with_metadata:
|
||||
ids_and_texts = ((id, text, {"meta": id}) for id, text in ids_and_texts)
|
||||
collection.embed_multi_with_metadata(ids_and_texts, **kwargs)
|
||||
else:
|
||||
# Exercise store=True here too
|
||||
collection.embed_multi(ids_and_texts, store=True, **kwargs)
|
||||
rows = list(db["embeddings"].rows)
|
||||
assert len(rows) == 1000
|
||||
rows_with_metadata = [row for row in rows if row["metadata"] is not None]
|
||||
rows_with_content = [row for row in rows if row["content"] is not None]
|
||||
if with_metadata:
|
||||
assert len(rows_with_metadata) == 1000
|
||||
assert len(rows_with_content) == 0
|
||||
else:
|
||||
assert len(rows_with_metadata) == 0
|
||||
assert len(rows_with_content) == 1000
|
||||
# Every row should have content_hash set
|
||||
assert all(row["content_hash"] is not None for row in rows)
|
||||
# Check batch count
|
||||
assert collection.model().batch_count == expected_batches
|
||||
|
||||
|
||||
def test_collection_delete(collection):
|
||||
db = collection.db
|
||||
assert db["embeddings"].count == 2
|
||||
assert db["collections"].count == 1
|
||||
collection.delete()
|
||||
assert db["embeddings"].count == 0
|
||||
assert db["collections"].count == 0
|
||||
|
||||
|
||||
def test_binary_only_and_text_only_embedding_models():
|
||||
binary_only = llm.get_embedding_model("embed-binary-only")
|
||||
text_only = llm.get_embedding_model("embed-text-only")
|
||||
|
||||
assert binary_only.supports_binary
|
||||
assert not binary_only.supports_text
|
||||
assert not text_only.supports_binary
|
||||
assert text_only.supports_text
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
binary_only.embed("hello world")
|
||||
|
||||
binary_only.embed(b"hello world")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
text_only.embed(b"hello world")
|
||||
|
||||
text_only.embed("hello world")
|
||||
|
||||
# Try the multi versions too
|
||||
# Have to call list() on this or the generator is not evaluated
|
||||
with pytest.raises(ValueError):
|
||||
list(binary_only.embed_multi(["hello world"]))
|
||||
|
||||
list(binary_only.embed_multi([b"hello world"]))
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
list(text_only.embed_multi([b"hello world"]))
|
||||
|
||||
list(text_only.embed_multi(["hello world"]))
|
||||
@@ -0,0 +1,715 @@
|
||||
from click.testing import CliRunner
|
||||
from llm.cli import cli
|
||||
from llm import Collection
|
||||
import json
|
||||
import pathlib
|
||||
import pytest
|
||||
import sqlite_utils
|
||||
import sys
|
||||
from unittest.mock import ANY
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"format_,expected",
|
||||
(
|
||||
("json", "[5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"),
|
||||
(
|
||||
"base64",
|
||||
(
|
||||
"AACgQAAAoEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==\n"
|
||||
),
|
||||
),
|
||||
(
|
||||
"hex",
|
||||
(
|
||||
"0000a0400000a04000000000000000000000000000000000000000000"
|
||||
"000000000000000000000000000000000000000000000000000000000"
|
||||
"00000000000000\n"
|
||||
),
|
||||
),
|
||||
(
|
||||
"blob",
|
||||
(
|
||||
b"\x00\x00\xef\xbf\xbd@\x00\x00\xef\xbf\xbd@\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n"
|
||||
).decode("utf-8"),
|
||||
),
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize("scenario", ("argument", "file", "stdin"))
|
||||
def test_embed_output_format(tmpdir, format_, expected, scenario):
|
||||
runner = CliRunner()
|
||||
args = ["embed", "--format", format_, "-m", "embed-demo"]
|
||||
input = None
|
||||
if scenario == "argument":
|
||||
args.extend(["-c", "hello world"])
|
||||
elif scenario == "file":
|
||||
path = tmpdir / "input.txt"
|
||||
path.write_text("hello world", "utf-8")
|
||||
args.extend(["-i", str(path)])
|
||||
elif scenario == "stdin":
|
||||
input = "hello world"
|
||||
args.extend(["-i", "-"])
|
||||
result = runner.invoke(cli, args, input=input)
|
||||
assert result.exit_code == 0
|
||||
assert result.output == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args,expected_error",
|
||||
((["-c", "Content", "stories"], "Must provide both collection and id"),),
|
||||
)
|
||||
def test_embed_errors(args, expected_error):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["embed"] + args)
|
||||
assert result.exit_code == 1
|
||||
assert expected_error in result.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metadata,metadata_error",
|
||||
(
|
||||
(None, None),
|
||||
('{"foo": "bar"}', None),
|
||||
('{"foo": [1, 2, 3]}', None),
|
||||
("[1, 2, 3]", "metadata must be a JSON object"), # Must be a dictionary
|
||||
('{"foo": "incomplete}', "metadata must be valid JSON"),
|
||||
),
|
||||
)
|
||||
def test_embed_store(user_path, metadata, metadata_error):
|
||||
embeddings_db = user_path / "embeddings.db"
|
||||
assert not embeddings_db.exists()
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["embed", "-c", "hello", "-m", "embed-demo"])
|
||||
assert result.exit_code == 0
|
||||
# Should not have created the table
|
||||
assert not embeddings_db.exists()
|
||||
# Now run it to store
|
||||
args = ["embed", "-c", "hello", "-m", "embed-demo", "items", "1"]
|
||||
if metadata is not None:
|
||||
args.extend(("--metadata", metadata))
|
||||
result = runner.invoke(cli, args)
|
||||
if metadata_error:
|
||||
# Should have returned an error message about invalid metadata
|
||||
assert result.exit_code == 2
|
||||
assert metadata_error in result.output
|
||||
return
|
||||
# No error, should have succeeded and stored the data
|
||||
assert result.exit_code == 0
|
||||
assert embeddings_db.exists()
|
||||
# Check the contents
|
||||
db = sqlite_utils.Database(str(embeddings_db))
|
||||
rows = list(db["collections"].rows)
|
||||
assert rows == [{"id": 1, "name": "items", "model": "embed-demo"}]
|
||||
expected_metadata = None
|
||||
if metadata and not metadata_error:
|
||||
expected_metadata = metadata
|
||||
rows = list(db["embeddings"].rows)
|
||||
assert rows == [
|
||||
{
|
||||
"collection_id": 1,
|
||||
"id": "1",
|
||||
"embedding": (
|
||||
b"\x00\x00\xa0@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00"
|
||||
),
|
||||
"content": None,
|
||||
"content_blob": None,
|
||||
"content_hash": Collection.content_hash("hello"),
|
||||
"metadata": expected_metadata,
|
||||
"updated": ANY,
|
||||
}
|
||||
]
|
||||
# Should show up in 'llm collections list'
|
||||
for is_json in (False, True):
|
||||
args = ["collections"]
|
||||
if is_json:
|
||||
args.extend(["list", "--json"])
|
||||
result2 = runner.invoke(cli, args)
|
||||
assert result2.exit_code == 0
|
||||
if is_json:
|
||||
assert json.loads(result2.output) == [
|
||||
{"name": "items", "model": "embed-demo", "num_embeddings": 1}
|
||||
]
|
||||
else:
|
||||
assert result2.output == "items: embed-demo\n 1 embedding\n"
|
||||
|
||||
# And test deleting it too
|
||||
result = runner.invoke(cli, ["collections", "delete", "items"])
|
||||
assert result.exit_code == 0
|
||||
assert db["collections"].count == 0
|
||||
assert db["embeddings"].count == 0
|
||||
|
||||
|
||||
def test_embed_store_binary(user_path):
|
||||
runner = CliRunner()
|
||||
args = ["embed", "-m", "embed-demo", "items", "2", "--binary", "--store"]
|
||||
result = runner.invoke(cli, args, input=b"\x00\x01\x02")
|
||||
assert result.exit_code == 0
|
||||
db = sqlite_utils.Database(str(user_path / "embeddings.db"))
|
||||
rows = list(db["embeddings"].rows)
|
||||
assert rows == [
|
||||
{
|
||||
"collection_id": 1,
|
||||
"id": "2",
|
||||
"embedding": (
|
||||
b"\x00\x00@@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
),
|
||||
"content": None,
|
||||
"content_blob": b"\x00\x01\x02",
|
||||
"content_hash": b'\xb9_g\xf6\x1e\xbb\x03a\x96"\xd7\x98\xf4_\xc2\xd3',
|
||||
"metadata": None,
|
||||
"updated": ANY,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_collection_delete_errors(user_path):
|
||||
db = sqlite_utils.Database(str(user_path / "embeddings.db"))
|
||||
collection = Collection("items", db, model_id="embed-demo")
|
||||
collection.embed("1", "hello")
|
||||
assert db["collections"].count == 1
|
||||
assert db["embeddings"].count == 1
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli, ["collections", "delete", "does-not-exist"], catch_exceptions=False
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "Collection does not exist" in result.output
|
||||
assert db["collections"].count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args,expected_error",
|
||||
(
|
||||
([], "Missing argument 'COLLECTION'"),
|
||||
(["badcollection", "-c", "content"], "Collection does not exist"),
|
||||
(["demo", "bad-id"], "ID not found in collection"),
|
||||
),
|
||||
)
|
||||
def test_similar_errors(args, expected_error, user_path_with_embeddings):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["similar"] + args, catch_exceptions=False)
|
||||
assert result.exit_code != 0
|
||||
assert expected_error in result.output
|
||||
|
||||
|
||||
def test_similar_by_id_cli(user_path_with_embeddings):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["similar", "demo", "1"], catch_exceptions=False)
|
||||
assert result.exit_code == 0
|
||||
assert json.loads(result.output) == {
|
||||
"id": "2",
|
||||
"score": pytest.approx(0.9863939238321437),
|
||||
"content": "goodbye world",
|
||||
"metadata": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("option", ("-p", "--plain"))
|
||||
def test_similar_by_id_cli_output_plain(user_path_with_embeddings, option):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli, ["similar", "demo", "1", option], catch_exceptions=False
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
# Replace score with a placeholder
|
||||
output = result.output.split("(")[0] + "(score)" + result.output.split(")")[1]
|
||||
assert output == "2 (score)\n\n goodbye world\n\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ("argument", "file", "stdin"))
|
||||
def test_similar_by_content_cli(tmpdir, user_path_with_embeddings, scenario):
|
||||
runner = CliRunner()
|
||||
args = ["similar", "demo"]
|
||||
input = None
|
||||
if scenario == "argument":
|
||||
args.extend(["-c", "hello world"])
|
||||
elif scenario == "file":
|
||||
path = tmpdir / "content.txt"
|
||||
path.write_text("hello world", "utf-8")
|
||||
args.extend(["-i", str(path)])
|
||||
elif scenario == "stdin":
|
||||
input = "hello world"
|
||||
args.extend(["-i", "-"])
|
||||
result = runner.invoke(cli, args, input=input, catch_exceptions=False)
|
||||
assert result.exit_code == 0
|
||||
lines = [line for line in result.output.splitlines() if line.strip()]
|
||||
assert len(lines) == 2
|
||||
assert json.loads(lines[0]) == {
|
||||
"id": "1",
|
||||
"score": pytest.approx(0.9999999999999999),
|
||||
"content": "hello world",
|
||||
"metadata": None,
|
||||
}
|
||||
assert json.loads(lines[1]) == {
|
||||
"id": "2",
|
||||
"score": pytest.approx(0.9863939238321437),
|
||||
"content": "goodbye world",
|
||||
"metadata": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prefix,expected_result",
|
||||
(
|
||||
(
|
||||
1,
|
||||
{
|
||||
"id": "1",
|
||||
"score": pytest.approx(0.7071067811865475),
|
||||
"content": "hello world",
|
||||
"metadata": None,
|
||||
},
|
||||
),
|
||||
(
|
||||
2,
|
||||
{
|
||||
"id": "2",
|
||||
"score": pytest.approx(0.8137334712067349),
|
||||
"content": "goodbye world",
|
||||
"metadata": None,
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_similar_by_content_prefixed(
|
||||
user_path_with_embeddings, prefix, expected_result
|
||||
):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["similar", "demo", "-c", "world", "--prefix", prefix, "-n", "1"],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert json.loads(result.output) == expected_result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_stdin", (False, True))
|
||||
@pytest.mark.parametrize("prefix", (None, "prefix"))
|
||||
@pytest.mark.parametrize("prepend", (None, "search_document: "))
|
||||
@pytest.mark.parametrize(
|
||||
"filename,content",
|
||||
(
|
||||
("phrases.csv", "id,phrase\n1,hello world\n2,goodbye world"),
|
||||
("phrases.tsv", "id\tphrase\n1\thello world\n2\tgoodbye world"),
|
||||
(
|
||||
"phrases.jsonl",
|
||||
'{"id": 1, "phrase": "hello world"}\n{"id": 2, "phrase": "goodbye world"}',
|
||||
),
|
||||
(
|
||||
"phrases.json",
|
||||
'[{"id": 1, "phrase": "hello world"}, {"id": 2, "phrase": "goodbye world"}]',
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_embed_multi_file_input(tmpdir, use_stdin, prefix, prepend, filename, content):
|
||||
db_path = tmpdir / "embeddings.db"
|
||||
args = ["embed-multi", "phrases", "-d", str(db_path), "-m", "embed-demo"]
|
||||
input = None
|
||||
if use_stdin:
|
||||
input = content
|
||||
args.append("-")
|
||||
else:
|
||||
path = tmpdir / filename
|
||||
path.write_text(content, "utf-8")
|
||||
args.append(str(path))
|
||||
if prefix:
|
||||
args.extend(("--prefix", prefix))
|
||||
if prepend:
|
||||
args.extend(("--prepend", prepend))
|
||||
# Auto-detection can't detect JSON-nl, so make that explicit
|
||||
if filename.endswith(".jsonl"):
|
||||
args.extend(("--format", "nl"))
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, args, input=input, catch_exceptions=False)
|
||||
assert result.exit_code == 0
|
||||
# Check that everything was embedded correctly
|
||||
db = sqlite_utils.Database(str(db_path))
|
||||
assert db["embeddings"].count == 2
|
||||
ids = [row["id"] for row in db["embeddings"].rows]
|
||||
expected_ids = ["1", "2"]
|
||||
if prefix:
|
||||
expected_ids = ["prefix1", "prefix2"]
|
||||
assert ids == expected_ids
|
||||
|
||||
|
||||
def test_embed_multi_files_binary_store(tmpdir):
|
||||
db_path = tmpdir / "embeddings.db"
|
||||
args = ["embed-multi", "binfiles", "-d", str(db_path), "-m", "embed-demo"]
|
||||
bin_path = tmpdir / "file.bin"
|
||||
bin_path.write(b"\x00\x01\x02")
|
||||
args.extend(("--files", str(tmpdir), "*.bin", "--store", "--binary"))
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, args, catch_exceptions=False)
|
||||
assert result.exit_code == 0
|
||||
db = sqlite_utils.Database(str(db_path))
|
||||
assert db["embeddings"].count == 1
|
||||
row = list(db["embeddings"].rows)[0]
|
||||
assert row == {
|
||||
"collection_id": 1,
|
||||
"id": "file.bin",
|
||||
"embedding": (
|
||||
b"\x00\x00@@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
|
||||
),
|
||||
"content": None,
|
||||
"content_blob": b"\x00\x01\x02",
|
||||
"content_hash": b'\xb9_g\xf6\x1e\xbb\x03a\x96"\xd7\x98\xf4_\xc2\xd3',
|
||||
"metadata": None,
|
||||
"updated": ANY,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_other_db", (True, False))
|
||||
@pytest.mark.parametrize("prefix", (None, "prefix"))
|
||||
@pytest.mark.parametrize("prepend", (None, "search_document: "))
|
||||
def test_embed_multi_sql(tmpdir, use_other_db, prefix, prepend):
|
||||
db_path = str(tmpdir / "embeddings.db")
|
||||
db = sqlite_utils.Database(db_path)
|
||||
extra_args = []
|
||||
if use_other_db:
|
||||
db_path2 = str(tmpdir / "other.db")
|
||||
db = sqlite_utils.Database(db_path2)
|
||||
extra_args = ["--attach", "other", db_path2]
|
||||
|
||||
if prefix:
|
||||
extra_args.extend(("--prefix", prefix))
|
||||
if prepend:
|
||||
extra_args.extend(("--prepend", prepend))
|
||||
|
||||
db["content"].insert_all(
|
||||
[
|
||||
{"id": 1, "name": "cli", "description": "Command line interface"},
|
||||
{"id": 2, "name": "sql", "description": "Structured query language"},
|
||||
],
|
||||
pk="id",
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"embed-multi",
|
||||
"stuff",
|
||||
"-d",
|
||||
db_path,
|
||||
"--sql",
|
||||
"select * from content",
|
||||
"-m",
|
||||
"embed-demo",
|
||||
"--store",
|
||||
]
|
||||
+ extra_args,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
embeddings_db = sqlite_utils.Database(db_path)
|
||||
assert embeddings_db["embeddings"].count == 2
|
||||
rows = list(embeddings_db.query("select id, content from embeddings order by id"))
|
||||
assert rows == [
|
||||
{
|
||||
"id": (prefix or "") + "1",
|
||||
"content": (prepend or "") + "cli Command line interface",
|
||||
},
|
||||
{
|
||||
"id": (prefix or "") + "2",
|
||||
"content": (prepend or "") + "sql Structured query language",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_embed_multi_batch_size(embed_demo, tmpdir):
|
||||
db_path = str(tmpdir / "data.db")
|
||||
runner = CliRunner()
|
||||
sql = """
|
||||
with recursive cte (id) as (
|
||||
select 1
|
||||
union all
|
||||
select id+1 from cte where id < 100
|
||||
)
|
||||
select id, 'Row ' || cast(id as text) as value from cte
|
||||
"""
|
||||
assert getattr(embed_demo, "batch_count", 0) == 0
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"embed-multi",
|
||||
"rows",
|
||||
"--sql",
|
||||
sql,
|
||||
"-d",
|
||||
db_path,
|
||||
"-m",
|
||||
"embed-demo",
|
||||
"--store",
|
||||
"--batch-size",
|
||||
"8",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
db = sqlite_utils.Database(db_path)
|
||||
assert db["embeddings"].count == 100
|
||||
assert embed_demo.batch_count == 13
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def multi_files(tmpdir):
|
||||
db_path = str(tmpdir / "files.db")
|
||||
files = tmpdir / "files"
|
||||
for filename, content in (
|
||||
("file1.txt", b"hello world"),
|
||||
("file2.txt", b"goodbye world"),
|
||||
("nested/one.txt", b"one"),
|
||||
("nested/two.txt", b"two"),
|
||||
("nested/more/three.txt", b"three"),
|
||||
# This tests the fallback to latin-1 encoding:
|
||||
("nested/more/ignored.ini", b"Has weird \x96 character"),
|
||||
):
|
||||
path = pathlib.Path(files / filename)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(content)
|
||||
return db_path, tmpdir / "files"
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
@pytest.mark.parametrize("scenario", ("single", "multi"))
|
||||
@pytest.mark.parametrize("prepend", (None, "search_document: "))
|
||||
def test_embed_multi_files(multi_files, scenario, prepend):
|
||||
db_path, files = multi_files
|
||||
for filename, content in (
|
||||
("file1.txt", b"hello world"),
|
||||
("file2.txt", b"goodbye world"),
|
||||
("nested/one.txt", b"one"),
|
||||
("nested/two.txt", b"two"),
|
||||
("nested/more/three.txt", b"three"),
|
||||
# This tests the fallback to latin-1 encoding:
|
||||
("nested/more.txt/ignored.ini", b"Has weird \x96 character"),
|
||||
):
|
||||
path = pathlib.Path(files / filename)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(content)
|
||||
|
||||
extra_args = []
|
||||
|
||||
if prepend:
|
||||
extra_args.extend(("--prepend", prepend))
|
||||
if scenario == "single":
|
||||
extra_args.extend(["--files", str(files), "**/*.txt"])
|
||||
else:
|
||||
extra_args.extend(
|
||||
[
|
||||
"--files",
|
||||
str(files / "nested" / "more"),
|
||||
"**/*.ini",
|
||||
"--files",
|
||||
str(files / "nested"),
|
||||
"*.txt",
|
||||
]
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"embed-multi",
|
||||
"files",
|
||||
"-d",
|
||||
db_path,
|
||||
"-m",
|
||||
"embed-demo",
|
||||
"--store",
|
||||
]
|
||||
+ extra_args,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
embeddings_db = sqlite_utils.Database(db_path)
|
||||
rows = list(embeddings_db.query("select id, content from embeddings order by id"))
|
||||
if scenario == "single":
|
||||
assert rows == [
|
||||
{"id": "file1.txt", "content": (prepend or "") + "hello world"},
|
||||
{"id": "file2.txt", "content": (prepend or "") + "goodbye world"},
|
||||
{"id": "nested/more/three.txt", "content": (prepend or "") + "three"},
|
||||
{"id": "nested/one.txt", "content": (prepend or "") + "one"},
|
||||
{"id": "nested/two.txt", "content": (prepend or "") + "two"},
|
||||
]
|
||||
else:
|
||||
assert rows == [
|
||||
{
|
||||
"id": "ignored.ini",
|
||||
"content": (prepend or "") + "Has weird \x96 character",
|
||||
},
|
||||
{"id": "one.txt", "content": (prepend or "") + "one"},
|
||||
{"id": "two.txt", "content": (prepend or "") + "two"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args,expected_error",
|
||||
((["not-a-dir", "*.txt"], "Invalid directory: not-a-dir"),),
|
||||
)
|
||||
def test_embed_multi_files_errors(multi_files, args, expected_error):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
["embed-multi", "files", "-m", "embed-demo", "--files"] + args,
|
||||
)
|
||||
assert result.exit_code == 2
|
||||
assert expected_error in result.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra_args,expected_error",
|
||||
(
|
||||
# With no args default utf-8 with latin-1 fallback should work
|
||||
([], None),
|
||||
(["--encoding", "utf-8"], "Could not decode text in file"),
|
||||
(["--encoding", "latin-1"], None),
|
||||
(["--encoding", "latin-1", "--encoding", "utf-8"], None),
|
||||
(["--encoding", "utf-8", "--encoding", "latin-1"], None),
|
||||
),
|
||||
)
|
||||
def test_embed_multi_files_encoding(multi_files, extra_args, expected_error):
|
||||
db_path, files = multi_files
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
cli,
|
||||
[
|
||||
"embed-multi",
|
||||
"files",
|
||||
"-d",
|
||||
db_path,
|
||||
"-m",
|
||||
"embed-demo",
|
||||
"--files",
|
||||
str(files / "nested" / "more"),
|
||||
"*.ini",
|
||||
"--store",
|
||||
]
|
||||
+ extra_args,
|
||||
)
|
||||
if expected_error:
|
||||
# Should still succeed with 0, but show a warning
|
||||
assert result.exit_code == 0
|
||||
assert expected_error in result.stderr
|
||||
else:
|
||||
assert result.exit_code == 0
|
||||
assert not result.stderr
|
||||
embeddings_db = sqlite_utils.Database(db_path)
|
||||
rows = list(
|
||||
embeddings_db.query("select id, content from embeddings order by id")
|
||||
)
|
||||
assert rows == [
|
||||
{"id": "ignored.ini", "content": "Has weird \x96 character"},
|
||||
]
|
||||
|
||||
|
||||
def test_default_embedding_model():
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["embed-models", "default"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == "<No default embedding model set>\n"
|
||||
result2 = runner.invoke(cli, ["embed-models", "default", "ada-002"])
|
||||
assert result2.exit_code == 0
|
||||
result3 = runner.invoke(cli, ["embed-models", "default"])
|
||||
assert result3.exit_code == 0
|
||||
assert result3.output == "text-embedding-ada-002\n"
|
||||
result4 = runner.invoke(cli, ["embed-models", "default", "--remove-default"])
|
||||
assert result4.exit_code == 0
|
||||
result5 = runner.invoke(cli, ["embed-models", "default"])
|
||||
assert result5.exit_code == 0
|
||||
assert result5.output == "<No default embedding model set>\n"
|
||||
# Now set the default and actually use it
|
||||
result6 = runner.invoke(cli, ["embed-models", "default", "embed-demo"])
|
||||
assert result6.exit_code == 0
|
||||
result7 = runner.invoke(cli, ["embed", "-c", "hello world"])
|
||||
assert result7.exit_code == 0
|
||||
assert result7.output == "[5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args,expected_model_id",
|
||||
(
|
||||
(["-q", "text-embedding-3-large"], "text-embedding-3-large"),
|
||||
(["-q", "text", "-q", "3"], "text-embedding-3-large"),
|
||||
),
|
||||
)
|
||||
def test_llm_embed_models_query(user_path, args, expected_model_id):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["embed-models"] + args, catch_exceptions=False)
|
||||
assert result.exit_code == 0
|
||||
assert expected_model_id in result.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("default_is_set", (False, True))
|
||||
@pytest.mark.parametrize("command", ("embed", "embed-multi"))
|
||||
def test_default_embed_model_errors(user_path, default_is_set, command):
|
||||
runner = CliRunner()
|
||||
if default_is_set:
|
||||
(user_path / "default_embedding_model.txt").write_text(
|
||||
"embed-demo", encoding="utf8"
|
||||
)
|
||||
args = []
|
||||
input = None
|
||||
if command == "embed-multi":
|
||||
args = ["embed-multi", "example", "-"]
|
||||
input = "id,name\n1,hello"
|
||||
else:
|
||||
args = ["embed", "example", "1", "-c", "hello world"]
|
||||
result = runner.invoke(cli, args, input=input, catch_exceptions=False)
|
||||
if default_is_set:
|
||||
assert result.exit_code == 0
|
||||
else:
|
||||
assert result.exit_code == 1
|
||||
assert (
|
||||
"You need to specify an embedding model (no default model is set)"
|
||||
in result.output
|
||||
)
|
||||
# Now set the default model and try again
|
||||
result2 = runner.invoke(cli, ["embed-models", "default", "embed-demo"])
|
||||
assert result2.exit_code == 0
|
||||
result3 = runner.invoke(cli, args, input=input, catch_exceptions=False)
|
||||
assert result3.exit_code == 0
|
||||
# At the end of this, there should be 2 embeddings
|
||||
db = sqlite_utils.Database(str(user_path / "embeddings.db"))
|
||||
assert db["embeddings"].count == 1
|
||||
|
||||
|
||||
def test_duplicate_content_embedded_only_once(embed_demo):
|
||||
# content_hash should avoid embedding the same content twice
|
||||
# per collection
|
||||
db = sqlite_utils.Database(memory=True)
|
||||
assert len(embed_demo.embedded_content) == 0
|
||||
collection = Collection("test", db, model_id="embed-demo")
|
||||
collection.embed("1", "hello world")
|
||||
assert len(embed_demo.embedded_content) == 1
|
||||
collection.embed("2", "goodbye world")
|
||||
assert db["embeddings"].count == 2
|
||||
assert len(embed_demo.embedded_content) == 2
|
||||
collection.embed("1", "hello world")
|
||||
assert db["embeddings"].count == 2
|
||||
assert len(embed_demo.embedded_content) == 2
|
||||
# The same string in another collection should be embedded
|
||||
c2 = Collection("test2", db, model_id="embed-demo")
|
||||
c2.embed("1", "hello world")
|
||||
assert db["embeddings"].count == 3
|
||||
assert len(embed_demo.embedded_content) == 3
|
||||
|
||||
# Same again for embed_multi
|
||||
collection.embed_multi(
|
||||
(("1", "hello world"), ("2", "goodbye world"), ("3", "this is new"))
|
||||
)
|
||||
# Should have only embedded one more thing
|
||||
assert db["embeddings"].count == 4
|
||||
assert len(embed_demo.embedded_content) == 4
|
||||
@@ -0,0 +1,19 @@
|
||||
import llm
|
||||
import pytest
|
||||
import numpy as np
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"array",
|
||||
(
|
||||
(0.0, 1.0, 1.5),
|
||||
(3423.0, 222.0, -1234.5),
|
||||
),
|
||||
)
|
||||
def test_roundtrip(array):
|
||||
encoded = llm.encode(array)
|
||||
decoded = llm.decode(encoded)
|
||||
assert decoded == array
|
||||
# Try with numpy as well
|
||||
numpy_decoded = np.frombuffer(encoded, "<f4")
|
||||
assert tuple(numpy_decoded.tolist()) == array
|
||||
@@ -0,0 +1,144 @@
|
||||
from click.testing import CliRunner
|
||||
from importlib.metadata import version
|
||||
from llm.cli import cli
|
||||
from llm.migrations import migrate
|
||||
from unittest import mock
|
||||
import os
|
||||
import yaml
|
||||
import sqlite_utils
|
||||
import textwrap
|
||||
|
||||
|
||||
def test_fragments_set_show_remove(user_path):
|
||||
runner = CliRunner()
|
||||
with runner.isolated_filesystem():
|
||||
with open("fragment1.txt", "w") as f:
|
||||
f.write("Hello fragment 1")
|
||||
|
||||
# llm fragments --aliases should return nothing
|
||||
assert runner.invoke(cli, ["fragments", "list", "--aliases"]).output == ""
|
||||
assert (
|
||||
runner.invoke(cli, ["fragments", "set", "f1", "fragment1.txt"]).exit_code
|
||||
== 0
|
||||
)
|
||||
result1 = runner.invoke(cli, ["fragments", "show", "f1"])
|
||||
assert result1.exit_code == 0
|
||||
assert result1.output == "Hello fragment 1\n"
|
||||
|
||||
# Should be in the list now
|
||||
def get_list():
|
||||
result2 = runner.invoke(cli, ["fragments", "list"])
|
||||
assert result2.exit_code == 0
|
||||
return yaml.safe_load(result2.output)
|
||||
|
||||
# And in llm fragments --aliases
|
||||
assert "f1" in runner.invoke(cli, ["fragments", "list", "--aliases"]).output
|
||||
|
||||
loaded1 = get_list()
|
||||
assert set(loaded1[0].keys()) == {
|
||||
"aliases",
|
||||
"content",
|
||||
"datetime_utc",
|
||||
"source",
|
||||
"hash",
|
||||
}
|
||||
assert loaded1[0]["content"] == "Hello fragment 1"
|
||||
assert loaded1[0]["aliases"] == ["f1"]
|
||||
|
||||
# Show should work against both alias and hash
|
||||
for key in ("f1", loaded1[0]["hash"]):
|
||||
result3 = runner.invoke(cli, ["fragments", "show", key])
|
||||
assert result3.exit_code == 0
|
||||
assert result3.output == "Hello fragment 1\n"
|
||||
|
||||
# But not for an invalid alias
|
||||
result4 = runner.invoke(cli, ["fragments", "show", "badalias"])
|
||||
assert result4.exit_code == 1
|
||||
assert "Fragment 'badalias' not found" in result4.output
|
||||
|
||||
# Remove that alias
|
||||
result5 = runner.invoke(cli, ["fragments", "remove", "f1"])
|
||||
assert result5.exit_code == 0
|
||||
# Should still be in list but no alias
|
||||
loaded2 = get_list()
|
||||
assert loaded2[0]["aliases"] == []
|
||||
assert loaded2[0]["content"] == "Hello fragment 1"
|
||||
|
||||
# And --aliases list should be empty
|
||||
assert runner.invoke(cli, ["fragments", "list", "--aliases"]).output == ""
|
||||
|
||||
|
||||
def test_fragments_list(user_path):
|
||||
runner = CliRunner()
|
||||
db = sqlite_utils.Database(str(user_path / "logs.db"))
|
||||
with db.conn:
|
||||
migrate(db)
|
||||
db["fragments"].insert_all(
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"content": "1",
|
||||
"datetime_utc": "2023-10-01T00:00:00Z",
|
||||
"source": "file1.txt",
|
||||
"hash": "hash1",
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"content": "2",
|
||||
"datetime_utc": "2022-10-01T00:00:00Z",
|
||||
"source": "file2.txt",
|
||||
"hash": "hash2",
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"content": "3",
|
||||
"datetime_utc": "2024-10-01T00:00:00Z",
|
||||
"source": "file3.txt",
|
||||
"hash": "hash3",
|
||||
},
|
||||
]
|
||||
)
|
||||
db["fragment_aliases"].insert(
|
||||
{
|
||||
"alias": "f1",
|
||||
"fragment_id": 1,
|
||||
}
|
||||
)
|
||||
result = runner.invoke(cli, ["fragments", "list"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output.strip() == (textwrap.dedent("""
|
||||
- hash: hash2
|
||||
aliases: []
|
||||
datetime_utc: '2022-10-01T00:00:00Z'
|
||||
source: file2.txt
|
||||
content: '2'
|
||||
- hash: hash1
|
||||
aliases:
|
||||
- f1
|
||||
datetime_utc: '2023-10-01T00:00:00Z'
|
||||
source: file1.txt
|
||||
content: '1'
|
||||
- hash: hash3
|
||||
aliases: []
|
||||
datetime_utc: '2024-10-01T00:00:00Z'
|
||||
source: file3.txt
|
||||
content: '3'
|
||||
""").strip())
|
||||
|
||||
|
||||
@mock.patch.dict(os.environ, {"OPENAI_API_KEY": "X"})
|
||||
def test_fragment_url_user_agent(mocked_openai_chat, user_path):
|
||||
mocked_openai_chat.add_response(
|
||||
url="https://example.com/fragment.txt",
|
||||
text="Hello from URL",
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["prompt", "-f", "https://example.com/fragment.txt"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Verify the User-Agent header was sent for the fragment URL request
|
||||
requests = mocked_openai_chat.get_requests()
|
||||
fragment_request = [r for r in requests if "example.com" in str(r.url)][0]
|
||||
llm_version = version("llm")
|
||||
expected_user_agent = f"llm/{llm_version} (https://llm.datasette.io/)"
|
||||
assert fragment_request.headers["User-Agent"] == expected_user_agent
|
||||
@@ -0,0 +1,112 @@
|
||||
from click.testing import CliRunner
|
||||
import json
|
||||
from llm.cli import cli
|
||||
import pathlib
|
||||
import pytest
|
||||
import sys
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
@pytest.mark.parametrize("env", ({}, {"LLM_USER_PATH": "/tmp/llm-keys-test"}))
|
||||
def test_keys_in_user_path(monkeypatch, env, user_path):
|
||||
for key, value in env.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["keys", "path"])
|
||||
assert result.exit_code == 0
|
||||
if env:
|
||||
expected = env["LLM_USER_PATH"] + "/keys.json"
|
||||
else:
|
||||
expected = user_path + "/keys.json"
|
||||
assert result.output.strip() == expected
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_keys_set(monkeypatch, tmpdir):
|
||||
user_path = tmpdir / "user/keys"
|
||||
monkeypatch.setenv("LLM_USER_PATH", str(user_path))
|
||||
keys_path = user_path / "keys.json"
|
||||
assert not keys_path.exists()
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["keys", "set", "openai"], input="foo")
|
||||
assert result.exit_code == 0
|
||||
assert keys_path.exists()
|
||||
# Should be chmod 600
|
||||
assert oct(keys_path.stat().mode)[-3:] == "600"
|
||||
content = keys_path.read_text("utf-8")
|
||||
assert json.loads(content) == {
|
||||
"// Note": "This file stores secret API credentials. Do not share!",
|
||||
"openai": "foo",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
||||
def test_keys_get(monkeypatch, tmpdir):
|
||||
user_path = tmpdir / "user/keys"
|
||||
monkeypatch.setenv("LLM_USER_PATH", str(user_path))
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["keys", "set", "openai"], input="fx")
|
||||
assert result.exit_code == 0
|
||||
result2 = runner.invoke(cli, ["keys", "get", "openai"])
|
||||
assert result2.exit_code == 0
|
||||
assert result2.output.strip() == "fx"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args", (["keys", "list"], ["keys"]))
|
||||
def test_keys_list(monkeypatch, tmpdir, args):
|
||||
user_path = str(tmpdir / "user/keys")
|
||||
monkeypatch.setenv("LLM_USER_PATH", user_path)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(cli, ["keys", "set", "openai"], input="foo")
|
||||
assert result.exit_code == 0
|
||||
result2 = runner.invoke(cli, args)
|
||||
assert result2.exit_code == 0
|
||||
assert result2.output.strip() == "openai"
|
||||
|
||||
|
||||
@pytest.mark.httpx_mock(
|
||||
assert_all_requests_were_expected=False, can_send_already_matched_responses=True
|
||||
)
|
||||
def test_uses_correct_key(mocked_openai_chat, monkeypatch, tmpdir):
|
||||
user_dir = tmpdir / "user-dir"
|
||||
pathlib.Path(user_dir).mkdir()
|
||||
keys_path = user_dir / "keys.json"
|
||||
KEYS = {
|
||||
"openai": "from-keys-file",
|
||||
"other": "other-key",
|
||||
}
|
||||
keys_path.write_text(json.dumps(KEYS), "utf-8")
|
||||
monkeypatch.setenv("LLM_USER_PATH", str(user_dir))
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "from-env")
|
||||
|
||||
def assert_key(key):
|
||||
request = mocked_openai_chat.get_requests()[-1]
|
||||
assert request.headers["Authorization"] == "Bearer {}".format(key)
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
# Called without --key uses stored key
|
||||
result = runner.invoke(cli, ["hello", "--no-stream"], catch_exceptions=False)
|
||||
assert result.exit_code == 0
|
||||
assert_key("from-keys-file")
|
||||
|
||||
# Called without --key and without keys.json uses environment variable
|
||||
keys_path.write_text("{}", "utf-8")
|
||||
result2 = runner.invoke(cli, ["hello", "--no-stream"], catch_exceptions=False)
|
||||
assert result2.exit_code == 0
|
||||
assert_key("from-env")
|
||||
keys_path.write_text(json.dumps(KEYS), "utf-8")
|
||||
|
||||
# Called with --key name-in-keys.json uses that value
|
||||
result3 = runner.invoke(
|
||||
cli, ["hello", "--key", "other", "--no-stream"], catch_exceptions=False
|
||||
)
|
||||
assert result3.exit_code == 0
|
||||
assert_key("other-key")
|
||||
|
||||
# Called with --key something-else uses exactly that
|
||||
result4 = runner.invoke(
|
||||
cli, ["hello", "--key", "custom-key", "--no-stream"], catch_exceptions=False
|
||||
)
|
||||
assert result4.exit_code == 0
|
||||
assert_key("custom-key")
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user