commit 75c67150d0d9ee02828256f5afc384648b82bc47 Author: wehub-resource-sync Date: Mon Jul 13 13:36:10 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.claude/skills/memu/SKILL.md b/.claude/skills/memu/SKILL.md new file mode 100644 index 0000000..0a5c703 --- /dev/null +++ b/.claude/skills/memu/SKILL.md @@ -0,0 +1,46 @@ +--- +name: memu +description: Give this agent persistent memory via memU. Use to memorize (save files/folders so context survives this session) when the user asks to remember or sync something, or after completing work worth keeping; and to retrieve (search memory from earlier sessions) when the user asks what is known about a person/project/topic or references context not in this conversation. +--- + +# memU: memorize and retrieve persistent memory + +memU compiles sources into a persistent local store (`./data/memu.sqlite3` + a browsable `./data/memory/` markdown tree, relative to CWD). What one session memorizes, the next can retrieve — always run from the project root so every command hits the same store. + +Use only the two commands below. Do not use the legacy `memu memorize` / `memu retrieve` commands. + +Both directions need an API key: make sure `OPENAI_API_KEY` (or `MEMU_LLM_PROVIDER` + its matching key) is set, and tell the user if it is missing. + +## Locate the CLI + +Use the first available runner: + +1. `memu` (installed via `pip install memu-py`) +2. `uvx --from memu-py memu` +3. `npx memu-cli` + +## Memorize + +```bash +memu memorize-workspace +``` + +- Incremental and safe to re-run: diffs against `/.memu_manifest.json`, so only added/modified files are processed and memory from deleted files is removed. +- Top-level directory decides the treatment: `chat/` → memory topics, `agent/` → skills, everything else → indexed workspace context (modality inferred per file from its extension). +- To memorize a single file, place (or copy) it into the workspace folder and sync — there is no separate single-file path. + +Report the printed diff (added/modified/deleted) to the user; pass `--json` if you need to parse the result. + +## Retrieve + +```bash +memu retrieve-workspace "" +``` + +Single-shot embedding search, no LLM calls. Returns JSON in three layers: + +- `segments` — the matched slices, ranked by similarity (check `score`) +- `files` — the memory/skill documents they belong to (usually what you want) +- `resources` — matching raw sources, when summaries are not enough + +If a query misses, retry with different phrasing or more specific terms. Low scores across the board usually mean nothing relevant is stored — say so rather than stretching weak matches. You can also read `./data/memory/MEMORY.md` / `SKILL.md` / `INDEX.md` directly for a browsable overview. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..096c80f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,75 @@ +name: Bug Report +description: Report a bug or issue with memU +title: "[BUG] " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to report a bug in memU! Please fill in the following details. + + - type: textarea + id: description + attributes: + label: Description + description: A clear and detailed description of the bug. + placeholder: "Enter a clear and concise description of what the bug is." + validations: + required: true + + - type: input + id: environment + attributes: + label: Environment + description: The environment where this bug occurred (e.g., operating system, CPU arch, etc.). + placeholder: "Enter details about the environment." + validations: + required: true + + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: What are the steps to reproduce this issue? + placeholder: "Enter step-by-step instructions to reproduce the issue." + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behavior + description: What should have happened instead? + placeholder: "Describe what you expected to happen." + validations: + required: true + + - type: input + id: version + attributes: + label: Version + description: What version of memU are you using? + placeholder: e.g., v1.0.0 + validations: + required: true + + - type: dropdown + id: severity + attributes: + label: Severity + description: How severe is the bug? + options: + - Critical + - Major + - Minor + validations: + required: true + + - type: textarea + id: additional_info + attributes: + label: Additional Information + description: Any other context or screenshots related to the bug. + placeholder: "Enter additional context or information." + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..6a94220 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: GitHub Discussions + url: https://github.com/NevaMind-AI/memU/discussions + about: For questions and general discussions about memU + - name: Documentation + url: https://github.com/NevaMind-AI/memU/blob/main/README.md + about: Check out the memU documentation diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..7149c6c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,61 @@ +name: Feature Request +description: Request a feature for memU +title: "[FEATURE] " +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to request a feature for memU! Please fill in the following details. + + - type: textarea + id: description + attributes: + label: Description + description: A clear and detailed description of the feature request. + placeholder: "Enter a clear and concise description of what the feature request is." + validations: + required: true + + - type: textarea + id: motivation + attributes: + label: Motivation + description: Why do we need this feature? What problem does it solve? + placeholder: "Explain the use case and why this feature would be valuable." + validations: + required: true + + - type: dropdown + id: platform + attributes: + label: Platform + description: Which platform(s) does this feature apply to? + options: + - Platform Independent + - Linux + - macOS + - Windows + validations: + required: true + + - type: dropdown + id: priority + attributes: + label: Priority + description: How important is this feature? + options: + - Critical + - Major + - Minor + validations: + required: true + + - type: textarea + id: additional_info + attributes: + label: Additional Information + description: Any other context or screenshots related to the feature request. + placeholder: "Enter additional context or information." + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/hackathon_task.yml b/.github/ISSUE_TEMPLATE/hackathon_task.yml new file mode 100644 index 0000000..77bba5c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/hackathon_task.yml @@ -0,0 +1,65 @@ +name: Hackathon Task +description: The official designated task for the memU track in the 2026 New Year Challenge +title: "[2026NewYearChallenge] " +labels: ["hackathon-task"] +assignees: [] +body: + - type: markdown + attributes: + value: | + Thank you for taking the time to join the challenge, and best of luck! + + - type: dropdown + id: label_hint + attributes: + label: Label + description: Pick the most appropriate label for this task. + options: + - hackathon-task + validations: + required: true + + - type: input + id: assignees_hint + attributes: + label: Assignees + description: GitHub usernames to assign (comma-separated). If empty, maintainers will triage. + placeholder: "e.g., alice,bob" + validations: + required: false + + - type: textarea + id: description + attributes: + label: Description* + description: What will this task implement? + placeholder: "Enter a clear and concise description of the task you want to implement." + validations: + required: true + + - type: textarea + id: requirements + attributes: + label: Requirements (optional) + description: What will your planned pull request include? (APIs, modules, tests, docs, etc.) + placeholder: | + - [ ] Implementation details + - [ ] Tests + - [ ] Docs / examples + - [ ] Compatibility notes + validations: + required: false + + - type: textarea + id: review_criteria + attributes: + label: Review Criteria (optional) + description: How will this task be evaluated? + placeholder: | + Example: + - Correctness (passes tests, no regressions) + - Quality (readability, modularity) + - DX (good docs / examples) + - Impact (useful for memU users) + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/improvement_suggestion.yml b/.github/ISSUE_TEMPLATE/improvement_suggestion.yml new file mode 100644 index 0000000..126fb2c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/improvement_suggestion.yml @@ -0,0 +1,63 @@ +name: Improvement Suggestion +description: Suggest an improvement for memU +title: "[IMPROVEMENT] " +labels: ["enhancement", "improvement"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to suggest an improvement for memU! Please fill in the following details. + + - type: textarea + id: description + attributes: + label: Description + description: A clear and detailed description of the improvement suggestion. + placeholder: "Enter a clear and concise description of what you'd like to improve." + validations: + required: true + + - type: textarea + id: current_behavior + attributes: + label: Current Behavior + description: How does it work currently? + placeholder: "Describe the current behavior or implementation." + validations: + required: false + + - type: textarea + id: proposed_improvement + attributes: + label: Proposed Improvement + description: How would you like it to work? + placeholder: "Describe your proposed improvement." + validations: + required: true + + - type: textarea + id: benefits + attributes: + label: Benefits + description: What are the benefits of this improvement? + placeholder: "Explain why this improvement would be valuable." + validations: + required: false + + - type: input + id: version + attributes: + label: Version + description: What version of memU are you using? + placeholder: e.g., v1.0.0 + validations: + required: false + + - type: textarea + id: additional_info + attributes: + label: Additional Information + description: Any other context or screenshots related to the improvement. + placeholder: "Enter additional context or information." + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..9498f92 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,44 @@ +## 📝 Pull Request Summary + +Please provide a short summary explaining the purpose of this PR. + +--- + +## ✅ What does this PR do? +- Clearly describe the change introduced. +- Mention the motivation or problem it solves. + +--- + +## 🤔 Why is this change needed? +- Explain the context or user impact. +- Link any relevant issue or discussion. + +--- + +## 🔍 Type of Change +Please check what applies: + +- [ ] Bug fix +- [ ] New feature +- [ ] Documentation update +- [ ] Refactor / cleanup +- [ ] Other (please explain) + +--- + +## ✅ PR Quality Checklist + +- [ ] PR title follows an allowed format (for example `feat:`, `fix:`, `docs:`, `memory base:`) +- [ ] Changes are limited in scope and easy to review +- [ ] Documentation updated where applicable +- [ ] No breaking changes (or clearly documented) +- [ ] Related issues or discussions linked + +--- + +## 📌 Optional + +- [ ] Screenshots or examples added (if applicable) +- [ ] Edge cases considered +- [ ] Follow-up tasks mentioned diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..12fa998 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,35 @@ +name: build + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + strategy: + matrix: + python-version: ["3.13"] + + steps: + - uses: actions/checkout@v6 + + - name: Install uv & set Python + uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.python-version }} + enable-cache: true + cache-python: true + + - name: Ensure Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} + + - name: Sync dependencies + run: | + uv sync --frozen --extra document + uv run pre-commit install + + - name: Run style checks + run: uv run make check + + - name: Run tests + run: uv run make test diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml new file mode 100644 index 0000000..abfee29 --- /dev/null +++ b/.github/workflows/pr-title.yml @@ -0,0 +1,47 @@ +name: pr-title + +on: + pull_request_target: + types: + - opened + - edited + - reopened + - synchronize + +permissions: + contents: read + statuses: write + +jobs: + validate: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + headerPattern: '^([\w ]*)(?:\(([\w$.\-*/ ]*)\))?: (.*)$' + headerPatternCorrespondence: type, scope, subject + types: | + build + chore + ci + docs + feat + fix + memory base + perf + refactor + revert + style + test + - name: Disallow trailing ellipsis + shell: bash + env: + PR_TITLE: ${{ github.event.pull_request.title }} + run: | + if [[ "$PR_TITLE" == *... ]]; then + echo "::error title=Invalid PR title::PR title must not end with an ellipsis (...)" + exit 1 + fi diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..efaa9e1 --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,160 @@ +on: + push: + branches: + - main + +permissions: + contents: write + issues: write + pull-requests: write + +name: release-please + +jobs: + release-please: + runs-on: ubuntu-latest + outputs: + releases_created: ${{ steps.release.outputs.releases_created }} + tag_name: ${{ steps.release.outputs.tag_name }} + steps: + - uses: googleapis/release-please-action@v4 + id: release + with: + release-type: python + target-branch: main + + build-wheels: + name: build wheels (${{ matrix.label }}) + runs-on: ${{ matrix.os }} + needs: release-please + if: ${{ needs.release-please.outputs.releases_created == 'true' && needs.release-please.outputs.tag_name != '' }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + label: linux-x86_64 + target: x86_64-unknown-linux-gnu + extra-args: "--compatibility manylinux_2_39" + python-version: "3.13" + - os: ubuntu-latest + label: linux-aarch64 + target: aarch64-unknown-linux-gnu + extra-args: "--compatibility manylinux_2_39" + python-version: "3.13" + - os: macos-15-intel + label: macos-x86_64 + target: "" + extra-args: "" + python-version: "3.13" + - os: macos-latest + label: macos-aarch64 + target: "" + extra-args: "" + python-version: "3.13" + - os: windows-latest + label: windows-x86_64 + target: "" + extra-args: "" + python-version: "3.13" + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.python-version }} + + - name: Install maturin + run: uv tool install maturin + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Setup cross-compilation for Linux ARM64 + if: matrix.label == 'linux-aarch64' + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + + - name: Build wheel + run: uvx maturin build --release --out dist ${{ matrix.extra-args }} ${{ matrix.target && format('--target {0}', matrix.target) || '' }} + env: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc + + - name: Upload wheel artifact + uses: actions/upload-artifact@v6 + with: + name: wheels-${{ matrix.label }} + path: dist/*.whl + if-no-files-found: error + + build-sdist: + name: build sdist + runs-on: ubuntu-latest + needs: release-please + if: ${{ needs.release-please.outputs.releases_created == 'true' && needs.release-please.outputs.tag_name != '' }} + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install maturin + run: uv tool install maturin + + - name: Build sdist + run: uvx maturin sdist --out dist + + - name: Upload sdist artifact + uses: actions/upload-artifact@v6 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: error + + publish: + name: publish release artifacts + runs-on: ubuntu-latest + needs: + - release-please + - build-wheels + - build-sdist + if: ${{ needs.release-please.outputs.releases_created == 'true' && needs.release-please.outputs.tag_name != '' }} + environment: pypi + permissions: + id-token: write + contents: write + steps: + - name: Download wheel artifacts + uses: actions/download-artifact@v7 + with: + pattern: wheels-* + merge-multiple: true + path: dist + + - name: Download sdist artifact + uses: actions/download-artifact@v7 + with: + name: sdist + path: dist + + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: dist/* + tag: ${{ needs.release-please.outputs.tag_name }} + file_glob: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c6b5ec5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,218 @@ +data/ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.env.* +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be added to the global gitignore or merged into this project gitignore. For a PyCharm +# project, it is common to ignore sensitive information such as project configurations. +.idea/ + +# Database files +*.db +*.sqlite +*.sqlite3 +persona_memory.db +persona_conversations.db +basic_memory_demo.db +demo_memory.db +demo_conversations.db +chatbot_memory.db +chatbot_conversations.db +integration_memory.db +integration_conversations.db +conversation_demo.db + +# Log files +*.log + +# Temporary files +tmp/ +temp/ +.tmp/ + +# OS specific +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# IDE +.vscode/ +*.swp +*.swo +*~ + +# Model files (if using local models) +models/ +*.bin +*.pt +*.pth +*.onnx + +# API keys and secrets +.env.local +.env.*.local +secrets.txt +api_keys.txt +.cursor + +.local/ + +# Locally built Rust extension (offline dev convenience; normally built by maturin) +src/memu/_core.abi3.so diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b7dffdd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v6.0.0" + hooks: + - id: check-case-conflict + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + - id: check-json + - id: pretty-format-json + args: [--autofix, --no-sort-keys] + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.14.3" + hooks: + - id: ruff + args: [--exit-non-zero-on-fix] + - id: ruff-format diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..925fb80 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,165 @@ +# AGENTS.md + +Operational guide for AI coding agents working in this repository. + +## Mission + +Ship small, verified feature and bugfix changes while preserving memU's current architecture. +Treat `docs/architecture.md` as the runtime source of truth and keep this file aligned with it. + +Core invariants: + +- `MemoryService` is the composition root for config, storage, LLM/VLM/embedding clients, workflows, interceptors, and memory file export. +- Public APIs should remain stable unless the requested change explicitly requires an API change. +- Major behavior runs through workflows: `memorize`, `retrieve_rag`, `retrieve_llm`, and CRUD/patch operations. +- Workflow steps must keep explicit `requires`/`produces` contracts and declared capability tags. +- Storage is pluggable across `inmemory`, `sqlite`, and `postgres`; repository contract changes require backend parity. +- Scope data comes from `UserConfig.model`; validate `where` filters and do not bypass scope filtering. +- LLM routing is profile-based across chat, embedding, VLM, and custom profiles. +- The exported memory workspace should stay human-readable and agent-traversable: `INDEX.md`, `MEMORY.md`, `memory/`, `resource/`, and `skill/`. + +## First Move + +Before editing, classify the request by the layer it touches: + +- Service/runtime wiring: `src/memu/app/service.py` +- Memorize pipeline: `src/memu/app/memorize.py` +- Retrieve pipelines: `src/memu/app/retrieve.py` +- CRUD/patch behavior: `src/memu/app/crud.py` +- Config models/defaults: `src/memu/app/settings.py` +- Workflow engine and pipeline mutation: `src/memu/workflow/*` +- Memory file export/synthesis: `src/memu/app/memory_files.py`, `src/memu/memory_fs/*`, `src/memu/prompts/memory_fs*` +- Storage protocols/factory: `src/memu/database/interfaces.py`, `src/memu/database/factory.py` +- In-memory backend: `src/memu/database/inmemory/*` +- SQLite backend: `src/memu/database/sqlite/*` +- Postgres backend: `src/memu/database/postgres/*` +- Vector math/ranking: `src/memu/vector.py` +- LLM clients, wrappers, interceptors: `src/memu/llm/*` +- Embedding clients: `src/memu/embedding/*` +- Vision clients: `src/memu/vlm/*` +- Integrations/client surfaces: `src/memu/integrations/*`, `src/memu/client/*` +- Tests: `tests/*` +- User-facing docs/examples: `README.md`, `readme/*`, `docs/*`, `examples/*` + +Read the relevant implementation and nearby tests first. Prefer local patterns over new abstractions. + +## Implementation Rules + +- Keep changes narrow and localized to the affected layer. +- Preserve async behavior and avoid sync work in async paths unless existing code already does it. +- Prefer adding or modifying a workflow step over embedding new flow logic directly in `MemoryService`. +- When adding configurable behavior, update typed settings and wire it through step config or service config. +- Preserve result shapes unless the task explicitly asks for a breaking change. +- Maintain type hints and mypy compatibility. +- Keep provider-specific logic inside provider/backend modules. +- Keep storage-neutral logic outside concrete backends; vector ranking helpers belong in shared modules such as `memu.vector`. +- Do not duplicate client caching. Use the existing client pool/profile routing patterns. +- Do not silently swallow errors that should be visible to callers or tests. +- Avoid broad refactors unless they are required to make the requested change safe. + +## Workflow Guidance + +For memorize work: + +- Keep modality routing in the preprocessing pipeline. +- Use VLM clients for image/video paths and chat clients for conversation/document/audio paths. +- Preserve the flow: ingest resource, preprocess, extract items, dedupe/merge, categorize, persist index, build response. +- If rich document ingestion changes, consider the optional `document` extra and MarkItDown behavior. + +For retrieve work: + +- Check both `retrieve_rag` and `retrieve_llm` when behavior is shared. +- Validate `where` filters against the configured user model before querying. +- Keep category, item, and resource recall responsibilities separated. +- Preserve early-stop/sufficiency behavior when modifying ranking or response build steps. + +For CRUD/patch work: + +- Respect repository contracts and scope fields. +- Preserve existing output shapes for list/create/update/delete operations. +- Add backend coverage when contracts change. + +For memory file export: + +- Keep raw resources copied verbatim into `resource/`. +- Keep category memory split under `memory/.md`. +- Keep synthesized skills under `skill//SKILL.md`. +- Keep root files as indexes/overviews, not hidden sources of untracked state. +- Treat synthesized skill output as derived from source descriptions, not from unrelated runtime state. + +## Backend Parity + +If a repository method, model field, filter behavior, or vector search contract changes: + +1. Update the protocol/interface. +2. Update `inmemory`, `sqlite`, and `postgres` implementations where applicable. +3. Add or extend conformance tests. +4. Check migrations/bootstrap behavior for SQL backends. +5. Verify fallback behavior when pgvector or optional extras are unavailable. + +## LLM, VLM, And Embedding Routing + +- Chat-like work uses LLM profiles. +- Embedding work uses embedding profiles and dedicated embedding clients. +- Image/video understanding uses VLM clients derived from LLM profiles unless explicitly configured otherwise. +- Step config should select profiles with existing keys such as `llm_profile`, `chat_llm_profile`, and `embed_llm_profile`. +- Keep interceptors and usage metadata behavior consistent across capabilities. +- Do not add embeddings back into chat clients; embedding is intentionally decoupled. + +## Feature Checklist + +1. Locate the affected flow, backend, integration, or export path. +2. Update settings/defaults only if behavior is configurable. +3. Wire the change through `MemoryService`, pipeline registration, or step config as appropriate. +4. Implement repository/backend changes for all impacted providers. +5. Add focused tests for the happy path and edge cases. +6. Update docs/examples for user-visible behavior. +7. Add or update ADRs under `docs/adr/` for architectural changes. + +## Bug Fix Checklist + +1. Reproduce the bug with an existing or new failing test. +2. Fix the smallest correct layer. +3. Add a regression test that fails before and passes after. +4. Check cross-backend effects when storage behavior changes. +5. Check both retrieval modes when recall/ranking/response behavior changes. +6. Confirm no unintended public API or output-shape changes. + +## Testing And Validation + +Use `uv` for local runs. + +- Setup: `make install` +- Run all tests: `make test` +- Run focused tests: `uv run python -m pytest tests/.py` +- Full quality checks: `make check` + +At minimum, run targeted tests for touched code. Run `make check` for broad, cross-cutting, or release-critical changes. +If a required check cannot be run, say why in the final summary. + +Useful focused areas: + +- Storage parity: `tests/test_backend_conformance.py`, `tests/test_inmemory.py`, `tests/test_sqlite.py`, `tests/test_postgres.py` +- Retrieval/vector behavior: `tests/test_vector.py`, `tests/test_salience.py`, retrieve-related tests +- Multimodal preprocessing: `tests/test_conversation_preprocess.py`, `tests/test_document_text.py`, `tests/test_audio_preprocess.py`, `tests/test_vlm_preprocess.py` +- Memory file export/synthesis: `tests/test_memory_files.py`, `tests/test_memory_fs_synthesis.py` +- Provider routing: `tests/test_embedding.py`, `tests/test_openrouter.py`, `tests/test_openai_max_tokens.py`, `tests/test_lazyllm.py` +- Integrations/tool memory: `tests/test_tool_memory.py`, LangGraph-related tests + +## Documentation Rules + +- Update `README.md`, localized `readme/*`, `docs/*`, or examples when behavior visible to users changes. +- Keep architecture docs current when composition, workflow contracts, backends, or profile routing changes. +- Keep examples runnable and aligned with the public API. +- Do not document speculative features as implemented. + +## Done Criteria + +Before finishing: + +- Code compiles for touched paths. +- Tests for changed behavior pass. +- New behavior has test coverage. +- User-visible or architectural behavior is documented. +- Backend parity has been considered and implemented where needed. +- No unrelated files were modified. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..04f3d16 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,290 @@ +# Changelog + +## [1.5.1](https://github.com/NevaMind-AI/memU/compare/v1.5.0...v1.5.1) (2026-03-23) + + +### Bug Fixes + +* use correct interpolation in alembic url ([#390](https://github.com/NevaMind-AI/memU/issues/390)) ([fd87ceb](https://github.com/NevaMind-AI/memU/commit/fd87ceb558eaa800aeb694b045847971958f23a3)) + +## [1.5.0](https://github.com/NevaMind-AI/memU/compare/v1.4.0...v1.5.0) (2026-03-17) + + +### Features + +* add non-propagate option for memory patch ([#386](https://github.com/NevaMind-AI/memU/issues/386)) ([3b67458](https://github.com/NevaMind-AI/memU/commit/3b67458c6325b4fcdaccbc34f6e0fea491b7ba96)) +* **http:** add HTTP proxy support for LLM & embedding clients ([#310](https://github.com/NevaMind-AI/memU/issues/310)) ([a3d45e2](https://github.com/NevaMind-AI/memU/commit/a3d45e2f4174702fd8fd83cd13862285cd47cad9)) + + +### Bug Fixes + +* httpx base_url path discarded when endpoint starts with / ([#328](https://github.com/NevaMind-AI/memU/issues/328), [#336](https://github.com/NevaMind-AI/memU/issues/336), [#341](https://github.com/NevaMind-AI/memU/issues/341), [#329](https://github.com/NevaMind-AI/memU/issues/329)) ([#344](https://github.com/NevaMind-AI/memU/issues/344)) ([9e31ef2](https://github.com/NevaMind-AI/memU/commit/9e31ef235d46317f383613825090c52943cb55a2)) + + +### Documentation + +* add architecture explanation and adrs ([#353](https://github.com/NevaMind-AI/memU/issues/353)) ([8676a27](https://github.com/NevaMind-AI/memU/commit/8676a2721917f4e0baee6d01a9d6b95bd642be97)) +* add pull request template to improve contribution quality ([#261](https://github.com/NevaMind-AI/memU/issues/261)) ([2256119](https://github.com/NevaMind-AI/memU/commit/2256119fb2d177890c7f738c926f437a355e7359)) +* Update Discord link in README.md ([#377](https://github.com/NevaMind-AI/memU/issues/377)) ([163d050](https://github.com/NevaMind-AI/memU/commit/163d050299b77143226e9727f67d4826c9a69f92)) +* update memU Bot section with GitHub reference and "Now open source" ([#366](https://github.com/NevaMind-AI/memU/issues/366)) ([dc6880a](https://github.com/NevaMind-AI/memU/commit/dc6880a793d40b04cd599298a110af5f0f0b9cb5)) + +## [1.4.0](https://github.com/NevaMind-AI/memU/compare/v1.3.0...v1.4.0) (2026-02-06) + + +### Features + +* Add inline memory item references in category summaries ([#202](https://github.com/NevaMind-AI/memU/issues/202)) ([#205](https://github.com/NevaMind-AI/memU/issues/205)) ([5213571](https://github.com/NevaMind-AI/memU/commit/5213571b218d85784e0771f7a721eafd7da1c1ff)) +* Add salience-aware memory with reinforcement tracking ([#186](https://github.com/NevaMind-AI/memU/issues/186)) ([#206](https://github.com/NevaMind-AI/memU/issues/206)) ([2bdbcce](https://github.com/NevaMind-AI/memU/commit/2bdbcce1a87ae1017d5930901fb0ae8d2924dcee)) +* Add Tool Memory type with specialized metadata and statistics ([#247](https://github.com/NevaMind-AI/memU/issues/247)) ([4e8a035](https://github.com/NevaMind-AI/memU/commit/4e8a03578641afd0e07f9700629dff8d91d2b3fb)) + + +### Bug Fixes + +* add chat() API and stop misusing summarize() ([#208](https://github.com/NevaMind-AI/memU/issues/208)) ([be0a5c7](https://github.com/NevaMind-AI/memU/commit/be0a5c73250f0a21ad5e0d39b0e66e3018809e0f)) +* remove unused type: ignore comment and add lazyllm mypy override ([#275](https://github.com/NevaMind-AI/memU/issues/275)) ([0e490f7](https://github.com/NevaMind-AI/memU/commit/0e490f7333feecffaef0901cccb1c9a5dbb7bafb)) +* **video:** cleanup temp files on extraction failure ([#295](https://github.com/NevaMind-AI/memU/issues/295)) ([16b65e5](https://github.com/NevaMind-AI/memU/commit/16b65e54aef69dfe617835ca534c12752e06eda8)) + + +### Documentation + +* file system in readme ([#301](https://github.com/NevaMind-AI/memU/issues/301)) ([ee27d8a](https://github.com/NevaMind-AI/memU/commit/ee27d8aa5e42d249963b08d5775338080b0255f8)) +* fix name ([#291](https://github.com/NevaMind-AI/memU/issues/291)) ([fd0d179](https://github.com/NevaMind-AI/memU/commit/fd0d17998f6c0c9ba5c7e91964c39ee909f2b366)) +* fix readme lint ([#290](https://github.com/NevaMind-AI/memU/issues/290)) ([a7a5516](https://github.com/NevaMind-AI/memU/commit/a7a55169ecbc1579ae87dc68c5e09b1d62f07b56)) +* lint error ([#287](https://github.com/NevaMind-AI/memU/issues/287)) ([7f61dc7](https://github.com/NevaMind-AI/memU/commit/7f61dc72ffbadf849f227cf314bba7ce7964761b)) +* readme memubot ([#289](https://github.com/NevaMind-AI/memU/issues/289)) ([2f30798](https://github.com/NevaMind-AI/memU/commit/2f30798367e3591107e1e66a184d0dd4ad6c2f93)) +* Update README.md ([#300](https://github.com/NevaMind-AI/memU/issues/300)) ([1075d7c](https://github.com/NevaMind-AI/memU/commit/1075d7c9ec3a272846c9fdb8c22ec58ff73cf6e7)) + +## [1.3.0](https://github.com/NevaMind-AI/memU/compare/v1.2.0...v1.3.0) (2026-01-29) + + +### Features + +* add happened at and extra fields to memory item ([#262](https://github.com/NevaMind-AI/memU/issues/262)) ([77938e9](https://github.com/NevaMind-AI/memU/commit/77938e9c282e1c0eda11088675c35975d85c4ff0)) +* add proactive example ([#268](https://github.com/NevaMind-AI/memU/issues/268)) ([d3d1de1](https://github.com/NevaMind-AI/memU/commit/d3d1de1d9b0f45d9b14479cbaa4462458b172005)) +* add Sealos support agent use case (Track G) ([#255](https://github.com/NevaMind-AI/memU/issues/255)) ([8fbdf3c](https://github.com/NevaMind-AI/memU/commit/8fbdf3c301f74f2aa85604837e00bb305b8801ec)) +* integrate LazyLLM to provide more llm services ([#265](https://github.com/NevaMind-AI/memU/issues/265)) ([c03f639](https://github.com/NevaMind-AI/memU/commit/c03f639677d6c897b75dfe28d0cd92d5b5270957)) +* **integrations:** Add LangGraph Adapter for MemU (Track A) ([#258](https://github.com/NevaMind-AI/memU/issues/258)) ([50b5502](https://github.com/NevaMind-AI/memU/commit/50b5502ebcacd86401f98b1bb7e5a6577fab7126)) +* **llm:** add Grok (xAI) integration ([#179](https://github.com/NevaMind-AI/memU/issues/179)) ([#236](https://github.com/NevaMind-AI/memU/issues/236)) ([1e16309](https://github.com/NevaMind-AI/memU/commit/1e1630948af535f8ed9d608e6c4f9d2748d4ce8e)) +* Openrouter integration as backend provider ([#182](https://github.com/NevaMind-AI/memU/issues/182)) ([cba667a](https://github.com/NevaMind-AI/memU/commit/cba667a56daca5093c9b79a598c7d2ffda813756)) + + +### Bug Fixes + +* memory type & proactive example ([#272](https://github.com/NevaMind-AI/memU/issues/272)) ([710f14d](https://github.com/NevaMind-AI/memU/commit/710f14d4b171c5cde609a9dc2caf454681ab94b3)) +* proactive examples ([#273](https://github.com/NevaMind-AI/memU/issues/273)) ([603ae12](https://github.com/NevaMind-AI/memU/commit/603ae122b94741bb350656086960c4e2bb868c2a)) + + +### Documentation + +* Add link to memU bot ([#276](https://github.com/NevaMind-AI/memU/issues/276)) ([2f84231](https://github.com/NevaMind-AI/memU/commit/2f842311ed592c1a45cdb9e24f7a128da97a0a39)) +* multilingual readme ([#271](https://github.com/NevaMind-AI/memU/issues/271)) ([200f47a](https://github.com/NevaMind-AI/memU/commit/200f47a15ff7d05fc435abe1d2cefbb3774548fe)) +* Update memU bot link to include full URL ([#277](https://github.com/NevaMind-AI/memU/issues/277)) ([6874eaf](https://github.com/NevaMind-AI/memU/commit/6874eaf278e910d07a4427dd4c06f9c4c21283ce)) +* update readme ([#266](https://github.com/NevaMind-AI/memU/issues/266)) ([16ae534](https://github.com/NevaMind-AI/memU/commit/16ae534675a5e0711256a5e2147b9190fd8b2524)) +* update README ([#270](https://github.com/NevaMind-AI/memU/issues/270)) ([b531d39](https://github.com/NevaMind-AI/memU/commit/b531d39e5538449b31cb212aea1deea24e12180f)) + +## [1.2.0](https://github.com/NevaMind-AI/memU/compare/v1.1.2...v1.2.0) (2026-01-14) + + +### Features + +* clear memory ([#239](https://github.com/NevaMind-AI/memU/issues/239)) ([7da36da](https://github.com/NevaMind-AI/memU/commit/7da36da57d7013df213e537b1f238f5a526e69d9)) +* **database:** add SQLite storage backend ([#201](https://github.com/NevaMind-AI/memU/issues/201)) ([4b899d7](https://github.com/NevaMind-AI/memU/commit/4b899d7b768fcd48b6456234800033e3b2e3173c)) +* improve issue template ([#199](https://github.com/NevaMind-AI/memU/issues/199)) ([abe0f1b](https://github.com/NevaMind-AI/memU/commit/abe0f1b77d5f8294ff45c9f57f5aeb6de33977d6)) +* optimize topk pick function ([#196](https://github.com/NevaMind-AI/memU/issues/196)) ([b474c54](https://github.com/NevaMind-AI/memU/commit/b474c54a7eea06ba56844d6482d5e78b3cb3f020)) +* workflow step interceptor ([#240](https://github.com/NevaMind-AI/memU/issues/240)) ([bf2ac96](https://github.com/NevaMind-AI/memU/commit/bf2ac96f1cb26b196f27b543d4c3e157f408f343)) + + +### Bug Fixes + +* allow nullable resource_id in Postgres items ([#232](https://github.com/NevaMind-AI/memU/issues/232)) ([ae2ffbb](https://github.com/NevaMind-AI/memU/commit/ae2ffbb0bcd69170875a47cebf322b71a15f4bd3)) +* correct coverage source to track memu package instead of tests ([#220](https://github.com/NevaMind-AI/memU/issues/220)) ([2460bbd](https://github.com/NevaMind-AI/memU/commit/2460bbd2a2c4df5b3664db81d89886b926b51031)) +* correct typo in PromptBlock class label attribute ([#231](https://github.com/NevaMind-AI/memU/issues/231)) ([d69053c](https://github.com/NevaMind-AI/memU/commit/d69053cece3467dd7c8cbf0634e72447649095f7)) +* default embed size ([#192](https://github.com/NevaMind-AI/memU/issues/192)) ([144fd32](https://github.com/NevaMind-AI/memU/commit/144fd32cfd6917b08f29cf56f251f239c360d2ae)) +* readme partners link & github issue link ([#198](https://github.com/NevaMind-AI/memU/issues/198)) ([849f881](https://github.com/NevaMind-AI/memU/commit/849f881f004b0772f372069e40f937731c114c89)) + + +### Documentation + +* add Chinese translation of README ([#212](https://github.com/NevaMind-AI/memU/issues/212)) ([d15ecf9](https://github.com/NevaMind-AI/memU/commit/d15ecf97647d8fd9f1d2ae95008a08bbba93a44a)) +* add How to Contribute section to README ([#215](https://github.com/NevaMind-AI/memU/issues/215)) ([bf44de7](https://github.com/NevaMind-AI/memU/commit/bf44de722965fd611c0dc585712eac587caa9e6c)) +* **readme:** update Chinese README with fixes and improvements ([#221](https://github.com/NevaMind-AI/memU/issues/221)) ([75a8f65](https://github.com/NevaMind-AI/memU/commit/75a8f651bb4b491ab5b0363de789c0dcd0291287)) + +## [1.1.2](https://github.com/NevaMind-AI/memU/compare/v1.1.1...v1.1.2) (2026-01-08) + + +### Bug Fixes + +* custom memory type default prompt ([#169](https://github.com/NevaMind-AI/memU/issues/169)) ([5a0032f](https://github.com/NevaMind-AI/memU/commit/5a0032fc0f29229524d0356d454a3f5991e04f7b)) +* prompt & item fallback ([#183](https://github.com/NevaMind-AI/memU/issues/183)) ([bc95ade](https://github.com/NevaMind-AI/memU/commit/bc95adeb26789104c0bbe199e126cf05def27941)) + + +### Documentation + +* add docs folder ([#181](https://github.com/NevaMind-AI/memU/issues/181)) ([919d2ca](https://github.com/NevaMind-AI/memU/commit/919d2caef23107d539c36f30c44d4fb5aa38b324)) +* fix issue template dropdown ([#167](https://github.com/NevaMind-AI/memU/issues/167)) ([14d0333](https://github.com/NevaMind-AI/memU/commit/14d03331ed14f1e593fdebbf561189a3775651be)) +* issue template fix ([#165](https://github.com/NevaMind-AI/memU/issues/165)) ([5d91237](https://github.com/NevaMind-AI/memU/commit/5d912372e1bf70d7ff573cd5b92e917118048e25)) + +## [1.1.1](https://github.com/NevaMind-AI/memU/compare/v1.1.0...v1.1.1) (2026-01-07) + + +### Bug Fixes + +* ensure both Linux x86_64 and ARM64 wheels are built ([#162](https://github.com/NevaMind-AI/memU/issues/162)) ([51c9ea4](https://github.com/NevaMind-AI/memU/commit/51c9ea4335a1e9eac227c79783d7aa8cca2b883b)) + + +### Documentation + +* add custom LLM and embedding configuration guide ([#160](https://github.com/NevaMind-AI/memU/issues/160)) ([29c414a](https://github.com/NevaMind-AI/memU/commit/29c414a84d1c9c2f989e2165d0b6508c3fe62862)) +* add template ([#158](https://github.com/NevaMind-AI/memU/issues/158)) ([b79a78d](https://github.com/NevaMind-AI/memU/commit/b79a78d25b96e670888c18b8f09277db33803865)) + +## [1.1.0](https://github.com/NevaMind-AI/memU/compare/v1.0.1...v1.1.0) (2026-01-07) + + +### Features + +* add Linux ARM64 (aarch64) build target ([#156](https://github.com/NevaMind-AI/memU/issues/156)) ([0c90fcf](https://github.com/NevaMind-AI/memU/commit/0c90fcfb19fc3e91b951f4ba7454798e4b83e42c)) + + +### Bug Fixes + +* llm client profile & wrapper ([#157](https://github.com/NevaMind-AI/memU/issues/157)) ([e55c668](https://github.com/NevaMind-AI/memU/commit/e55c66847eac3ceaf276587d58b76d953d7be9f8)) + + +### Documentation + +* remove legacy docs ([#154](https://github.com/NevaMind-AI/memU/issues/154)) ([11fda41](https://github.com/NevaMind-AI/memU/commit/11fda41dda8a5c38b790d3c2fb7053b57b16606e)) + +## [1.0.1](https://github.com/NevaMind-AI/memU/compare/v1.0.0...v1.0.1) (2026-01-06) + + +### Bug Fixes + +* get embedding client ([#152](https://github.com/NevaMind-AI/memU/issues/152)) ([76716a4](https://github.com/NevaMind-AI/memU/commit/76716a4f00127a3cc21444996f32a9cf810c9282)) +* Readme incomplete ([#148](https://github.com/NevaMind-AI/memU/issues/148)) ([f8bc748](https://github.com/NevaMind-AI/memU/commit/f8bc748b9ebfb652ca8a5e06edbbe7eb648ed4f6)) + +## [1.0.0](https://github.com/NevaMind-AI/memU/compare/v0.9.0...v1.0.0) (2026-01-05) + + +### ⚠ BREAKING CHANGES + +* v1.0.0 ([#147](https://github.com/NevaMind-AI/memU/issues/147)) + +### Bug Fixes + +* v1.0.0 ([#147](https://github.com/NevaMind-AI/memU/issues/147)) ([23f37ee](https://github.com/NevaMind-AI/memU/commit/23f37ee403ecaf88b48af0144e3d701c22ccfddd)) + + +### Documentation + +* add readme cloud api ([#144](https://github.com/NevaMind-AI/memU/issues/144)) ([42fd5ba](https://github.com/NevaMind-AI/memU/commit/42fd5babe6748ef54254cf114a54bdefea304f07)) +* fix api doc link ([#146](https://github.com/NevaMind-AI/memU/issues/146)) ([23ce7d1](https://github.com/NevaMind-AI/memU/commit/23ce7d19b1d68f4fd0a5a5ac6b756f69fede8d09)) + +## [0.9.0](https://github.com/NevaMind-AI/memU/compare/v0.8.0...v0.9.0) (2026-01-04) + + +### Features + +* add GitHub issue templates (bug report, designer feedback, feat… ([#132](https://github.com/NevaMind-AI/memU/issues/132)) ([aee22ee](https://github.com/NevaMind-AI/memU/commit/aee22ee94f275749f69367b83a02a2e819cfd001)) +* add LLM wrapper and interceptors for LLM calls ([#131](https://github.com/NevaMind-AI/memU/issues/131)) ([416e102](https://github.com/NevaMind-AI/memU/commit/416e1029e7752f173c133ebc83bc42801a313059)) +* patch & crud workflows ([#127](https://github.com/NevaMind-AI/memU/issues/127)) ([3cd3dc6](https://github.com/NevaMind-AI/memU/commit/3cd3dc65ae9488207ff8fb0c81e4adb0d22c0f91)) + + +### Bug Fixes + +* category summary & category user scopes ([#125](https://github.com/NevaMind-AI/memU/issues/125)) ([a24efd5](https://github.com/NevaMind-AI/memU/commit/a24efd57df2e305fa3eae8622ea889318979c2f7)) +* config & resource caption ([#142](https://github.com/NevaMind-AI/memU/issues/142)) ([ea4be13](https://github.com/NevaMind-AI/memU/commit/ea4be1396c0f55b02d706819f6c2b4d5c6e68be8)) +* remove duplicate and unnecessary issue templates ([#133](https://github.com/NevaMind-AI/memU/issues/133)) ([559ec14](https://github.com/NevaMind-AI/memU/commit/559ec14d2561ac09162bbb93f178e0f74cc58b23)) + + +### Documentation + +* fix README and CONTRIBUTING to match actual codebase ([#130](https://github.com/NevaMind-AI/memU/issues/130)) ([65d7ef4](https://github.com/NevaMind-AI/memU/commit/65d7ef414563395c6cfa0a75343864671c11b62d)) +* modify readme.md 0102 ([#136](https://github.com/NevaMind-AI/memU/issues/136)) ([f114ee4](https://github.com/NevaMind-AI/memU/commit/f114ee46c8639b256472a8e989695b2f2215f4d4)) + +## [0.8.0](https://github.com/NevaMind-AI/memU/compare/v0.7.0...v0.8.0) (2025-12-24) + + +### Features + +* add configurable batch_size for embedding API calls ([#114](https://github.com/NevaMind-AI/memU/issues/114)) ([060067a](https://github.com/NevaMind-AI/memU/commit/060067a5ebe8ad36c4ca4ad2cc6033303c3f1a36)) +* add conversation created at ([#120](https://github.com/NevaMind-AI/memU/issues/120)) ([825df56](https://github.com/NevaMind-AI/memU/commit/825df5640037fa2345c3153c3df89174059551b3)) +* add workflow implementation and postgres store ([#122](https://github.com/NevaMind-AI/memU/issues/122)) ([a175811](https://github.com/NevaMind-AI/memU/commit/a1758110f859f3725960d308fa0344a1056be52c)) + + +### Bug Fixes + +* example 3 output ([#117](https://github.com/NevaMind-AI/memU/issues/117)) ([65ef7c6](https://github.com/NevaMind-AI/memU/commit/65ef7c6a497047f0f86938cf77ee4602a630216b)) +* postgres model definitions & database initialization ([#124](https://github.com/NevaMind-AI/memU/issues/124)) ([7d5e0cb](https://github.com/NevaMind-AI/memU/commit/7d5e0cb02837a6e2071d3c3344ef6dad613f9a43)) + + +### Documentation + +* Add memU-experiment link to README ([#119](https://github.com/NevaMind-AI/memU/issues/119)) ([2d908e1](https://github.com/NevaMind-AI/memU/commit/2d908e17ee2e15b95d4eb2f35dd09f924d57f8cf)) +* clearer introduction and new agent examples ([#115](https://github.com/NevaMind-AI/memU/issues/115)) ([da27875](https://github.com/NevaMind-AI/memU/commit/da2787536d91490f7a96a0c7379abd1ad1c6d9ac)) + +## [0.7.0](https://github.com/NevaMind-AI/memU/compare/v0.6.0...v0.7.0) (2025-12-05) + + +### Features + +* add user model and user context store ([#113](https://github.com/NevaMind-AI/memU/issues/113)) ([7c37fb1](https://github.com/NevaMind-AI/memU/commit/7c37fb166b0a85bf7c89d0b109cbaa882fa80064)) +* add valcano model support ([#110](https://github.com/NevaMind-AI/memU/issues/110)) ([704c302](https://github.com/NevaMind-AI/memU/commit/704c3024946e8d4a1d1b13ffef33126bb68bd9c4)) + + +### Bug Fixes + +* resource caption miss problem ([#111](https://github.com/NevaMind-AI/memU/issues/111)) ([85586f5](https://github.com/NevaMind-AI/memU/commit/85586f52d6bd6355ae440ad06d8cc6779c689ce8)) + + +### Documentation + +* fix example file path ([#105](https://github.com/NevaMind-AI/memU/issues/105)) ([21aad6a](https://github.com/NevaMind-AI/memU/commit/21aad6a7f070e7666ac6a41c91980a4fa9696918)) +* fix readme test case ([#107](https://github.com/NevaMind-AI/memU/issues/107)) ([228306c](https://github.com/NevaMind-AI/memU/commit/228306c897402c633f57c7aa31e8c6cd4e995d5d)) +* highlight OpenAI key ([#106](https://github.com/NevaMind-AI/memU/issues/106)) ([5b6ce54](https://github.com/NevaMind-AI/memU/commit/5b6ce54def5aa7743a85bec629da65bfa9d8333b)) +* highlight readme openai key ([#103](https://github.com/NevaMind-AI/memU/issues/103)) ([fc4154a](https://github.com/NevaMind-AI/memU/commit/fc4154a707220ced4359e7296218451c43cf0681)) + +## [0.6.0](https://github.com/NevaMind-AI/memU/compare/v0.5.0...v0.6.0) (2025-11-20) + + +### Features + +* retrieve args change conversation_history to queries ([#98](https://github.com/NevaMind-AI/memU/issues/98)) ([6370c6e](https://github.com/NevaMind-AI/memU/commit/6370c6e968c9d0922120bf2a41e8b4206bab87cb)) + +## [0.5.0](https://github.com/NevaMind-AI/memU/compare/v0.4.0...v0.5.0) (2025-11-18) + + +### Features + +* add usecase examples ([#94](https://github.com/NevaMind-AI/memU/issues/94)) ([47b5b39](https://github.com/NevaMind-AI/memU/commit/47b5b390e065ccac1cd2173fa2d6c41549e01063)) + +## [0.4.0](https://github.com/NevaMind-AI/memU/compare/v0.3.0...v0.4.0) (2025-11-18) + + +### Features + +* add non-RAG retrieve solution ([#84](https://github.com/NevaMind-AI/memU/issues/84)) ([fb96e54](https://github.com/NevaMind-AI/memU/commit/fb96e5405f1c0e3477929b7d9874e624dd0453cb)) + + +### Bug Fixes + +* correct binary name after making a release ([#91](https://github.com/NevaMind-AI/memU/issues/91)) ([0fa721a](https://github.com/NevaMind-AI/memU/commit/0fa721aaae294c6f230221af11084a0a6c1f478d)) + + +### Documentation + +* fix several words in README ([#89](https://github.com/NevaMind-AI/memU/issues/89)) ([1e3baf9](https://github.com/NevaMind-AI/memU/commit/1e3baf92d5a08ec51a7eda3249bbd4b40530f56c)) +* revise README for clarity and roadmap inclusion ([#86](https://github.com/NevaMind-AI/memU/issues/86)) ([2235b09](https://github.com/NevaMind-AI/memU/commit/2235b099acc7e92ac52b2613fa731f85259d58fd)) +* update images and refine punctuation in README ([#88](https://github.com/NevaMind-AI/memU/issues/88)) ([cc375e8](https://github.com/NevaMind-AI/memU/commit/cc375e89a9b49bda0b7057eee696d74c4c1d9cee)) + +## [0.3.0](https://github.com/NevaMind-AI/memU/compare/v0.2.2...v0.3.0) (2025-11-17) + + +### Features + +* initialize the memorize and retrieve workflows with the new 3-layer architecture ([#81](https://github.com/NevaMind-AI/memU/issues/81)) ([4a2e86c](https://github.com/NevaMind-AI/memU/commit/4a2e86c0e8bc3e50c82c4fde33d07ad741c8a65e)) + + +### Documentation + +* add German translation for README ([f6d6ab1](https://github.com/NevaMind-AI/memU/commit/f6d6ab1a51b76bd4312542422aebac89410b8da9)) +* add German translation for README ([76c3fc4](https://github.com/NevaMind-AI/memU/commit/76c3fc4227a8e349fbcf1e5bfec29d68e984ff7a)) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..99041ea --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,239 @@ +# Contributing to MemU + +Thank you for your interest in contributing to MemU! This document provides guidelines and information for contributors. + +## 🌟 Ways to Contribute + +We welcome all types of contributions: + +- 🐛 **Bug Reports** - Help us identify and fix issues +- 💡 **Feature Requests** - Suggest new capabilities and improvements +- 📝 **Documentation** - Improve guides, examples, and API docs +- 🔧 **Code Contributions** - Add features, fix bugs, optimize performance +- 🧪 **Testing** - Write tests, improve coverage, test edge cases +- 🎨 **UI/UX** - Enhance user experience and interface design +- 🌐 **Translations** - Help make MemU accessible globally +- 📢 **Community** - Help others in discussions and support channels + +## 🚀 Quick Start for Contributors + +### Prerequisites +- Python 3.13+ +- Git +- [uv](https://github.com/astral-sh/uv) (Python package manager) +- A code editor (VS Code recommended) + +### Development Setup + +```bash +# 1. Fork the repository on GitHub +# 2. Clone your fork locally +git clone https://github.com/YOUR_USERNAME/MemU.git +cd MemU + +# 3. Install uv if you haven't already +curl -LsSf https://astral.sh/uv/install.sh | sh + +# 4. Install development dependencies and pre-commit hooks +make install + +# 5. Verify setup by running tests +make test +``` + +### Available Commands + +```bash +make install # Create virtual environment and install dependencies with uv +make test # Run tests with pytest and coverage +make check # Run all checks (lock file, pre-commit, mypy, deptry) +``` + +## 🔧 Development Guidelines + +### Code Style +- Follow **PEP 8** Python style guidelines +- Use **Ruff** for code formatting and linting (line length: 120) +- Use **type hints** for all functions and methods +- Write **docstrings** for public APIs + +### Code Quality Standards +- Maintain **test coverage > 80%** +- All code must pass **linting** (ruff, mypy) +- Use **meaningful variable and function names** +- Keep functions **focused and small** +- Follow **SOLID principles** + +### Testing +```bash +# Run all tests with coverage +make test + +# Run tests directly with pytest +uv run python -m pytest + +# Run with coverage report +uv run python -m pytest --cov --cov-config=pyproject.toml --cov-report=html + +# Run specific test file +uv run python -m pytest tests/rust_entry_test.py + +# Run tests with specific marker +uv run python -m pytest -m "not slow" +``` + +## 📝 Submitting Changes + +### Before You Start +1. **Search existing issues** to avoid duplicates +2. **Create an issue** for new features or major changes +3. **Discuss your approach** in the issue before implementing + +### Creating Issues + +When reporting bugs, please include: +- **Environment details** (Python version, OS, MemU version) +- **Reproduction steps** with minimal code example +- **Expected vs actual behavior** +- **Error messages** or stack traces + +For feature requests, please describe: +- **The problem** you're trying to solve +- **Proposed solution** or approach +- **Alternative solutions** you've considered +- **Use cases** and examples + +### Pull Request Process + +1. **Create a feature branch** + ```bash + git checkout -b feature/amazing-feature + # or for bug fixes + git checkout -b bugfix/fix-memory-leak + ``` + +2. **Make your changes** + - Write clear, descriptive commit messages + - Keep commits focused and atomic + - Add tests for new functionality + - Update documentation as needed + +3. **Test your changes** + ```bash + make test + make lint + make coverage + ``` + +4. **Submit pull request** + - Use descriptive title and description + - Reference related issues (e.g., "Fixes #123") + - Include testing instructions + - Add screenshots for UI changes + +### Commit Message Format + +Use conventional commit format: + +``` +type(scope): description + +Examples: +feat(memory): add semantic search functionality +fix(llm): resolve OpenAI timeout issues +docs(readme): update installation instructions +test(agent): add unit tests for memory retrieval +refactor(core): restructure memory storage logic +``` + +**Types:** +- `feat`: New feature +- `fix`: Bug fix +- `docs`: Documentation changes +- `test`: Adding or fixing tests +- `refactor`: Code restructuring without feature changes +- `perf`: Performance improvements +- `chore`: Maintenance tasks + +## 🎯 Current Priorities + +We're currently focusing on: + +| Priority | Area | Description | +|----------|------|-------------| +| 🔥 **High** | Multi-modal Support | Images, audio, video memory processing | +| 🔥 **High** | Performance | Memory retrieval optimization, caching | +| 🔥 **High** | LLM Providers | Additional provider integrations | +| 🟡 **Medium** | Enterprise Features | SSO, RBAC, audit logging | +| 🟡 **Medium** | Mobile SDKs | React Native, Flutter support | +| 🟢 **Low** | UI Improvements | Dashboard, memory visualization | + +## 🏷️ Issue Labels + +| Label | Description | +|-------|-------------| +| `good first issue` | Perfect for newcomers | +| `help wanted` | Extra attention needed | +| `bug` | Something isn't working | +| `enhancement` | New feature request | +| `documentation` | Improvements to docs | +| `performance` | Performance optimization | +| `breaking change` | Requires version bump | +| `priority:high` | Urgent issues | +| `priority:medium` | Important issues | +| `priority:low` | Nice to have | + +## 📋 Code Review Process + +### For Contributors +- Be open to feedback and constructive criticism +- Respond promptly to review comments +- Make requested changes in new commits (don't force push) +- Ask questions if feedback is unclear + +### For Reviewers +- Be constructive and respectful in feedback +- Focus on code quality, maintainability, and project goals +- Suggest improvements with explanations +- Approve when ready, request changes when needed + +## 🔒 Security + +**Reporting Security Issues:** +- **DO NOT** create public issues for security vulnerabilities +- Email security issues privately to [contact@nevamind.ai](mailto:contact@nevamind.ai) +- Include detailed reproduction steps and impact assessment +- We'll acknowledge receipt within 24 hours + +## 📄 License and Attribution + +By contributing to MemU, you agree that: +- Your contributions will be licensed under the **Apache License 2.0** +- You have the right to contribute the code/content +- Your contribution doesn't violate any third-party rights + +## 🌍 Community Guidelines + +- Be respectful and inclusive +- Follow our [Code of Conduct](CODE_OF_CONDUCT.md) +- Help others learn and grow +- Share knowledge and best practices +- Celebrate diverse perspectives and experiences + +## 📞 Getting Help + +| Channel | Best For | +|---------|----------| +| 💬 [Discord](https://discord.gg/memu) | Real-time chat, quick questions | +| 🗣️ [GitHub Discussions](https://github.com/NevaMind-AI/MemU/discussions) | Feature discussions, Q&A | +| 🐛 [GitHub Issues](https://github.com/NevaMind-AI/MemU/issues) | Bug reports, feature requests | +| 📧 [Email](mailto:contact@nevamind.ai) | Private inquiries | + +## 🎉 Recognition + +Contributors are recognized in: +- README.md contributors section +- Release notes for significant contributions +- Our [Contributors](https://github.com/NevaMind-AI/MemU/graphs/contributors) page + +Thank you for helping make MemU better! 🚀 diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..a841b5e --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,172 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memu" +version = "0.1.0" +dependencies = [ + "pyo3", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a6df7eab65fc7bee654a421404947e10a0f7085b6951bf2ea395f4659fb0cf" +dependencies = [ + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f77d387774f6f6eec64a004eac0ed525aab7fa1966d94b42f743797b3e395afb" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd13844a4242793e02df3e2ec093f540d948299a6a77ea9ce7afd8623f542be" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf8f9f1108270b90d3676b8679586385430e5c0bb78bb5f043f95499c821a71" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a3b2274450ba5288bc9b8c1b69ff569d1d61189d4bff38f8d22e03d17f932b" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "syn" +version = "2.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..4949405 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "memu" +version = "0.1.0" +edition = "2024" + +[lib] +name = "_core" +# "cdylib" is necessary to produce a shared library for Python to import from. +crate-type = ["cdylib"] + +[dependencies] +# "extension-module" tells pyo3 we want to build an extension module (skips linking against libpython.so) +# "abi3-py313" tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.13 +pyo3 = { version = "0.27.1", features = ["extension-module", "abi3-py313"] } diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..037b3bc --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,194 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (which shall not include communication that is conspicuously + marked or otherwise designated in writing by the copyright owner + as "Not a Contribution"). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based upon (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and derivative works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control + systems, and issue tracking systems that are managed by, or on behalf + of, the Licensor for the purpose of discussing and improving the Work, + but excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution". + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to use, reproduce, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Work, and to + permit persons to whom the Work is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Work. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, trademark, patent, + attribution and other notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright notice to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Support. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or other liability + obligations and/or rights consistent with this License. However, in + accepting such obligations, You may act only on Your own behalf and + on Your sole responsibility, not on behalf of any other Contributor, + and only if You agree to indemnify, defend, and hold each Contributor + harmless for any liability incurred by, or claims asserted against, + such Contributor by reason of your accepting any such warranty or + support. + + END OF TERMS AND CONDITIONS + + Copyright 2024 MemU Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..988fe52 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,21 @@ +include README.md +recursive-include memu *.py +prune example +include setup_postgres_env.sh +exclude .env +exclude .env.example +exclude setup.py.backup +exclude */__pycache__/* +exclude *.pyc +exclude *.pyo +exclude .git/* +exclude .github/* +exclude server/* +exclude docs/* +exclude scripts/* +exclude docker-compose.yml +exclude Dockerfile +exclude .dockerignore +exclude PROJECT_RELEASE_SUMMARY.md +exclude Makefile +exclude .pre-commit-config.yaml diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6f516bc --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +.PHONY: install +install: + @echo "🚀 Creating virtual environment using uv" + @uv sync + @uv run pre-commit install + +.PHONY: check +check: + @echo "🚀 Checking lock file consistency with 'pyproject.toml'" + @uv lock --locked + @echo "🚀 Linting code: Running pre-commit" + @uv run pre-commit run -a + @echo "🚀 Static type checking: Running mypy" + @uv run mypy + @echo "🚀 Checking for obsolete dependencies: Running deptry" + @uv run deptry src + + +.PHONY: test +test: + @echo "🚀 Testing code: Running pytest" + @uv run python -m pytest --cov --cov-config=pyproject.toml --cov-report=xml diff --git a/README.md b/README.md new file mode 100644 index 0000000..7daf112 --- /dev/null +++ b/README.md @@ -0,0 +1,466 @@ +![MemU Banner](assets/banner.png) + +
+ +# memU + +### Personal memory, stored as files + +**Fast retrieval. Higher accuracy. Lower cost.** + +[![PyPI version](https://badge.fury.io/py/memu-py.svg)](https://badge.fury.io/py/memu-py) +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) +[![Discord](https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white)](https://discord.com/invite/hQZntfGsbJ) +[![Twitter](https://img.shields.io/badge/Twitter-Follow-1DA1F2?logo=x&logoColor=white)](https://x.com/memU_ai) + +NevaMind-AI%2FmemU | Trendshift + +**[English](readme/README_en.md) | [中文](readme/README_zh.md) | [日本語](readme/README_ja.md) | [한국어](readme/README_ko.md) | [Español](readme/README_es.md) | [Français](readme/README_fr.md)** + +
+ +--- + +> [!WARNING] +> 🚧 **Under heavy construction** — memU is undergoing a major rework. APIs, CLI commands, and docs may change without notice. Things are expected to stabilize around **July 15, 2026**. + +> 🤖 **Agents**: read [`.claude/skills/memu/SKILL.md`](.claude/skills/memu/SKILL.md) and you can `memorize-workspace` and `retrieve-workspace` right away. + +memU compiles conversations, documents, code, images, audio, video, URLs, and tool traces into human-readable Markdown files (`INDEX.md`, `MEMORY.md`, `SKILL.md`). Agents traverse the tree and load only what the moment needs — instead of rescanning everything or stuffing long histories into every prompt. + + +```python +await service.memorize_workspace(folder="./workspace") + +context = await service.retrieve_workspace("What should I know about this user's launch preferences?") +``` + +Or straight from the terminal — no code: + +```bash +npx memu-cli memorize-workspace ./workspace +npx memu-cli retrieve-workspace "What should I know about this user's launch preferences?" +``` + +That's it. Instead of one giant prompt about a person or their workspace, your agent gets three durable layers it can traverse: + +```txt +workspace/ +├── INDEX.md ← Index: a map of everything — raw sources and summaries +├── MEMORY.md ← Memory: an overview that links into memory/ +├── SKILL.md ← Skill: an overview that links into skill/ +├── resource/ ← the raw source files, copied verbatim +├── memory/ +│ └── .md ← one memory file per topic: facts, preferences, goals, events +└── skill/ + └── .md ← one skill file per learned pattern, workflow, or mistake to avoid +``` + +- **Index (`INDEX.md`)** — a map of your memories: what exists, where it came from, and where to look first +- **Memory (`MEMORY.md`)** — personal facts, preferences, goals, events, and decisions extracted from source data +- **Skill (`SKILL.md`)** — **auto-extracted from agent traces and refined on every workspace sync** so the agent improves at recurring tasks + +When you sync a folder with `memorize_workspace`, the top-level directory decides the treatment: files under `chat/` become memory, files under `agent/` become skills, and everything else is indexed as workspace context. + +Three things make it different from stuffing everything into the prompt: + +- **Fast retrieval** — walk to the right folder and rank the right files instead of scanning everything every time. +- **Higher accuracy** — scope by user, task, or session, and trace every item back to the exact conversation, document, image, or log it came from. +- **Lower cost** — retrieve compact, scoped context instead of reinjecting long histories, documents, logs, and media-derived text into every prompt. +- **Yours to inspect** — a human-readable file tree you can audit, edit, scope, and route through your own storage (`inmemory`, `sqlite`, `postgres`) and LLM providers. + + +--- + +## ⭐️ Star the repository + + + +If you find memU useful or interesting, a GitHub Star ⭐️ would be greatly appreciated. + +--- + +## ✨ Core Features + +| Capability | Description | +|------------|-------------| +| 🗂️ **Multimodal Ingestion** | Write conversations, documents, images, video, audio, URLs, logs, and local files into memory | +| 📁 **Compiled Memory Workspace** | Persist the Index, Skill, and Memory layers — folders (categories), files (items), source artifacts, links, summaries, and embeddings | +| 🧠 **Typed Memory Extraction** | Extract profile, event, knowledge, behavior, skill, and tool memories from raw sources | +| 🛠️ **Self-Evolving Skills** | Auto-extract reusable tool patterns and workflows from agent traces, then merge and refine them on every workspace sync instead of relearning | +| 🧭 **Self-Organizing Folders** | Auto-build categories, links, summaries, and embeddings without manual tagging | +| 🤖 **Agent-Ready Retrieval** | LLM-free `retrieve_workspace()` ranks memory segments, files, and source resources directly | +| 🔄 **Incremental Workspace Sync** | `memorize_workspace()` diffs a folder against a manifest — only changed files are (re)processed, deletions cascade | +| 🧱 **Pluggable Storage** | Use in-memory, SQLite, or Postgres backends with the same repository contracts | +| 🔀 **Profile-Based LLM Routing** | Route chat, embedding, vision, and transcription work through configurable LLM profiles | +| ⌨️ **CLI** | `memu` command (pip) and `npx memu-cli` (npm) — memorize and retrieve from the terminal or CI | + +--- + +## 🎯 Use Cases + +Every use case is the same loop: drop sources into a folder, sync it with `memorize_workspace()`, then ask with `retrieve_workspace()`. The sync is incremental (only changed files are reprocessed), and the top-level directory decides the treatment — `chat/` → memory topics, `agent/` → skills, everything else → indexed context. + +### 1. **Personal Memory** +*Turn chat logs into user preferences, goals, events, decisions, and relationship context.* + +```python +# workspace/chat/*.json — conversation logs become memory topic files +await service.memorize_workspace(folder="./workspace") + +context = await service.retrieve_workspace("What should I remember about this user?") +``` + +### 2. **Workspace Context for Coding Agents** +*Convert docs, PR notes, logs, and design decisions into reusable project memory.* + +```python +# docs, notes, and logs anywhere in the folder are captioned and indexed +await service.memorize_workspace(folder="./workspace") + +context = await service.retrieve_workspace("How should I structure this module?") +``` + +### 3. **Multimodal Knowledge Layer** +*Extract searchable facts from documents, screenshots, images, videos, and audio notes.* + +```python +# modality is inferred per file: .pdf/.docx/.pptx/.xlsx/.html (via MarkItDown — +# pip install 'memu-py[document]'), .png/.jpg, .mp3/.wav, .mp4/.mov, ... +await service.memorize_workspace(folder="./workspace") + +context = await service.retrieve_workspace("What matters for the next research plan?") +``` + +### 4. **Tool and Agent Learning** +*Turn execution traces into skills that tell future agents what worked and what to avoid.* + +```python +# workspace/agent/*.txt — execution traces are distilled into skill files +await service.memorize_workspace(folder="./workspace") + +context = await service.retrieve_workspace("Which tools worked for config editing?") +``` + +--- + +## 🗂️ Architecture + +The compiled workspace is easiest to read as two directions: + +- `memorize_workspace()` writes a folder into durable memory files, skill files, resource records, segments, links, and embeddings. +- `retrieve_workspace()` reads those layers directly, ranking segments first and rolling results up to the files and resources an agent should load. + +Memory is stored in three representation layers: + +| Layer | What it holds | Retrieval Role | +|-------|---------------|----------------| +| **File** (`RecallFile`) | A synthesized memory topic or skill document | The unit returned to the agent — hit segments roll up to their file | +| **Segment** | Fine slices of a file (paragraph lines, skill descriptions) | The embedded search unit — queries rank segments first | +| **Resource** | The raw source artifact with its caption | Recall original context when synthesized summaries are not enough | + +`retrieve_workspace()` embeds the query once, ranks segments and resources by similarity, and returns compact context with zero chat-LLM calls. + +See [docs/architecture.md](docs/architecture.md) for the runtime view of `MemoryService`, workflow pipelines, storage backends, and LLM routing, and [docs/adr/](docs/adr/README.md) for the decision records behind the layered design. + +--- + +## 🧰 Agent Skills + +The repo ships one [Agent Skill](https://docs.claude.com/en/docs/agents-and-tools/agent-skills) — [`.claude/skills/memu/SKILL.md`](.claude/skills/memu/SKILL.md) — that gives Claude Code (and any skills-compatible agent) the workspace pair. The agent decides when to use each direction: + +- **memorize** (`memu memorize-workspace`) — "remember this", "sync this folder into memory", finishing work worth persisting +- **retrieve** (`memu retrieve-workspace`) — "what do we know about…", starting a task with likely prior context + +It works out of the box inside this repo. To use it in your own project, copy the skill folder into that project's `.claude/skills/` (or `~/.claude/skills/` to enable it everywhere): + +```bash +cp -r .claude/skills/memu /path/to/your-project/.claude/skills/ +``` + +The skill locates the CLI automatically (`memu`, `uvx --from memu-py memu`, or `npx memu-cli`) and keeps state in the project-local `./data/memu.sqlite3`, so what one session memorizes the next can retrieve. For LangGraph agents, see the [LangGraph integration](docs/langgraph_integration.md) instead. + +--- + +## 🚀 Quick Start + +### Option 1: Cloud Version + +👉 **[memu.so](https://memu.so)** — Hosted API for managed ingestion, structured memory, and retrieval + +For enterprise deployment: **info@nevamind.ai** + +#### Cloud API (v3) + +| Base URL | `https://api.memu.so` | +|----------|----------------------| +| Auth | `Authorization: Bearer ` | + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/api/v3/memory/memorize` | Ingest raw data and build structured memory | +| `GET` | `/api/v3/memory/memorize/status/{task_id}` | Check processing status | +| `POST` | `/api/v3/memory/categories` | List auto-generated categories | +| `POST` | `/api/v3/memory/retrieve` | Query memory for agent context | + +📚 **[Full API Documentation](https://memu.pro/docs#cloud-version)** + +--- + +### Option 2: Self-Hosted + +#### Installation + +From a clone of this repository: + +```bash +uv sync +# or, for the full development setup: +make install +``` + +To install the published package instead: + +```bash +pip install memu-py # library + `memu` CLI +# or from the JS ecosystem (thin launcher over memu-py, uses uvx/pipx automatically): +npx memu-cli --help +``` + +> **Requirements**: Python 3.13+. The default examples use OpenAI, so set `OPENAI_API_KEY` or pass another provider through `llm_profiles`. + +#### Command line + +The `memu` command wraps the same service the library exposes. State persists in a local SQLite database (`./data/memu.sqlite3` by default), so memorize in one invocation and retrieve in the next: + +```bash +export OPENAI_API_KEY=your_key + +memu memorize-workspace ./workspace # diff-sync a folder (alias: memu sync) +memu retrieve-workspace "deploy checklist" # LLM-free embedding retrieval (alias: memu search) +memu export # rebuild the INDEX.md/MEMORY.md/SKILL.md tree +``` + +Every flag has a `MEMU_*` environment variable (`--provider`/`MEMU_LLM_PROVIDER`, `--model`/`MEMU_CHAT_MODEL`, `--db`/`MEMU_DB`, ...) — run `memu --help` for the full list. `--db` accepts a SQLite path, a `postgres://` DSN, or `:memory:`. + +**Run an in-memory smoke script:** +```bash +export OPENAI_API_KEY=your_key +cd tests +uv run python test_inmemory.py +``` + +**Run with PostgreSQL + pgvector:** +```bash +uv sync --extra postgres +docker run -d --name memu-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=memu \ + -p 5432:5432 \ + pgvector/pgvector:pg16 + +export OPENAI_API_KEY=your_key +export POSTGRES_DSN=postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu +cd tests +uv run python test_postgres.py +``` + +--- + +### Custom LLM and Embedding Providers + +```python +from memu import MemoryService + +service = MemoryService( + llm_profiles={ + "default": { + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "api_key": "your_key", + "chat_model": "qwen3-max", + "client_backend": "sdk" + }, + "embedding": { + "base_url": "https://api.voyageai.com/v1", + "api_key": "your_key", + "embed_model": "voyage-3.5-lite" + } + }, +) +``` + +--- + +### OpenRouter Integration + +```python +from memu import MemoryService + +service = MemoryService( + llm_profiles={ + "default": { + "provider": "openrouter", + "client_backend": "httpx", + "base_url": "https://openrouter.ai", + "api_key": "your_key", + "chat_model": "anthropic/claude-3.5-sonnet", + "embed_model": "openai/text-embedding-3-small", + }, + }, + database_config={"metadata_store": {"provider": "inmemory"}}, +) +``` + +--- + +## 📖 Core APIs + +The primary API pair is `memorize_workspace()` / `retrieve_workspace()` — folder in, ranked context out. + +### `memorize_workspace()` — Sync a Folder + +memorize_workspace + +```python +result = await service.memorize_workspace( + folder="./workspace", # scanned recursively; modality inferred per file + user={"user_id": "123"}, # optional scope +) +# Returns the diff plus what changed: +# { "added": [...], "modified": [...], "deleted": [...], +# "resources": [...], "entries": [...], "files": [...] } +``` + +- Diffs the folder against a sidecar `.memu_manifest.json` — only added/modified files are processed, memory from deleted files is cascade-removed +- Routes by top-level directory: `chat/` → memory files, `agent/` → skill files, everything else → indexed workspace context +- Rebuilds the markdown memory tree (`INDEX.md` / `MEMORY.md` / `SKILL.md`) when `memory_files_config.enabled=True` + +--- + +### `retrieve_workspace()` — Fast, LLM-Free Retrieval + +retrieve_workspace + +```python +result = await service.retrieve_workspace( + "deploy checklist", + where={"user_id": "123"}, +) +# Returns: +# { "segments": [...], # embedded slices ranked by similarity +# "files": [...], # the memory/skill files those segments roll up to +# "resources": [...] } # workspace resources ranked by similarity +``` + +The query is embedded once and ranked by vector similarity — no intention routing, no query rewriting, no sufficiency checks, zero LLM calls. Use it for high-frequency lookups where latency and cost matter more than deep reasoning. + +--- + +## 💡 Example Workflows + +### Always-Learning Assistant +```bash +export OPENAI_API_KEY=your_key +uv run python examples/example_1_conversation_memory.py +``` +Automatically extracts preferences, builds relationship models, and surfaces relevant context in future conversations. + +### Self-Improving Agent +```bash +uv run python examples/example_2_skill_extraction.py +``` +Monitors agent actions, identifies patterns in successes and failures, auto-generates skill guides from experience. + +### Multimodal Context Builder +```bash +uv run python examples/example_3_multimodal_memory.py +``` +Cross-references text, images, and documents automatically into a unified memory layer. + +--- + +## 📊 Performance + +memU achieves **92.09% average accuracy** on the Locomo benchmark across all reasoning tasks. + +benchmark + +View detailed results: [memU-experiment](https://github.com/NevaMind-AI/memU-experiment) + +--- + +## 🧩 Ecosystem + +| Repository | Description | +|------------|-------------| +| **[memU](https://github.com/NevaMind-AI/memU)** | Personal memory as files — fast retrieval, higher accuracy, lower cost | +| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | Backend with real-time sync and webhook triggers | +| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | Visual dashboard for browsing and monitoring memory | + +**Quick Links:** +- 🚀 [Try MemU Cloud](https://app.memu.so/quick-start) +- 📚 [API Documentation](https://memu.pro/docs) +- 💬 [Discord Community](https://discord.com/invite/hQZntfGsbJ) + +--- + +## 🤝 Partners + +
+ +Ten +OpenAgents +Milvus +xRoute +Jazz +Buddie +Bytebase +LazyLLM +Clawdchat + +
+ +--- + +## 🤝 Contributing + +```bash +# Fork and clone +git clone https://github.com/YOUR_USERNAME/memU.git +cd memU + +# Install dev dependencies +make install + +# Run quality checks before submitting +make check +``` + +See [CONTRIBUTING.md](CONTRIBUTING.md) for full guidelines. + +**Prerequisites:** Python 3.13+, [uv](https://github.com/astral-sh/uv), Git + +--- + +## 📄 License + +[Apache License 2.0](LICENSE.txt) + +--- + +## 🌍 Community + +- **GitHub Issues**: [Report bugs & request features](https://github.com/NevaMind-AI/memU/issues) +- **Discord**: [Join the community](https://discord.com/invite/hQZntfGsbJ) +- **X (Twitter)**: [Follow @memU_ai](https://x.com/memU_ai) +- **Contact**: info@nevamind.ai + +--- + +
+ +⭐ **Star us on GitHub** to get notified about new releases! + +
diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..d9e63c8 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目:`NevaMind-AI/memU` +- 原始仓库:https://github.com/NevaMind-AI/memU +- 导入方式:上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源,不代表 WeHub 是原项目作者 diff --git a/assets/banner.png b/assets/banner.png new file mode 100644 index 0000000..046aa13 Binary files /dev/null and b/assets/banner.png differ diff --git a/assets/benchmark.png b/assets/benchmark.png new file mode 100644 index 0000000..2348e08 Binary files /dev/null and b/assets/benchmark.png differ diff --git a/assets/memUbot.png b/assets/memUbot.png new file mode 100644 index 0000000..f7ee4b9 Binary files /dev/null and b/assets/memUbot.png differ diff --git a/assets/memorize.png b/assets/memorize.png new file mode 100644 index 0000000..d3fd175 Binary files /dev/null and b/assets/memorize.png differ diff --git a/assets/partners/Clawdchat.png b/assets/partners/Clawdchat.png new file mode 100644 index 0000000..1a83118 Binary files /dev/null and b/assets/partners/Clawdchat.png differ diff --git a/assets/partners/LazyLLM.png b/assets/partners/LazyLLM.png new file mode 100644 index 0000000..8e85fcb Binary files /dev/null and b/assets/partners/LazyLLM.png differ diff --git a/assets/partners/buddie.png b/assets/partners/buddie.png new file mode 100644 index 0000000..01a6bfe Binary files /dev/null and b/assets/partners/buddie.png differ diff --git a/assets/partners/bytebase.png b/assets/partners/bytebase.png new file mode 100644 index 0000000..957b6f2 Binary files /dev/null and b/assets/partners/bytebase.png differ diff --git a/assets/partners/jazz.png b/assets/partners/jazz.png new file mode 100644 index 0000000..1a662af Binary files /dev/null and b/assets/partners/jazz.png differ diff --git a/assets/partners/openagents.png b/assets/partners/openagents.png new file mode 100644 index 0000000..3519478 Binary files /dev/null and b/assets/partners/openagents.png differ diff --git a/assets/partners/xroute.png b/assets/partners/xroute.png new file mode 100644 index 0000000..f9e67bc Binary files /dev/null and b/assets/partners/xroute.png differ diff --git a/assets/qrcode.png b/assets/qrcode.png new file mode 100644 index 0000000..824ffc2 Binary files /dev/null and b/assets/qrcode.png differ diff --git a/assets/retrieve.png b/assets/retrieve.png new file mode 100644 index 0000000..0b0f4ac Binary files /dev/null and b/assets/retrieve.png differ diff --git a/assets/star.gif b/assets/star.gif new file mode 100644 index 0000000..83d7515 Binary files /dev/null and b/assets/star.gif differ diff --git a/assets/structure.png b/assets/structure.png new file mode 100644 index 0000000..9d3c435 Binary files /dev/null and b/assets/structure.png differ diff --git a/assets/usecase/ai_companion-0000.jpg b/assets/usecase/ai_companion-0000.jpg new file mode 100644 index 0000000..63e99cd Binary files /dev/null and b/assets/usecase/ai_companion-0000.jpg differ diff --git a/assets/usecase/ai_creation-0000.jpg b/assets/usecase/ai_creation-0000.jpg new file mode 100644 index 0000000..baf1812 Binary files /dev/null and b/assets/usecase/ai_creation-0000.jpg differ diff --git a/assets/usecase/ai_edu-0000.jpg b/assets/usecase/ai_edu-0000.jpg new file mode 100644 index 0000000..cc73cee Binary files /dev/null and b/assets/usecase/ai_edu-0000.jpg differ diff --git a/assets/usecase/ai_ip-0000.png b/assets/usecase/ai_ip-0000.png new file mode 100644 index 0000000..49eb719 Binary files /dev/null and b/assets/usecase/ai_ip-0000.png differ diff --git a/assets/usecase/ai_robot-0000.jpg b/assets/usecase/ai_robot-0000.jpg new file mode 100644 index 0000000..20c4bd0 Binary files /dev/null and b/assets/usecase/ai_robot-0000.jpg differ diff --git a/assets/usecase/ai_role_play-0000.jpg b/assets/usecase/ai_role_play-0000.jpg new file mode 100644 index 0000000..f88f97b Binary files /dev/null and b/assets/usecase/ai_role_play-0000.jpg differ diff --git a/assets/usecase/ai_therapy-0000.jpg b/assets/usecase/ai_therapy-0000.jpg new file mode 100644 index 0000000..f08abf0 Binary files /dev/null and b/assets/usecase/ai_therapy-0000.jpg differ diff --git a/docs/HACKATHON_ISSUE_DRAFT.md b/docs/HACKATHON_ISSUE_DRAFT.md new file mode 100644 index 0000000..7951c4b --- /dev/null +++ b/docs/HACKATHON_ISSUE_DRAFT.md @@ -0,0 +1,68 @@ +# GitHub Issue Draft: Memory Types + Tool Memory + +## Title +`[2026NewYearChallenge] Specialized Memory Types with Tool Learning` + +--- + +## Description + +### What will this task implement? + +This PR enhances MemU's memory type system to support specialized memory structures with type-specific metadata and introduces Tool Memory for agent self-improvement. + +**Current State:** MemU has a `memory_type` field with 5 types (profile, event, knowledge, behavior, skill) and uses different LLM prompts to extract each type. However, after extraction, all memories share the same storage schema - just `summary` and `embedding`. There's no type-specific metadata, no type-aware retrieval, and no way for agents to learn from their tool usage. + +**Enhancement:** Extend the memory system to support: +- Type-specific metadata fields (e.g., `when_to_use` for smarter retrieval) +- Tool Memory type for tracking tool execution history +- Tool usage statistics for agent self-improvement +- Type-aware retrieval filtering + +**Key Benefits:** +- Agents can learn from their own tool usage patterns +- Smarter retrieval based on memory context +- Foundation for agents that improve over time +- Better alignment with agentic application needs + +--- + +## Requirements + +- [x] Type-specific metadata schema with `when_to_use` field +- [x] Tool Memory implementation with execution tracking +- [x] Tool statistics (success_rate, avg_time_cost, avg_score) +- [ ] Type-aware retrieval filtering +- [x] Tests for Tool Memory CRUD and statistics +- [ ] Documentation and usage examples + +--- + +## Review Criteria + +- Correctness: All tests pass, no regressions +- Quality: Clean code, follows existing patterns +- DX: Clear documentation and examples +- Impact: Enables agent self-improvement use cases + +--- + +## Implementation Summary + +### Files Modified: +1. `src/memu/database/models.py` - Added `ToolCallResult` model, extended `MemoryItem` with `when_to_use`, `metadata`, `tool_calls` fields +2. `src/memu/database/repositories/memory_item.py` - Updated interface with new fields +3. `src/memu/database/inmemory/repositories/memory_item_repo.py` - Updated implementation +4. `src/memu/database/postgres/repositories/memory_item_repo.py` - Updated implementation +5. `src/memu/database/postgres/models.py` - Added JSON columns for new fields +6. `src/memu/prompts/memory_type/__init__.py` - Added tool type +7. `src/memu/prompts/memory_type/tool.py` - New prompt for tool memory extraction + +### Files Added: +1. `tests/test_tool_memory.py` - 14 unit tests for Tool Memory feature + +--- + +## Notes + +This builds on MemU's existing memory type foundation while adding the specialized structures needed for agentic applications. The Tool Memory feature is particularly valuable for agents that need to learn which tools work best in different situations. diff --git a/docs/HACKATHON_MAD_COMBOS.md b/docs/HACKATHON_MAD_COMBOS.md new file mode 100644 index 0000000..7b58469 --- /dev/null +++ b/docs/HACKATHON_MAD_COMBOS.md @@ -0,0 +1,659 @@ +# 🔥🔥🔥 MAD COMBO OPTIONS FOR MEMU HACKATHON 🔥🔥🔥 + +> **Goal:** Implement high-impact features that MemU is missing, sourced from competitor analysis of 7 memory repos (memoripy, memlayer, ReMe, memX, memphora-sdk, MemOS, memor). + +--- + +## 📋 COMPLETE FEATURE GAP ANALYSIS + +### FEATURES MEMU IS MISSING (Deep Scan Results) + +#### FROM MEMORIPY: +- ❌ `access_counts[]` - Track how often each memory is accessed +- ❌ `timestamps[]` - Track when memory was created/last accessed +- ❌ `decay_factor` - Exponential time-based decay: `np.exp(-decay_rate * time_diff)` +- ❌ `reinforcement_factor` - Log-scaled access boost: `np.log1p(access_count)` +- ❌ `adjusted_similarity` - `similarity * decay_factor * reinforcement_factor` +- ❌ Short-term → Long-term memory promotion (when `access_count > 10`) +- ❌ `nx.Graph()` concept associations (NetworkX graph) +- ❌ `spreading_activation()` - Spread activation through concept graph +- ❌ `cluster_interactions()` - KMeans clustering for hierarchical memory +- ❌ `semantic_memory` clusters - Retrieve from semantic memory clusters + +#### FROM MEMLAYER: +- ❌ `SalienceGate` - Filter what's worth saving vs noise +- ❌ `SalienceMode.LOCAL` - Local ML model for salience +- ❌ `SalienceMode.ONLINE` - OpenAI API for salience +- ❌ `SalienceMode.LIGHTWEIGHT` - Keyword-based salience (no embeddings) +- ❌ `SALIENT_PROTOTYPES` / `NON_SALIENT_PROTOTYPES` - Prototype sentences +- ❌ `is_worth_saving()` - Determine if text should be saved +- ❌ `CurationService` - Background memory decay/expiration +- ❌ `_calculate_relevance()` - Score based on age, recency, attention +- ❌ Auto-archive low-relevance memories +- ❌ Auto-delete expired memories (`expiration_timestamp`) +- ❌ `SchedulerService` - Background task scheduler +- ❌ `get_due_tasks_for_user()` - Check for pending scheduled tasks +- ❌ `ConsolidationService` - Background knowledge extraction +- ❌ `analyze_and_extract_knowledge()` - Extract facts, entities, relationships +- ❌ `NetworkXStorage` - Graph storage for entities/relationships +- ❌ `add_entity()` / `add_relationship()` - Knowledge graph operations +- ❌ `get_subgraph_context()` - Graph traversal for context +- ❌ `find_matching_nodes()` - Fuzzy entity matching +- ❌ `_find_canonical_entity()` - Entity deduplication +- ❌ `_merge_entity_nodes()` - Merge duplicate entities +- ❌ `importance_score` / `expiration_timestamp` metadata +- ❌ `track_memory_access()` - Track when memories are accessed +- ❌ Task reminders system (`add_task`, `get_pending_tasks`, `update_task_status`) + +#### FROM REME: +- ❌ `UpdateMemoryFreqOp` - Increment frequency counter on recall +- ❌ `metadata["freq"]` - Frequency counter in metadata +- ❌ `UpdateMemoryUtilityOp` - Increment utility score when useful +- ❌ `metadata["utility"]` - Utility score in metadata +- ❌ `DeleteMemoryOp` - Delete based on freq/utility thresholds +- ❌ `utility/freq < threshold` pruning - Prune low-value memories +- ❌ **MEMORY TYPES:** + - ❌ `TaskMemory` - Task-related information + - ❌ `PersonalMemory` - Personal info with `target` and `reflection_subject` + - ❌ `ToolMemory` - Tool call execution history + - ❌ `ToolCallResult` - Record tool execution results with hash deduplication +- ❌ `MemoryDeduplicationOp` - Remove duplicate memories using embedding similarity +- ❌ `WorkingMemory` operations: + - ❌ `MessageCompressOp` - LLM-based compression for long conversations + - ❌ `MessageCompactOp` - Compact verbose tool messages + - ❌ `MessageOffloadOp` - Orchestrate compaction + compression + - ❌ `WorkingSummaryMode.COMPACT/COMPRESS/AUTO` +- ❌ `UpdateMemory` tool - Update/edit existing memories +- ❌ `session_memory_id` tracking - Track memories per session +- ❌ Tool memory statistics (`avg_token_cost`, `success_rate`, `avg_time_cost`, `avg_score`) + +#### FROM MEMX: +- ❌ `pubsub.py` - Real-time pub/sub system +- ❌ `subscribe(key, websocket)` - WebSocket subscriptions +- ❌ `publish(key, payload)` - Broadcast updates to subscribers +- ❌ `set_value()` with timestamps - Last-write-wins with timestamps +- ❌ Redis-backed shared memory - Multi-agent shared state +- ❌ `register_schema()` / `validate_schema()` - JSON schema validation + +#### FROM MEMOS (MemOS): +- ❌ **Memory Scheduler System:** + - ❌ `BaseScheduler` - Full task scheduling infrastructure + - ❌ `SchedulerDispatcher` - Parallel task dispatch + - ❌ `ScheduleTaskQueue` - Priority task queue + - ❌ `TaskStatusTracker` - Track task status in Redis + - ❌ `TaskPriorityLevel` - Priority levels for tasks +- ❌ `MemoryMonitorItem` - Monitor memory with importance scores +- ❌ `replace_working_memory()` - Replace working memory after reranking +- ❌ `update_activation_memory()` - Update activation memory periodically +- ❌ `transform_working_memories_to_monitors()` - Convert memories to monitors +- ❌ `visibility` field - Public/private memory visibility +- ❌ `confidence` score - Confidence level for memories +- ❌ `status` field (activated/archived) - Memory activation status +- ❌ `tags` field - Memory tagging system +- ❌ `entities` extraction - Extract entities from memories + +#### FROM MEMPHORA-SDK: +- ❌ `store_shared()` - Store shared memory for groups +- ❌ Multi-agent crew memory - Shared memory for agent crews +- ❌ Per-agent namespaces - Isolated memory per agent +- ❌ Framework integrations (AutoGen, CrewAI, LangChain, LlamaIndex) + +--- + +--- + +## 🏆 COMBO 1: "INTELLIGENT MEMORY LIFECYCLE" + +**Theme:** Memory that learns, ages, and self-curates like human memory + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| Decay & Reinforcement | memoripy | 3 pts | LOW | +| Frequency & Utility Tracking | ReMe | 3 pts | LOW | +| Auto-Pruning Low-Value Memories | ReMe | 3 pts | LOW | + +**Total: 9 pts | LOW-MEDIUM effort** + +### Why it's MAD: + +``` +Memory accessed often → gets STRONGER (reinforcement) +Memory ignored → gets WEAKER (decay) +Memory with low utility/freq ratio → gets DELETED automatically + +Result: Self-healing, self-optimizing memory that mimics human forgetting! +``` + +### The Pitch: +> "MemU now has HUMAN-LIKE memory - it remembers what matters and forgets what doesn't!" + +### Technical Implementation: + +```python +# Decay formula (from memoripy) +decay_factor = np.exp(-decay_rate * time_diff) +reinforcement_factor = np.log1p(access_count) +adjusted_similarity = similarity * decay_factor * reinforcement_factor + +# Pruning logic (from ReMe) +if freq >= freq_threshold: + if utility / freq < utility_threshold: + delete_memory(memory_id) +``` + +--- + +## 🏆 COMBO 2: "SMART MEMORY GATE" + +**Theme:** Don't save garbage, only save gold + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| Salience Filtering | memlayer | 3-5 pts | MEDIUM | +| Decay & Reinforcement | memoripy | 3 pts | LOW | +| Background Curation Service | memlayer | 3 pts | MEDIUM | + +**Total: 9-11 pts | MEDIUM effort** + +### Why it's MAD: + +``` +INPUT: "Hello!" → BLOCKED (not salient) +INPUT: "My name is John, I work at Google" → SAVED (salient) +BACKGROUND: Old unused memories → AUTO-ARCHIVED +RETRIEVAL: Frequently accessed → BOOSTED + +Result: Clean, high-quality memory that doesn't bloat! +``` + +### The Pitch: +> "MemU now has a BOUNCER - only important memories get in, garbage stays out!" + +### Technical Implementation: + +```python +# Salience Gate (from memlayer) +class SalienceGate: + SALIENT_PROTOTYPES = ["My name is...", "I work at...", "The deadline is..."] + NON_SALIENT_PROTOTYPES = ["Hello", "Thanks", "Okay", "Got it"] + + def is_worth_saving(self, text: str) -> bool: + # Compare similarity to salient vs non-salient prototypes + salient_score = max_similarity(text, SALIENT_PROTOTYPES) + non_salient_score = max_similarity(text, NON_SALIENT_PROTOTYPES) + return salient_score > (non_salient_score + threshold) +``` + +--- + +## 🏆 COMBO 3: "KNOWLEDGE BRAIN" + +**Theme:** Memory that understands relationships + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| Knowledge Graph | memlayer | 5 pts | HIGH | +| Entity Extraction | memlayer | 3 pts | MEDIUM | +| Graph Traversal Retrieval | memlayer | 3 pts | MEDIUM | + +**Total: 11 pts | HIGH effort** + +### Why it's MAD: + +``` +INPUT: "John works at Google. Sarah is John's wife." + +GRAPH: + John --[works_at]--> Google + John --[married_to]--> Sarah + +QUERY: "Who is related to Google?" +RESULT: John (works there), Sarah (married to John who works there) + +Result: Memory that REASONS about relationships! +``` + +### The Pitch: +> "MemU now has a BRAIN - it understands how things connect!" + +### Technical Implementation: + +```python +# Knowledge Graph (from memlayer) +import networkx as nx + +class KnowledgeGraph: + def __init__(self): + self.graph = nx.Graph() + + def add_entity(self, name: str, node_type: str): + self.graph.add_node(name, type=node_type) + + def add_relationship(self, subject: str, predicate: str, obj: str): + self.graph.add_edge(subject, obj, relation=predicate) + + def get_subgraph_context(self, entity: str, depth: int = 2): + # Traverse graph for related entities + return nx.ego_graph(self.graph, entity, radius=depth) +``` + +--- + +## 🏆 COMBO 4: "MEMORY EVOLUTION" ⭐ TOP PICK + +**Theme:** Memory that evolves and improves itself + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| Salience Gate (LIGHTWEIGHT mode) | memlayer | 3 pts | LOW | +| Decay & Reinforcement | memoripy | 3 pts | LOW | +| Frequency & Utility | ReMe | 3 pts | LOW | +| Auto-Pruning | ReMe | 3 pts | LOW | + +**Total: 12 pts | LOW-MEDIUM effort** + +### Why it's MAD: + +``` +STAGE 1: Salience Gate filters noise at INPUT +STAGE 2: Decay/Reinforcement adjusts scores at RETRIEVAL +STAGE 3: Frequency/Utility tracks VALUE over time +STAGE 4: Auto-Pruning DELETES low-value memories + +Result: FULL LIFECYCLE MANAGEMENT - from birth to death! +``` + +### The Pitch: +> "MemU memories now have a LIFECYCLE - they're born, they grow, they age, they die!" + +### Memory Lifecycle Diagram: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ MEMORY LIFECYCLE │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ INPUT ──► [SALIENCE GATE] ──► SAVE or REJECT │ +│ │ │ +│ ▼ │ +│ ┌─────────┐ │ +│ │ MEMORY │ ◄── access_count, last_accessed │ +│ │ ITEM │ ◄── freq, utility, salience_score │ +│ └────┬────┘ │ +│ │ │ +│ ┌────────┴────────┐ │ +│ ▼ ▼ │ +│ [RETRIEVAL] [BACKGROUND] │ +│ │ │ │ +│ decay_factor auto_prune() │ +│ reinforcement if utility/freq < threshold │ +│ │ │ │ +│ ▼ ▼ │ +│ BOOSTED SCORE DELETE MEMORY │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 📊 COMPARISON TABLE + +| Combo | Points | Effort | WOW Factor | Complexity | Recommendation | +|-------|--------|--------|------------|------------|----------------| +| 1. Intelligent Lifecycle | 9 | LOW | ⭐⭐⭐⭐ | LOW | ✅ SAFE BET | +| 2. Smart Gate | 9-11 | MEDIUM | ⭐⭐⭐⭐ | MEDIUM | ✅ GOOD | +| 3. Knowledge Brain | 11 | HIGH | ⭐⭐⭐⭐⭐ | HIGH | ⚠️ RISKY | +| 4. Memory Evolution | 12 | LOW-MED | ⭐⭐⭐⭐⭐ | MEDIUM | 🏆 **BEST COMBO** | + +--- + +## 🎯 RECOMMENDATION: COMBO 4 "MEMORY EVOLUTION" + +### Why This Combo Wins: + +1. **12 points** - highest point potential +2. **LOW-MEDIUM effort** - achievable in hackathon timeframe +3. **4 features that SYNERGIZE** - each builds on the other +4. **UNIQUE story** - "memory lifecycle" is a compelling narrative +5. **Easy to demo** - show memory being filtered, decaying, getting pruned + +### Implementation Order: + +``` +Step 1: Add fields to MemoryItem model + - access_count: int = 0 + - last_accessed: datetime + - freq: int = 0 + - utility: int = 0 + - salience_score: float = 0.0 + +Step 2: Implement lightweight salience gate (keyword-based, no ML) + - SALIENT_KEYWORDS list + - NON_SALIENT_KEYWORDS list + - is_worth_saving() function + +Step 3: Implement decay-aware retrieval + - Modify cosine_topk() to apply decay formula + - Update access_count and last_accessed on retrieval + +Step 4: Implement frequency/utility tracking + - Increment freq on every retrieval + - Increment utility when memory contributes to response + +Step 5: Implement auto-pruning + - Background check for low utility/freq ratio + - Delete memories below threshold +``` + +### Files to Modify: + +``` +memU/src/memu/database/models.py # Add new fields +memU/src/memu/database/inmemory/vector.py # Decay-aware retrieval +memU/src/memu/app/memorize.py # Salience gate +memU/src/memu/app/retrieve.py # Frequency/utility tracking +memU/src/memu/app/service.py # Auto-pruning service +``` + +--- + +## 📚 Reference Implementations + +### From memoripy (Decay & Reinforcement): +- File: `prospects/memoripy/memoripy/memory_store.py` +- Key functions: `retrieve()`, `classify_memory()` + +### From memlayer (Salience Gate): +- File: `prospects/memlayer/memlayer/ml_gate.py` +- Key class: `SalienceGate` + +### From ReMe (Frequency & Utility): +- Files: + - `prospects/ReMe/reme_ai/vector_store/update_memory_freq_op.py` + - `prospects/ReMe/reme_ai/vector_store/update_memory_utility_op.py` + - `prospects/ReMe/reme_ai/vector_store/delete_memory_op.py` + +--- + +## 🚀 Ready to Implement? + +Choose your combo and let's build! 🔥 + + +## 🏆 UPDATED MAD COMBOS (After Deep Scan) + +--- + +## 🏆 COMBO 1: "INTELLIGENT MEMORY LIFECYCLE" + +**Theme:** Memory that learns, ages, and self-curates like human memory + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| Decay & Reinforcement | memoripy | 3 pts | LOW | +| Frequency & Utility Tracking | ReMe | 3 pts | LOW | +| Auto-Pruning Low-Value Memories | ReMe | 3 pts | LOW | + +**Total: 9 pts | LOW-MEDIUM effort** + +### Why it's MAD: + +``` +Memory accessed often → gets STRONGER (reinforcement) +Memory ignored → gets WEAKER (decay) +Memory with low utility/freq ratio → gets DELETED automatically + +Result: Self-healing, self-optimizing memory that mimics human forgetting! +``` + +### The Pitch: +> "MemU now has HUMAN-LIKE memory - it remembers what matters and forgets what doesn't!" + +--- + +## 🏆 COMBO 2: "SMART MEMORY GATE" + +**Theme:** Don't save garbage, only save gold + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| Salience Filtering | memlayer | 3-5 pts | MEDIUM | +| Decay & Reinforcement | memoripy | 3 pts | LOW | +| Background Curation Service | memlayer | 3 pts | MEDIUM | + +**Total: 9-11 pts | MEDIUM effort** + +### Why it's MAD: + +``` +INPUT: "Hello!" → BLOCKED (not salient) +INPUT: "My name is John, I work at Google" → SAVED (salient) +BACKGROUND: Old unused memories → AUTO-ARCHIVED +RETRIEVAL: Frequently accessed → BOOSTED + +Result: Clean, high-quality memory that doesn't bloat! +``` + +### The Pitch: +> "MemU now has a BOUNCER - only important memories get in, garbage stays out!" + +--- + +## 🏆 COMBO 3: "KNOWLEDGE BRAIN" + +**Theme:** Memory that understands relationships + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| Knowledge Graph (NetworkX) | memlayer | 5 pts | HIGH | +| Entity Extraction & Deduplication | memlayer | 3 pts | MEDIUM | +| Graph Traversal Retrieval | memlayer | 3 pts | MEDIUM | + +**Total: 11 pts | HIGH effort** + +### Why it's MAD: + +``` +INPUT: "John works at Google. Sarah is John's wife." + +GRAPH: + John --[works_at]--> Google + John --[married_to]--> Sarah + +QUERY: "Who is related to Google?" +RESULT: John (works there), Sarah (married to John who works there) + +Result: Memory that REASONS about relationships! +``` + +### The Pitch: +> "MemU now has a BRAIN - it understands how things connect!" + +--- + +## 🏆 COMBO 4: "MEMORY EVOLUTION" ⭐ TOP PICK + +**Theme:** Memory that evolves and improves itself + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| Salience Gate (LIGHTWEIGHT mode) | memlayer | 3 pts | LOW | +| Decay & Reinforcement | memoripy | 3 pts | LOW | +| Frequency & Utility | ReMe | 3 pts | LOW | +| Auto-Pruning | ReMe | 3 pts | LOW | + +**Total: 12 pts | LOW-MEDIUM effort** + +### Why it's MAD: + +``` +STAGE 1: Salience Gate filters noise at INPUT +STAGE 2: Decay/Reinforcement adjusts scores at RETRIEVAL +STAGE 3: Frequency/Utility tracks VALUE over time +STAGE 4: Auto-Pruning DELETES low-value memories + +Result: FULL LIFECYCLE MANAGEMENT - from birth to death! +``` + +### The Pitch: +> "MemU memories now have a LIFECYCLE - they're born, they grow, they age, they die!" + +--- + +## 🏆 COMBO 5: "MEMORY TYPES" (NEW!) + +**Theme:** Different memory types for different purposes + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| TaskMemory type | ReMe | 3 pts | MEDIUM | +| PersonalMemory type | ReMe | 3 pts | MEDIUM | +| ToolMemory type | ReMe | 5 pts | HIGH | +| Memory Deduplication | ReMe | 3 pts | MEDIUM | + +**Total: 14 pts | MEDIUM-HIGH effort** + +### Why it's MAD: + +``` +TaskMemory: "Complete the report by Friday" + - when_to_use: "When user asks about deadlines" + - content: "Report due Friday" + +PersonalMemory: "User prefers dark mode" + - target: "user_preferences" + - reflection_subject: "ui_settings" + +ToolMemory: "file_reader tool usage history" + - tool_call_results: [...] + - statistics: {avg_token_cost, success_rate, avg_score} + +Result: Specialized memory for specialized tasks! +``` + +### The Pitch: +> "MemU now has SPECIALIZED MEMORY - task memory, personal memory, tool memory!" + +--- + +## 🏆 COMBO 6: "WORKING MEMORY COMPRESSION" (NEW!) + +**Theme:** Handle long conversations without losing context + +| Component | Source | Points | Effort | +|-----------|--------|--------|--------| +| MessageCompressOp | ReMe | 3 pts | MEDIUM | +| MessageCompactOp | ReMe | 3 pts | MEDIUM | +| MessageOffloadOp | ReMe | 3 pts | MEDIUM | + +**Total: 9 pts | MEDIUM effort** + +### Why it's MAD: + +``` +LONG CONVERSATION (50k tokens) → COMPRESS → STATE SNAPSHOT (5k tokens) + +Modes: +- COMPACT: Store full content externally, keep short previews +- COMPRESS: LLM-based compression to generate dense summaries +- AUTO: Compact first, then compress if needed + +Result: Handle infinite conversations without context overflow! +``` + +### The Pitch: +> "MemU now handles INFINITE conversations - compress, compact, never forget!" + +--- + +## 📊 UPDATED COMPARISON TABLE + +| Combo | Points | Effort | WOW Factor | Complexity | Recommendation | +|-------|--------|--------|------------|------------|----------------| +| 1. Intelligent Lifecycle | 9 | LOW | ⭐⭐⭐⭐ | LOW | ✅ SAFE BET | +| 2. Smart Gate | 9-11 | MEDIUM | ⭐⭐⭐⭐ | MEDIUM | ✅ GOOD | +| 3. Knowledge Brain | 11 | HIGH | ⭐⭐⭐⭐⭐ | HIGH | ⚠️ RISKY | +| 4. Memory Evolution | 12 | LOW-MED | ⭐⭐⭐⭐⭐ | MEDIUM | 🏆 **BEST COMBO** | +| 5. Memory Types | 14 | MED-HIGH | ⭐⭐⭐⭐⭐ | HIGH | 🔥 HIGH POINTS | +| 6. Working Memory | 9 | MEDIUM | ⭐⭐⭐⭐ | MEDIUM | ✅ GOOD | + +--- + +## 🎯 FINAL RECOMMENDATION + +### For MAX POINTS with REASONABLE EFFORT: **COMBO 4 "MEMORY EVOLUTION"** + +**Why?** +1. **12 points** - highest point potential for effort +2. **LOW-MEDIUM effort** - achievable in hackathon timeframe +3. **4 features that SYNERGIZE** - each builds on the other +4. **UNIQUE story** - "memory lifecycle" is a compelling narrative +5. **Easy to demo** - show memory being filtered, decaying, getting pruned + +### Implementation Order: + +``` +Step 1: Add fields to MemoryItem model + - access_count: int = 0 + - last_accessed: datetime + - freq: int = 0 + - utility: int = 0 + - salience_score: float = 0.0 + +Step 2: Implement lightweight salience gate (keyword-based, no ML) + - SALIENT_KEYWORDS list + - NON_SALIENT_KEYWORDS list + - is_worth_saving() function + +Step 3: Implement decay-aware retrieval + - Modify cosine_topk() to apply decay formula + - Update access_count and last_accessed on retrieval + +Step 4: Implement frequency/utility tracking + - Increment freq on every retrieval + - Increment utility when memory contributes to response + +Step 5: Implement auto-pruning + - Background check for low utility/freq ratio + - Delete memories below threshold +``` + +### Files to Modify: + +``` +memU/src/memu/database/models.py # Add new fields +memU/src/memu/database/inmemory/vector.py # Decay-aware retrieval +memU/src/memu/app/memorize.py # Salience gate +memU/src/memu/app/retrieve.py # Frequency/utility tracking +memU/src/memu/app/service.py # Auto-pruning service +``` + +--- + +## 📚 Reference Implementations + +### From memoripy (Decay & Reinforcement): +- File: `prospects/memoripy/memoripy/memory_store.py` +- Key functions: `retrieve()`, `classify_memory()` + +### From memlayer (Salience Gate + Knowledge Graph): +- File: `prospects/memlayer/memlayer/ml_gate.py` - SalienceGate +- File: `prospects/memlayer/memlayer/storage/networkx.py` - Knowledge Graph +- File: `prospects/memlayer/memlayer/services.py` - CurationService + +### From ReMe (Frequency & Utility + Memory Types): +- File: `prospects/ReMe/reme_ai/vector_store/update_memory_freq_op.py` +- File: `prospects/ReMe/reme_ai/vector_store/update_memory_utility_op.py` +- File: `prospects/ReMe/reme_ai/vector_store/delete_memory_op.py` +- File: `prospects/ReMe/reme_ai/schema/memory.py` - Memory types +- File: `prospects/ReMe/reme_ai/summary/task/memory_deduplication_op.py` +- File: `prospects/ReMe/reme_ai/summary/working/` - Working memory ops + +### From MemOS (Scheduler): +- File: `prospects/MemOS/src/memos/mem_scheduler/base_scheduler.py` + +--- + +## 🚀 Ready to Implement? + +Choose your combo and let's build! 🔥 diff --git a/docs/adr/0001-workflow-pipeline-architecture.md b/docs/adr/0001-workflow-pipeline-architecture.md new file mode 100644 index 0000000..527ec5d --- /dev/null +++ b/docs/adr/0001-workflow-pipeline-architecture.md @@ -0,0 +1,35 @@ +# ADR 0001: Use Workflow Pipelines for Core Operations + +- Status: Accepted +- Date: 2026-02-24 + +## Context + +memU has multiple high-level operations (`memorize`, `retrieve`, and CRUD/patch operations) that each require multi-stage execution, LLM calls, storage writes, and optional short-circuit behavior. + +A single monolithic function per operation would make these flows hard to extend, observe, and customize. + +## Decision + +Model each core operation as a named workflow pipeline composed of ordered `WorkflowStep` units. + +- Register pipelines centrally in `MemoryService` via `PipelineManager` +- Validate required/produced state keys at pipeline registration/mutation time +- Execute through a `WorkflowRunner` abstraction (`local` by default) +- Support runtime customization by step-level config and structural mutation (insert/replace/remove) +- Provide before/after/on_error step interceptors for instrumentation and control + +## Consequences + +Positive: + +- uniform execution model across memorize/retrieve/CRUD +- explicit, inspectable stage boundaries +- extension points for custom runners and step customization +- easier interception and observability around stage execution + +Negative: + +- dict-based workflow state relies on key naming discipline +- pipeline mutation can increase behavioral variance between deployments +- more framework code compared to direct function calls diff --git a/docs/adr/0002-pluggable-storage-and-vector-strategy.md b/docs/adr/0002-pluggable-storage-and-vector-strategy.md new file mode 100644 index 0000000..270c416 --- /dev/null +++ b/docs/adr/0002-pluggable-storage-and-vector-strategy.md @@ -0,0 +1,42 @@ +# ADR 0002: Use Pluggable Storage with Backend-Specific Vector Search + +- Status: Accepted +- Date: 2026-02-24 + +## Context + +memU must support: + +- zero-setup local development +- lightweight persisted deployments +- production deployments that need scalable vector similarity + +No single storage engine fits all three cases. + +## Decision + +Adopt repository-based storage abstraction behind a `Database` protocol, with selectable providers: + +- `inmemory`: in-process state, brute-force similarity +- `sqlite`: file-based persistence, embeddings stored as JSON text, brute-force similarity +- `postgres`: SQL persistence, pgvector-enabled similarity when configured + +Vector behavior is backend-aware: + +- brute-force cosine search remains available for portability +- Postgres can use pgvector distance queries when vector support is enabled +- salience ranking (reinforcement/recency-aware) uses local scoring logic + +## Consequences + +Positive: + +- one service API works across local and production footprints +- clear backend contracts through repository interfaces +- predictable fallback behavior when native vector index is unavailable + +Negative: + +- duplicate repository logic across backends +- behavior/performance differences between providers +- SQLite and in-memory vector search does not scale as well as indexed pgvector diff --git a/docs/adr/0003-user-scope-in-data-model.md b/docs/adr/0003-user-scope-in-data-model.md new file mode 100644 index 0000000..5f71bae --- /dev/null +++ b/docs/adr/0003-user-scope-in-data-model.md @@ -0,0 +1,32 @@ +# ADR 0003: Model User Scope as First-Class Fields on Memory Records + +- Status: Accepted +- Date: 2026-02-24 + +## Context + +memU retrieval and writes need scoped operation (for example per `user_id`, `agent_id`, or session) for multi-user and multi-agent scenarios. + +Keeping scope outside stored records would force ad-hoc filtering logic and weaken data isolation. + +## Decision + +Embed scope directly into all persisted entities by merging a configurable `UserConfig.model` with core record models. + +- Scope fields are part of resource/category/item/relation models +- Repositories accept `user_data` on writes and `where` filters on reads +- API-level `where` filters are validated against configured scope fields before execution + +## Consequences + +Positive: + +- consistent filtering model across memorize/retrieve/CRUD APIs +- backend-independent scoping semantics +- supports multi-tenant and multi-agent patterns without separate storage stacks + +Negative: + +- schema/model generation complexity increases +- schema and index shape can vary by chosen scope model +- callers must keep `where` and `user` payloads aligned with configured scope fields diff --git a/docs/adr/0004-workspace-memorize-and-memory-file-system.md b/docs/adr/0004-workspace-memorize-and-memory-file-system.md new file mode 100644 index 0000000..cf82227 --- /dev/null +++ b/docs/adr/0004-workspace-memorize-and-memory-file-system.md @@ -0,0 +1,79 @@ +# ADR 0004: Add Workspace Memorize Without Touching Single-File `memorize` + +- Status: Accepted +- Date: 2026-06-22 + +## Context + +A feature branch (`feat/memory-fs-synthesis`) added a markdown "memory file system" +export, LLM synthesis of `MEMORY.md`/`SKILL.md`, an incremental update path, an +embedding-decoupling refactor, and a folder/workspace memorize entry point. + +In its original arrangement that work mutates the existing single-file +`memorize(resource_url, modality)` entry point — and ultimately rewrites it to +take a `folder` argument. We want the existing `memorize` contract to stay +untouched and to introduce the directory behavior as a new, additive entry point +instead. + +## Decision + +Land the work as seven new, re-scoped commits: + +1. **Folder backbone** — add `blob/folder.py` (scan / manifest / diff) and a new + `memorize_workspace(folder)` entry that diff-syncs a directory + (add/modify/delete with cascade-delete of stale memory) by looping over the + **unchanged** `memorize`. No export yet. +2. **Export** — add the memory-file-system export and wire a best-effort full + re-export into `memorize_workspace`. +3. **Synthesis + update path** — add LLM synthesis of `MEMORY.md`/`SKILL.md` and + the stateful initialize-vs-incremental-update path. The update-on-memorize hook + attaches to `memorize_workspace`, not `memorize`. +4. **Finish `skill/` decoupling** in `memory_fs`: the `skill/` tree is always + synthesized from descriptions, never derived from extracted skill-type memory + items. (Unblocks the step-5 restructure.) +5. **Export-tree restructure** — restructure the export tree into `resource/` / + `memory/` / `skill/` with root indexes (`INDEX.md`/`MEMORY.md`/`SKILL.md`) and + bytes-aware diffing. Only the directory restructure is taken; `skill/` stays + synthesized from descriptions (the step-4 design), so the + synthesizer/`_build_memory_files` stay as they were and only `exporter.py` is + reworked. +6. **Flow + protocol cleanup** — split the oversized flows + (`retrieve_llm.py` / `memorize_parse.py`), narrow the `Database` protocol to + repositories only, and remove the Rust scaffolding. +7. **Decouple embedding** — route vectorization through the dedicated + `memu.embedding` clients and drop `embed()` from the chat clients. Kept strictly + last and isolated so it is independently revertable. + +`memorize(resource_url, modality)` remains byte-for-byte unchanged throughout. + +Embedding can be last because the decouple only swaps the implementation behind +already-stable call sites (`_get_step_embedding_client` / `.embed()`); nothing in +steps 1–6 depends on the new embedding clients. + +## Notes + +- The export, synthesizer, and update-path work do not touch `memorize`, so those + steps are clean. Only the initialize-vs-update hook is relocated from `memorize` + to `memorize_workspace`. +- The source branch oscillated on where `skill/` content comes from — first + decoupling it to LLM synthesis, then re-coupling it to extracted skill-type + memory items. We keep the synthesis design (step 4) and take only the directory + restructure (step 5), so `exporter.py` is reworked by hand rather than applied + wholesale. +- Cleanup concerns that were originally bundled together are split so the embedding + decouple lands alone in step 7, isolated and easy to revert. + +## Consequences + +Positive: + +- existing `memorize` callers are unaffected; directory ingestion is purely additive +- embedding decoupling is isolated in its own commit (step 7) and easy to revert +- each new commit is independently reviewable + +Negative: + +- the work is split and reordered relative to the source branch, so the new history + is a re-scoped reconstruction rather than a replay of it +- `memorize_workspace` and `memorize` share the single-file core but are separate + entry points to maintain diff --git a/docs/adr/0005-dedicated-embedding-package.md b/docs/adr/0005-dedicated-embedding-package.md new file mode 100644 index 0000000..8d5e091 --- /dev/null +++ b/docs/adr/0005-dedicated-embedding-package.md @@ -0,0 +1,88 @@ +# ADR 0005: Extract Embedding into a Dedicated Package, Fully Decoupled from Chat Clients + +- Status: Accepted +- Date: 2026-06-24 + +## Context + +Embedding (vectorization) was coupled into the text/chat LLM clients: `embed()` +lived on `OpenAIClient`, `HTTPLLMClient` (which carried its own inline +`_EmbeddingBackend` classes), `AnthropicClient` (raising), and `LazyLLMClient`. +The service routed vectorization through an LLM profile (`_get_step_embedding_client` +returned a chat client), and some retrieval paths reused a single client for both +chat and `embed()`. + +This had several problems: + +- embedding providers were limited to whatever the chat client happened to + support, so embedding-only providers (Jina, Voyage) had no clean home; +- embedding payload/parse logic was duplicated (an orphaned `memu.embedding` + module plus the inline backends inside `HTTPLLMClient`); +- the `vision` capability already had a clean sibling package (`memu.vlm`, see the + LLM/VLM gateway refactor), so embedding was the odd one out; +- mixing chat and embedding on one client blurred capability boundaries and made + per-capability provider/transport selection impossible. + +ADR 0004 deliberately deferred this as an isolated, independently revertable step +("Decouple embedding"). + +## Decision + +Make embedding a first-class capability in its own package, `memu.embedding`, +structured identically to `memu.llm` and `memu.vlm`, and remove embedding from the +chat clients entirely. + +Package layout (`memu.embedding`): + +- `backends/`: per-provider request/response shapes — `openai`, `jina`, `voyage`, + `doubao` (incl. multimodal), `openrouter`; the HTTP client falls back to an + OpenAI-compatible backend for any other provider. +- `http_client` / `openai_sdk`: transport clients; `embed()` returns + `(vectors, raw_response)` so usage metadata is preserved. +- `gateway.build_embedding_client(cfg)`: dispatch on `client_backend` + (`sdk` / `httpx` / `lazyllm_backend`; `anthropic` raises — Claude has no + embeddings API). +- `defaults`: per-provider default models and embedding-only endpoints. + +Configuration and wiring: + +- New `EmbeddingConfig` / `EmbeddingProfilesConfig`, sibling to `LLMConfig` / + `VLMConfig`. `MemoryService` accepts an optional `embedding_profiles`; when + omitted, profiles are derived from the LLM profiles via + `embedding_config_from_llm`, so existing configs keep working. +- `MemoryService` builds/caches embedding clients per profile and wraps them with + the same `LLMClientWrapper`, so interceptors and usage tracking are identical to + chat/vision. +- All vectorization call sites (query embedding, category/item embedding, RAG + ranking) go through `_get_step_embedding_client` / `_get_embedding_client`. + +Decoupling the chat clients: + +- `OpenAIClient`, `HTTPLLMClient`, and `AnthropicClient` no longer expose + `embed()`; `HTTPLLMClient` no longer carries inline embedding backends. +- `LazyLLMClient` keeps `embed()` because it is a multi-capability framework + adapter and serves as the embedding transport for `lazyllm_backend`. +- `LLMConfig.embed_model` / `embed_batch_size` are retained **only** as a + backward-compat bridge consumed by `embedding_config_from_llm`; they no longer + affect chat clients. New configs should set `embedding_profiles` directly. + +## Consequences + +Positive: + +- clear capability boundaries: `memu.llm` (text), `memu.vlm` (vision), + `memu.embedding` (vectors) are isomorphic and independent; +- embedding-only providers (Jina, Voyage) are first-class; new providers are a + single backend module plus a registry entry; +- no duplicated embedding logic; one source of truth; +- per-capability provider/transport selection (e.g. chat via OpenAI, embed via + Voyage) with zero-config derivation as the default. + +Negative: + +- `LLMConfig` still carries `embed_model` / `embed_batch_size` as a bridge, so the + decoupling is not yet "pure" at the config layer (kept for backward + compatibility); +- an explicit `embedding_profiles` is required to use an embedding provider that + differs from the chat provider; +- one more package/gateway to maintain in parity with `llm`/`vlm`. diff --git a/docs/adr/0006-from-memory-item-category-to-tracked-workspace-memorization.md b/docs/adr/0006-from-memory-item-category-to-tracked-workspace-memorization.md new file mode 100644 index 0000000..d2b73fb --- /dev/null +++ b/docs/adr/0006-from-memory-item-category-to-tracked-workspace-memorization.md @@ -0,0 +1,162 @@ +ADR 0006: From Memory Item/Category to a Tracked RecallFile/RecallEntry Store for Workspace Memorize/Retrieve + +- Status: Accepted +- Date: 2026-06-30 + +## Context + +ADR 0004 landed `memorize_workspace` and the markdown "memory file system", and +deliberately kept the `skill/` tree **synthesized at export time** from the shared +per-source description trunk (never persisted, never derived from extracted items). +Skills therefore live only on disk: `MemoryFilesBuilder.build` synthesizes a +`slug -> body` map via `MemorySynthesizer`, the exporter writes `skill//SKILL.md`, +and an incremental update reads those files back off disk (`read_skills`) to feed the +"existing skills" half of the next synthesis. + +This has grown awkward: + +- skills are not first-class data — they cannot be scoped (`user_data`), queried, or + reasoned about alongside the structured store; the markdown tree *is* the database; +- the "existing skills" context for incremental synthesis is recovered by parsing + markdown back off disk, rather than read from a table; +- the memory track and the skill track are asymmetric: memory categories are + `RecallFile` rows rendered deterministically into `memory/`, while skills are an + ephemeral synthesis artifact with a parallel-but-different code path in the exporter. + +Meanwhile the memory track already does, per source file in the `memorize_workspace` +loop, exactly the shape skills want: take preprocessed content, fold it into a durable +`RecallFile.summary` via an LLM patch. Skills should reuse that machinery. + +This work lands on top of a domain-wide rename done as the first commit on the branch: +`MemoryItem` → `RecallEntry`, `MemoryCategory` → `RecallFile`, `CategoryItem` → +`RecallFileEntry` (and their repositories), a pure no-behavior-change rename that +establishes the `RecallFile`/`RecallEntry` vocabulary this ADR uses throughout. + +## Decision + +Make a **skill** a first-class `RecallFile`, distinguished from a memory category by a +new `track` column, and generate skills **inside the memorize workflow** (the +workspace path) rather than freshly at export time. + +### Data model + +- Add `RecallFile.track: str` with values `"memory" | "skill"`, defaulting to + `"memory"`. Existing rows backfill to `"memory"`. +- Rename `RecallFile.summary` → `RecallFile.content`. The `summary` name predates the + `MemoryCategory` → `RecallFile` rename; the column holds the file's body, so + `content` is the correct name. (`RecallEntry.summary` is **unchanged** — only the + file-level column is renamed.) Both changes ship in one migration since the `track` + add already forces one. +- `get_or_create_category` keys on `(name, track, *scope)`, and the Postgres + unique-with-scope index becomes `(name, track, *scope)`, so a skill and a memory + category may share a name without colliding. + +### Track isolation via `where` + +Every `RecallFile` reader scopes by track through the existing `where` filter rather +than ad-hoc post-filtering: + +- the memory/categorize/retrieve/CRUD paths and the exporter's `memory/` rendering + read `track="memory"`; +- the skill generation step and (future) skill rendering read `track="skill"`. + +This keeps skills out of memory retrieval/RAG scoring and out of `memory/`. + +### Skill generation in the workflow + +`_memorize_one` (the `memorize_workspace` per-file path) runs a **new** workflow +(`memorize_workspace`) — the existing `memorize` steps plus a skill-generation step — +rather than overloading the shared `memorize` pipeline (single-file `memorize()` is +unchanged, mirroring ADR 0004's "don't touch `memorize`" stance). The new step: + +- runs per file, after the memory persist step, taking `preprocessed_resources` as + input (it has no data dependency on the memory categorize/persist outputs); +- reads existing `track="skill"` `RecallFile`s as the "existing skills" context — read + from the DB, not from disk — so file *N* sees skills created by files *1..N-1*; +- reuses the existing skill-synthesis prompt/algorithm (processed content + existing + skills → skills to add/replace), now emitting `(name, description, body)`; +- persists each skill **directly** as a `RecallFile(track="skill", content=body)`, + embedding `name + description`, bypassing the `RecallEntry` plane entirely; +- is gated behind `memory_files_config.synthesize`. + +### Export (mirrors the memory track) + +The exporter mirrors the memory track exactly: it renders each `track="skill"` +`RecallFile` deterministically into `skill/.md` (just as `memory/.md` is +rendered from memory-track files — one shared, track-agnostic renderer), and the +synthesizer's skill role is reduced to producing a single overview `SKILL.md` body (the +mirror of `MEMORY.md`) synthesized from the skill files' bodies, rather than the +per-skill bodies. The old description-based skill synthesis, the `RecallEntry` skill +bypass, and the `skill//SKILL.md` folder layout are gone; `read_skills` is +replaced by `read_skill_body` (the SKILL.md-overview analog of `read_memory_body`). +`SKILL.md` falls back to a deterministic link index when `synthesize=False`, mirroring +`MEMORY.md`. + +### Retrieve (consuming the track) + +The `track` column also opens the read side. A new `retrieve_workspace` entry point — the +read-side analog of `memorize_workspace`, and the simple counterpart to the routing-heavy +`retrieve` — does single-shot, LLM-free retrieval: it embeds the query once and ranks the +file/entry/resource layers by vector similarity, with none of the intention routing, +sufficiency checks, or summarization of the RAG path. The file layer scopes by `track` +through `RetrieveFileConfig.tracks`, so skill-track `RecallFile`s can be retrieved (or +excluded) independently of the memory track — the read-side mirror of the track isolation +above. Driven by a new `RetrieveWorkspaceConfig`. + +## Scope of these commits + +This ADR spans four commits on the branch (a fifth — gating new-category creation behind +`allow_new_categories` — is an unrelated fix to the existing taxonomy path): + +1. **Domain rename** (foundation). `MemoryItem` → `RecallEntry`, `MemoryCategory` → + `RecallFile`, `CategoryItem` → `RecallFileEntry`, and the matching repositories. Pure + rename, no behavior change; establishes the vocabulary the rest of this work assumes. +2. **Skill track + workflow.** Schema foundation (`track` column, `summary` → `content`, + `(name, track)` keying, migration), track isolation (memory-track readers filter + `track="memory"`), and the new `memorize_workspace` workflow with its per-file skill + generation step that persists `track="skill"` `RecallFile`s. +3. **Export mirror.** `skill/.md` rendered deterministically from the DB + a + synthesized/indexed `SKILL.md` overview, symmetric with `memory/` + `MEMORY.md`. +4. **Workspace retrieve.** The LLM-free `retrieve_workspace` path that reads the track + back out — see "Retrieve (consuming the track)" above. + +## Open issues (deferred) + +- **Skill staleness / provenance on deletion.** By bypassing the `RecallEntry` plane, + skill `RecallFile`s have no link back to the resources that produced them. The memory + track invalidates correctly on file change/delete via + `RecallEntry.resource_id → RecallFileEntry → category` cascade + (`_cascade_delete_by_urls`); skills have no equivalent, so a skill derived from a + now-deleted source file lingers. We do not yet have a chosen solution (candidates: a + provenance link from skill files to contributing resources, or a full skill-track + rebuild on `diff.has_removals`). Until then the skill track is **append/merge-only** + and does not shrink on deletion. *A solution is expected in a future commit.* + +- **Cost / latency.** Skill synthesis moves from ~1 merged LLM call per workspace sync + to one call per added/modified file (consistent with how per-file memory summaries + already work, and gated behind `synthesize`). Plus one `SKILL.md`-overview synthesis + per export, the mirror of the existing `MEMORY.md` synthesis. Acceptable, but noted. + +## Consequences + +Positive: + +- skills are first-class, scoped, queryable rows; the markdown tree stops being the + only home for skills; +- the "existing skills" context for incremental generation is a table read, not a + markdown re-parse; +- memory and skill tracks become symmetric (`RecallFile` rows distinguished by + `track`), collapsing two divergent code paths; +- single-file `memorize()` stays untouched; the skill step lives only on the workspace + path; +- making skills first-class rows lets the new `retrieve_workspace` path read them back by + `track`, so the workspace gets a symmetric memorize/retrieve pair without skills needing + a bespoke read path. + +Negative: + +- a schema migration is required (`track` add + `summary`→`content` rename); existing + Postgres/SQLite deployments need it applied (fresh DBs get it from `create_all`); +- every `RecallFile` reader must now be track-aware, a cross-cutting concern threaded + through `where`; +- deletion/provenance for skills is a known, deferred gap (see open issues). diff --git a/docs/adr/0007-three-independent-memory-lines-wiki-graph.md b/docs/adr/0007-three-independent-memory-lines-wiki-graph.md new file mode 100644 index 0000000..be04261 --- /dev/null +++ b/docs/adr/0007-three-independent-memory-lines-wiki-graph.md @@ -0,0 +1,242 @@ +ADR 0007: Split Memorize/Retrieve into Three Independent Lines on a Layered Kernel + +- Status: Proposed +- Date: 2026-07-01 +- Supersedes: ADR 0004 (workspace memorize + memory file system), ADR 0006 (skill as a `RecallFile` track) + +## Context + +ADR 0004 added `memorize_workspace`/the markdown memory file system; ADR 0006 promoted +skills to a `track="skill"` `RecallFile` generated inside the workspace memorize workflow. +Living with both has surfaced a consistent coupling problem. + +1. **One pipeline does everything, per file.** `memorize_workspace(folder)` is a fan-out: + for each changed file it runs the full `memorize` workflow *plus* a `generate_skills` + step, then rebuilds the entire `INDEX.md`/`MEMORY.md`/`SKILL.md` tree. Memory + extraction, skill synthesis, and indexing all fire for every file regardless of what + that file is for. + +2. **`generate_skills` is a parallel mechanism that bypasses the entry plane.** It + re-synthesizes skills from the raw preprocessed text (not from the extracted + `RecallEntry`s), and persists them directly as `RecallFile(track="skill")`, bypassing + the `RecallEntry → RecallFileEntry → RecallFile` cascade. That bypass *is* the cause of + ADR 0006's own deferred open issue: skills have no provenance link, so they never + invalidate on source change/delete. + +3. **The `track` column exists only to jam two different things into one table.** Memory + categories and skills share `RecallFile` and are split by `track`. That forced sharing + is the root of the asymmetry and the `generate_skills` bypass. + +4. **The read side is forked, not composed.** `retrieve_rag`/`retrieve_llm` (heavy + intention-routing + sufficiency RAG over the structured store) and `retrieve_workspace` + (flat embedding recall) share **no** handlers — `_rag_*`/`_llm_*` vs `_ws_*` are + independent reimplementations of the same recall. The write side composes + (`memorize_workspace` = `memorize` + one step); the read side does not. + +The deeper observation: `INDEX` / `MEMORY` / `SKILL` are not three outputs of one process — +they are three different **treatments** of source material, with three different **read +models**. Routing them through one pipeline and one store is the coupling. + +A second observation refines the design: **embedding search and a graph are separable.** +Embedding search (embed → vector rank) is a flat capability over a set of pages; a graph +adds edges + traversal. Of the three layers, memory and the workspace index are +*relational* (entities interlink; documents reference each other and have hierarchy — +traversal matters), while skills are a *retrieval* problem ("which skill fits my current +task" = flat top-k), where a graph adds cost for little value. + +## Decision + +Refactor `memorize`/`retrieve` into **three independent lines**, each fed by its own +source, owning its own store and output, and sharing a **layered wiki-graph kernel**. + +### The three lines + +| Line | Source | Output index | L0 → L1 → L2 (raw → doc → item slices) | Read | +| --- | --- | --- | --- | --- | +| **chat** | conversation logs | `MEMORY.md` | raw chat → memory category → paragraph slices | hybrid: embedding + BM25 | +| **workspace** | workspace files | `INDEX.md` | raw media → caption paragraph → caption slices | hybrid: embedding + BM25 | +| **skill** | execution / tool traces | `SKILL.md` | raw logs → skill md → per-skill descriptions | hybrid: embedding + BM25 | + +Each line keeps the established output-file name as its L1 index/map, over its own L0/L1/L2 stores. + +> **Note.** This section's original "graph wiki / graph traversal" framing is **superseded** +> by "Refinement: hybrid retrieval (embedding + BM25), no graph" below; there is no graph. + +### Shared kernel (one implementation, three instances) + +All three lines share one **in-process** kernel: record stores (resource / item / category), +embedding, **hybrid search (cosine embedding + BM25 keyword)**, rank, markdown render, and +manifest diff. There is **no graph layer** and no external database — see "Refinement: hybrid +retrieval" below. + +### Per-line data model (independent stores, no `track`) + +``` +Node (a wiki page / file): id · slug · content(md) · embedding · source_ref · meta +Edge (a [[wikilink]]): from_node · to_node · type? ← graph lines only +``` + +`INDEX.md` / `MEMORY.md` / `SKILL.md` are rendered deterministically from each line's Node +set. The three lines have **separate stores** — there is no shared table and no `track` +discriminator. + +### Layer model: L0 / L1 / L2 (refines the data model above) + +The generic "Node/Edge" sketch above is refined into three **representation layers**. **Every +line has all three.** The layers share a uniform *role* but hold line-specific artifacts, and +each is **derived from the one below** (L0 → L1 → L2): + +| Layer | Role (uniform) | chat | workspace | skill | +| --- | --- | --- | --- | --- | +| **L0** | raw source (`resource`) | raw chat corpus | raw multimodal data | raw agent-run logs | +| **L1** | coarse document derived from L0 | the classified **memory category** file | the preprocessed **caption paragraph** | the **skill markdown** | +| **L2** (**item**) | fine slices/extracts of L1 — the **embed/search unit** | slices of the category's paragraphs | slices of the caption paragraph | the **description** extracted per skill | + +- **L1 ⊇ L2 in every line**: an *item* (L2) is a slice/extract of its L1 document. This + **inverts the earlier wording** ("L1 = atomic item, L2 = category"): now **L1 is the coarse + document and L2 is the fine item**, consistently across all three lines. +- **Derivation** runs L0 → L1 → L2: preprocess the raw source into the L1 document, then + slice/extract it into L2 items. +- Because the meanings are now per-line (and inverted from the old model), they **no longer map + 1:1 onto `Resource`/`RecallEntry`/`RecallFile`**; each line's store holds a resource (L0), a + document (L1), and item slices (L2). + +**Retrieval (hybrid, no graph, single pass).** The **L2 items are embedded**; a query runs +one hybrid pass — cosine embedding + BM25 keyword, each min-max normalized and fused — over +the L2 items, and the top items **roll up to their L1 document** (and its L0 resource) for the +result. No `item ↔ item` edges, no entity index, no traversal, no multi-hop. + +**Markdown output.** `MEMORY.md` / `SKILL.md` render the **L1** documents (memory categories / +skills); `INDEX.md` indexes the **L0** resources. L2 items live only in the store as the +embed/search units, not as separate files. + +**Seam with preprocessing.** Preprocessing is the **injected first step of every line's +memorize pipeline** (`preprocess → …`): it takes a source *reference* and produces the **L1 +document** (e.g. caption paragraph / classified memory / skill md) from the raw L0 source; +slicing/extracting that document into **L2 items** is the next step. Like the other treatments +(embed, and the per-line slice/extract), `preprocess` is injected so lines stay testable; a +real implementation wraps `memu.preprocess` (fetch via blob + modality decode + describe). + +> **Divergence note.** This redefinition inverts L1/L2 and gives every line all three layers, +> so it is **ahead of the current `memu.lines` code** (which still has L1 = atomic item, L2 = +> grouping, workspace with no L2, skill with no L1). The code will be re-aligned to this model. + +### Refinement: hybrid retrieval (embedding + BM25), no graph + +This **supersedes every "graph / edges / backlinks / traversal / entity linking / GraphRAG / +multi-hop" description earlier in this ADR.** L1 has **no connective structure**: + +- **No graph.** No `item ↔ item` edges, no `neighbors` / `backlinks` / `traverse`. +- **No entity layer / GraphRAG / multi-hop.** No parallel entity store, no item→entity index, + no entity-overlap ranking, no iterative expansion. + +Retrieval is a **single pass, hybrid**: a cosine embedding score and a BM25 keyword score are +each min-max normalized across candidates and fused into one rank. No iteration, no graph +walk. All three lines retrieve this way, differing only in *what* they search and *what* they +return: + +All three lines retrieve the same way — embed the **L2 items**, one hybrid pass, roll up to +the **L1 document**: + +| Line | embeds / searches (L2 items) | rolls up to / returns (L1) | +| --- | --- | --- | +| **chat** | slices of the memory category's paragraphs | the memory category (file) | +| **workspace** | slices of the caption paragraph | the caption paragraph (and its L0 resource) | +| **skill** | descriptions extracted per skill | the skill markdown | + +Rationale: without multi-hop traversal, an entity index is not a graph — it is only a ranking +feature, and it did not justify graph machinery here. Hybrid (embedding + keyword) is a +standard, cheap, single-pass retrieval covering both semantic and exact-term matches; a graph +/ entity / multi-hop signal can be added later as an explicit, measured enhancement if a +benchmark shows it helps. + +Concretely, `Edge` / `neighbors` / `backlinks` / `traverse`, any entity store / item→entity +index, and any multi-hop loop are **not part of the design**. The stores hold records + +embeddings and expose a single `search(query_vec, query_text, k)` over the L2 items that fuses +cosine + BM25; the hits then roll up to their L1 document before returning. + +### Per-line ingest treatment (the only code that differs) + +Each line's ingest is `preprocess (raw L0 → L1 document)` → `slice/extract (L1 → L2 items)` → `embed`: + +- **chat:** classify the conversation into a memory category (L1); slice its paragraphs into items (L2). +- **workspace:** preprocess media into a caption paragraph (L1); slice it into items (L2). +- **skill:** synthesize a skill markdown (L1) from the logs; extract a description per skill as the item (L2). + +### Independent triggers + +Each line has its own source manifest and change detection. A change under one line's +source rebuilds **only that line**; lines never trigger each other. (A change in `/chat` +no longer re-synthesizes skills or rebuilds the workspace index.) + +### Shared, not duplicated + +The kernel is one implementation; the three lines are instances of it. Existing low-level +utilities stay shared *calls*, not copies: `LocalFS` fetch, the preprocess registry +(modality handlers), the embedding clients / `ClientPool`, and the workflow engine. A +single **scope convention** (`user_id` / `where`) is preserved across all three lines so +cross-line scoping ("everything about user X") still works even though the stores are +separate. + +## What this removes + +- `RecallEntry` / `RecallFile` / `RecallFileEntry` and the `track` column → per-line + `Node`/`Edge`. +- The monolithic per-file `memorize_workspace` pipeline and the `generate_skills` step → + three ingest treatments. +- `retrieve_rag` / `retrieve_llm` / `retrieve_workspace` → one shared read mechanism + (embedding search, plus graph traversal on the two graph lines). +- The skill provenance/deletion gap (ADR 0006 open issue): the skill line owns its store + and bypasses nothing; deletion is driven by the line's own manifest. + +## Consequences + +Positive: + +- Three high-cohesion modules replace the two ~1500-line `memorize.py`/`retrieve.py` + mixins; each line is end-to-end self-contained (source → ingest → store → read → + output). +- The `track` column and the `generate_skills` bypass are gone; the skill provenance gap + is resolved structurally rather than deferred. +- The read side becomes coherent — one search mechanism (+ optional traversal) instead of + three duplicated recall paths. +- Each line's structure matches its real need (graph for relational lines, flat for + retrieval), instead of an over-imposed uniform shape. +- Independent triggers cut wasted work: a chat change no longer re-synthesizes skills or + rebuilds the index. + +Negative / costs: + +- A large refactor with a data migration (track-based rows → per-line stores). +- **No cross-line edges:** an entity relevant to more than one line is duplicated, with no + cross-domain traversal. Accepted as the separation tax; a read-only weak cross-line + reference (e.g. a skill page citing a memory entity by URI) is a future option, not built + now. +- The workspace graph relies on **structural** links to avoid LLM link-inference cost; richer + semantic linking there is deliberately out of scope. +- Three stores instead of one: a cross-line query must hit three lines (mitigated by the + shared scope convention). + +## Open issues (deferred) + +- **Skill source.** Assumed to be a `/log` (trace) folder. Whether skill is a primary + source line or is *distilled* from the chat/workspace lines is unresolved; it changes + whether skill is a true third line or a second-order derivation. +- **Migration mechanics** from the current `RecallFile(track=…)` schema to three per-line + stores (sketch: `track=memory` → chat nodes; `track=skill` → skill nodes; resources/INDEX + → workspace nodes). +- **Workspace linking** — keep it purely structural, or add light semantic linking later. + +## Implementation plan + +PoC the **workspace line first**: it has the least LLM (structural links, kept-whole pages) +and the clearest graph, so it best validates the Base + Graph kernel end-to-end before the +LLM-heavy chat and skill lines are built on the same kernel. + +## Related ADRs + +- Supersedes `docs/adr/0004-workspace-memorize-and-memory-file-system.md` +- Supersedes `docs/adr/0006-from-memory-item-category-to-tracked-workspace-memorization.md` +- Builds on `docs/adr/0001-workflow-pipeline-architecture.md` (the kernel still composes as + workflow steps), `docs/adr/0002-pluggable-storage-and-vector-strategy.md`, and + `docs/adr/0005-dedicated-embedding-package.md`. diff --git a/docs/adr/0008-two-integration-surfaces-hooks-and-api.md b/docs/adr/0008-two-integration-surfaces-hooks-and-api.md new file mode 100644 index 0000000..c8a16a4 --- /dev/null +++ b/docs/adr/0008-two-integration-surfaces-hooks-and-api.md @@ -0,0 +1,274 @@ +ADR 0008: Trajectory as the Source — Zero-Code Hooks over a Programmatic API + +- Status: Proposed +- Date: 2026-07-07 +- Builds on: ADR 0007 (three independent memory lines on a layered kernel), ADR 0001 (workflow pipelines) +- Refines: ADR 0007's source model (resolves its "skill source" open issue) and output model + (no rendered index files: per-line folders of L1 documents, searched via a pure per-line + embedding index of L2 items carrying L1-pointing metadata) + +## Context + +ADR 0007 settled the *engine*: three independent lines (chat / workspace / skill) over a +shared layered kernel, driven by `memorize-workspace` (write) and `retrieve-workspace` +(read), against a per-project store. What it did **not** settle is how an *agent* — Claude +Code, Codex, OpenClaw, Hermes, or an in-house one — actually feeds sources in and gets +memory back out during a live session. + +Today there is effectively one path: a human (or an agent following `SKILL.md`) runs the +CLI by hand against a **curated folder** — sources are whatever someone placed under `chat/` +or `agent/`, and the directory decides the treatment. That is fine for "sync this folder," +but it does not give an agent **continuous** memory: the user turns and the agent's own tool +trajectory are never captured unless someone remembers to save them, and retrieved memory is +never injected unless the agent explicitly asks. + +The deeper problem is the **source model itself**. The richest record of a working session is +the agent's **trajectory** — the interleaved user messages, agent reasoning, and tool actions. +Asking users (or agents) to pre-sort that record into per-line source directories throws away +the fact that a single stretch of work simultaneously contains *who the user is*, *how the +agent got something done*, and *where the project stands*. The structural, directory-based +routing of ADR 0007's sources cannot see any of that. + +Beyond the source model, two different consumers want two different things: + +1. **A drop-in user** wants memory to "just work" with **no code changes** to their agent — + install something, and the agent starts remembering across sessions. +2. **A builder** embedding memU inside their own agent wants **explicit control** — call + retrieve at a chosen point, memorize a chosen artifact, with their own batching and + scoping. + +Trying to serve both with one mechanism has been the tension. A pure API serves the builder +but leaves the drop-in user writing glue. A pure "magic" integration serves the drop-in user +but is a black box the builder cannot steer. And every host agent exposes a **different** hook +system (Claude Code `settings.json` events, Codex/OpenClaw/Hermes each their own), so "just +use hooks" is not one integration but N. + +## Decision + +Three decisions: one philosophical (the source), one structural (the surfaces), one of +naming (the CLI). + +### 1. Trajectory is the primary source; the three lines are three extractions from it + +The agent's **trajectory** (user turns + agent actions/tool traces) becomes the primary raw +source. Instead of routing pre-sorted directories to lines, memorize runs a **semantic +extraction** over the trajectory that distills three kinds of memory — each landing on the +ADR 0007 line built for it. Concretely, each line's **L0 is a different projection of the +same trajectory** (see the layer mapping below), rather than a separate source folder. + +The lines are **renamed after what they extract** — `memory` / `skill` / `project` are the +canonical terms from here on (ADR 0007's `chat` and `workspace` are the old names of the +`memory` and `project` lines respectively): + +| Line (canonical term) | What it captures | ADR 0007 name / source dir | Output | +| --- | --- | --- | --- | +| **memory** | user facts, preferences, conversational context | chat (`/chat`) | `memory/` folder (L1 docs) + embedding index | +| **skill** | reusable procedures — how the agent accomplished something | skill (`/skill`) | `skill/` folder (L1 docs) + embedding index | +| **project** | the files the work touched — project state as reflected in their content | workspace (`/workspace`) | `project/` folder (L1 docs) + embedding index | + +**Output model: per-line L1 folders + a pure per-line embedding index. No rendered index +files.** ADR 0007's single rendered files (`MEMORY.md` / `SKILL.md` / `INDEX.md`) are +**removed**, replaced by two artifacts per line: + +- **Browsable output** — a folder per line (`memory/`, `skill/`, `project/`) holding **one + markdown file per L1 document** (per memory category, per skill, per project document). + The folder *is* the human-facing output; nothing is rendered on top of it. +- **Searchable output** — a **pure embedding index per line**: one embedding per **L2 item**, + whose embedded content is the L2 slice itself and whose **metadata points to its L1 + document** (and through it, the L0 trajectory span). A retrieval hit rolls up L2 → L1 via + that metadata alone; L2 items never materialize as files. + +Rationale: a change to one document touches one file instead of rewriting a monolith; the +search path needs no rendered artifact at all — embeddings + metadata carry the full +L2 → L1 → L0 chain; and the on-disk names match the line terms exactly. + +**Layer mapping.** Each line keeps ADR 0007's L0 → L1 → L2 derivation, but the L0s are now +three projections of the **same trajectory** instead of three separate source folders: + +| Layer (role) | memory | skill | project | +| --- | --- | --- | --- | +| **L0** — the line's projection of the trajectory | the **user queries and agent responses only** (no tool/action traces) | the **entire user–agent trajectory** (turns + actions/tool traces) | the **contents of files** touched in the trajectory | +| **L1** — summarized document (one md file in the line's folder) | the summarized **memory file** | the synthesized **skill** | the **summary** of the file's content | +| **L2** — the embedded unit (metadata → L1) | **slices** of the memory file | the skill's **description** | **slices** of the summary | + +Two properties distinguish this from ADR 0007's directory routing: + +- **Extraction is one-to-many, not routing.** A single turn can yield a user fact *and* a + reusable skill *and* a project-state update; each extractor takes what it needs from the + same trajectory. Directory routing was one-to-one by construction and could never do this. +- **The intelligence moves from the caller into the pipeline.** Nobody pre-sorts sources; the + classify/extract step (the L0 → L1 `preprocess` seam of each line, per ADR 0007) decides + what a trajectory contains. Callers — human, hook, or code — just hand over raw trajectory. + +This **resolves ADR 0007's open "skill source" question**: skill is *distilled from the +trajectory*, not a primary source line fed by a separate `/log` folder. It also shifts the +**project** line's meaning from "indexed multimodal files" (the old workspace framing) toward +**project memory**; direct file ingestion (pointing `memu memorize` at a curated folder) +remains supported as a secondary path on the same line. + +### 2. Two integration surfaces, layered — not two parallel products + +- **The API/CLI is the base plane.** `memu memorize` / `memu retrieve` (Decision 3; and the + in-process service behind them) are the single source of truth. Everything writes to and + reads from the same per-project store defined in ADR 0007. +- **Hooks are a zero-code wrapper built *on* that base plane.** The hook integration is not a + second engine; it is an adapter layer that calls the same CLI/service on the agent's + behalf. Because both surfaces share one store, a project can **mix** them freely — record + via hooks, retrieve explicitly in code, or vice versa — with no divergence. + +### Surface A — Hook integration (zero-code, default distribution) + +An agent loads memU's `SKILL.md` for capability description, and a **host adapter** wires memU +into the agent's lifecycle. The adapter does exactly three things, and only these three: + +1. **Load** — make `SKILL.md` (and the store location) visible to the agent as a capability. +2. **Record trajectory** — on each turn, append the user input and the agent's action/tool + trace as raw trajectory, then trigger memorize. The adapter does **no sorting**: the + pipeline's extraction step (Decision 1) distills memory / skill / project from it. +3. **Inject memory** — before the agent generates, retrieve against the current prompt and + splice the results into the agent's context. + +### The hook contract (one abstraction, N thin adapters) + +Host hook systems are heterogeneous, so memU defines **one contract** with two seams, and each +host ships a **thin adapter** that maps its native events onto them: + +| Seam | Fires | Does | memU call | Execution | +| --- | --- | --- | --- | --- | +| **`on_turn`** | after a turn completes (user msg + agent trace available) | append the turn's raw trajectory, trigger memorize (extraction distills memory/skill/project) | `memu memorize` | **async / background, non-blocking** | +| **`on_prompt`** | before the agent generates | retrieve for the current prompt, splice hits into context | `memu retrieve` | **synchronous, in the critical path** | + +The adapter's whole job is to bind a host's native hook events to `on_turn` / `on_prompt`. +Concretely: + +- **Claude Code** — `settings.json` hooks: a `Stop`/post-turn hook → `on_turn`; a + `UserPromptSubmit` hook → `on_prompt`. +- **Codex / OpenClaw / Hermes** — each maps its own post-turn and pre-generation events onto + the same two seams. + +New hosts are added by writing one adapter, not by touching the kernel or the CLI. + +### Execution model (per the decision above) + +- **Recording is asynchronous.** `on_turn` enqueues; memorize runs in the background and must + never block the conversation. It reuses ADR 0007's incremental manifest diff, so repeated + turns are cheap and only changed sources are processed. +- **Injection is synchronous.** `on_prompt` does a single-shot `memu retrieve` (embedding + + BM25 hybrid search, no LLM per ADR 0007) in the critical path, under a **token budget**, and + **fails open**: a miss, a timeout, or an empty store injects nothing and the turn proceeds + normally. + +### Surface B — Programmatic API (code-level control) + +Builders embedding memU call retrieve / memorize directly from their agent code (in-process +service or CLI/JSON), choosing their own timing, batching, and scope (`user_id` / `where`). +This is the base plane exposed directly — the same calls the hook adapters make, with no hook +layer in between. + +### Shared invariants (hard constraints) + +- **One store, one semantics.** Both surfaces target the same per-project store and the same + three lines. Hook mode must not invent a private store or a divergent record format. +- **Scope convention preserved.** The ADR 0007 `user_id` / `where` scope convention holds + across both surfaces, so cross-surface queries ("everything about user X") work regardless + of how it was written. +- **Hooks add no engine.** Any behavior available via hooks must be expressible as base-plane + calls; the adapter is orchestration only. + +### 3. CLI: reclaim `memu memorize` / `memu retrieve` as the canonical pair + +With the lines renamed (Decision 1), "workspace" no longer names a line — leaving the +`memorize-workspace` / `retrieve-workspace` commands carrying a retired term that now reads +as "one line" when they actually operate on the whole store. Rename the pair: + +- **`memu memorize`** and **`memu retrieve`** become the **only** commands, carrying the new + semantics (trajectory ingestion + extraction on the write side; hybrid search over all + three lines on the read side). +- The **legacy single-file `memorize` / `retrieve`** (already deprecated, documented as + never-fall-back) are **removed first**; the names are reclaimed only after removal, so no + release ships two behaviors under one name. +- **`memorize-workspace` / `retrieve-workspace` are removed outright** — no alias, no + deprecation window. This is a breaking major-update cut; adapters and `SKILL.md` ship the + canonical names in the same release. + +The accepted risk: both prior command generations break at once — old legacy users find the +reclaimed names doing something new, and `-workspace` users get command-not-found. Accepted +deliberately: this ADR is the major update, and a clean cut beats carrying a retired term in +an alias. + +## What this changes + +- **The terms unify around the extractions**: the three lines are canonically named + **memory / skill / project** (renaming ADR 0007's `chat` and `workspace`); docs, CLI + output, and code should converge on these three terms. +- **The output changes shape**: `MEMORY.md` / `SKILL.md` / `INDEX.md` are gone; each line + yields a folder of L1 documents (`memory/` / `skill/` / `project/`, one file per document) + plus a pure embedding index of L2 items whose metadata points back to L1. +- **The source model inverts**: from "users curate per-line source folders" to "the trajectory + is the source; extraction curates." Directory-based routing (`chat/` → memory, `agent/` → + skill) becomes a legacy/secondary ingestion path. +- The project line's center of gravity shifts from multimodal file indexing to **project + memory**; ADR 0007's "skill source" open issue closes as *distilled from trajectory*. +- **The CLI collapses to one pair**: `memu memorize` / `memu retrieve` (new semantics); both + the legacy single-file pair and `memorize-workspace` / `retrieve-workspace` are removed — + no aliases. +- `SKILL.md` graduates from "instructions a human/agent follows to run the CLI" to "the + capability surface a host adapter loads," while remaining valid for manual use. +- memU gains a small **host-adapter layer** (one thin adapter per host) as the unit of new + integrations, with the two-seam contract as its only coupling to the engine. +- The API surface is stated as a **first-class, supported** integration path, not just an + internal detail behind the CLI. + +## Consequences + +Positive: + +- Drop-in users get continuous cross-session memory with no code changes; builders keep full + programmatic control — from one engine, not two. +- Host heterogeneity is contained: adding Codex/OpenClaw/Hermes/etc. is one adapter against a + fixed two-seam contract, not a bespoke integration each time. +- Async record + sync inject keeps the conversation responsive (no per-turn LLM blocking) + while still injecting fresh memory before each generation. +- Mixing surfaces is safe by construction because both hit the same store with the same scope + convention. + +Negative / costs: + +- **Extraction is not free.** Directory routing was structural (zero-cost); semantic + extraction over trajectory is LLM work on every memorize, and a misclassifying extractor + pollutes lines silently — quality now depends on prompts, not conventions. +- N host adapters to build and maintain; each host's hook system evolves independently and can + break its adapter. +- Async recording means a just-finished turn may not yet be retrievable on the immediately + following prompt (eventual, not read-your-writes, consistency) — accepted for + responsiveness. +- Synchronous injection adds retrieve latency to every prompt; bounded by the token/time + budget and fail-open behavior, but non-zero. +- Auto-recording every turn raises **privacy/scope** questions (what gets persisted, and under + whose `user_id`) that manual CLI use did not. + +## Open issues (deferred) + +- **Injection shape & budget.** Where retrieved memory lands (system-prompt append vs. a + dedicated context block), and the exact token budget / ranking cutoff, are unspecified here. +- **Record batching policy.** Per-turn vs. per-session flushing, and debounce/coalescing of + rapid turns, are left to the adapter; a shared default is TBD. +- **Extraction quality bar.** What counts as a *skill-worthy* procedure or a *project-worthy* + state change (vs. noise) is a prompt/heuristic question for the extraction step; the + one-to-many contract is decided here, the classifier itself is not. +- **Trajectory format.** The wire format a host adapter emits (full transcript vs. structured + turn events, tool-output truncation) is per-adapter for now; a shared schema is TBD. +- **Privacy controls.** Opt-out, redaction, and per-turn scoping of auto-recorded content are + not designed yet. + +## Related ADRs + +- Builds on `docs/adr/0007-three-independent-memory-lines-wiki-graph.md` (engine, three lines, + store, `memorize-workspace` / `retrieve-workspace`); **refines its source model** — + trajectory replaces per-line source folders as the primary source, its "skill source" + open issue is resolved as *distilled from trajectory*, its `chat` / `workspace` lines + are renamed **memory** / **project**, and its rendered `MEMORY.md` / `SKILL.md` / + `INDEX.md` outputs are replaced by per-line L1 folders plus a pure per-line embedding + index of L2 items. +- Builds on `docs/adr/0001-workflow-pipeline-architecture.md` (both surfaces drive the same + workflow pipelines). diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..8e53b8a --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,10 @@ +# Architecture Decision Records + +- [0001: Use Workflow Pipelines for Core Operations](0001-workflow-pipeline-architecture.md) +- [0002: Use Pluggable Storage with Backend-Specific Vector Search](0002-pluggable-storage-and-vector-strategy.md) +- [0003: Model User Scope as First-Class Fields on Memory Records](0003-user-scope-in-data-model.md) +- [0004: Add Workspace Memorize Without Touching Single-File `memorize`](0004-workspace-memorize-and-memory-file-system.md) +- [0005: Extract Embedding into a Dedicated Package, Fully Decoupled from Chat Clients](0005-dedicated-embedding-package.md) +- [0006: Promote Skills to a First-Class `RecallFile` Track, Generated Inside Memorize](0006-from-memory-item-category-to-tracked-workspace-memorization.md) +- [0007: Split Memorize/Retrieve into Three Independent Lines on a Layered Wiki-Graph Kernel](0007-three-independent-memory-lines-wiki-graph.md) +- [0008: Trajectory as the Source — Zero-Code Hooks over a Programmatic API](0008-two-integration-surfaces-hooks-and-api.md) diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..5a907e3 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,392 @@ +# memU Architecture + +## Purpose and scope + +This document describes the self-hosted `memu` Python package architecture as +implemented in this repository, reflecting the `RecallFile`/`RecallEntry` data +model and the workspace memorize/retrieve paths introduced by ADR 0006. + +The repository also describes a hosted Cloud product in `README.md`, but this +document focuses on the local `MemoryService` runtime and its code paths. + +## System overview + +memU implements structured agent memory with four persistent record types: + +- `Resource`: raw source artifacts (conversation/document/image/video/audio) +- `RecallEntry`: extracted atomic memories with embeddings (`memory_type` + `summary`) +- `RecallFile`: grouped topic files — a memory category **or** a skill, distinguished + by a `track` column (`"memory" | "skill"`) +- `RecallFileEntry`: entry-to-file relation edges + +At runtime, `MemoryService` orchestrates ingestion, retrieval, manual CRUD, and an +optional markdown export over these records. + +> **Vocabulary note.** Earlier revisions named these `MemoryItem` / +> `MemoryCategory` / `CategoryItem`. ADR 0006 renamed them to `RecallEntry` / +> `RecallFile` / `RecallFileEntry` (and renamed `RecallFile.summary` → `content`), +> a pure-rename foundation on top of which the skill track was added. A few +> back-compat dict views on the `Database` protocol (`items`, `categories`, +> `relations`) retain the older names. + +```mermaid +flowchart TD + A["Resource or Query"] --> B["MemoryService (composition root)"] + B --> C["Workflow Pipelines"] + C --> D["Capability Clients (LLM / VLM / Embedding)"] + C --> E["Database Repositories"] + E --> F["Resources"] + E --> G["RecallEntries"] + E --> H["RecallFiles (track: memory | skill)"] + E --> I["RecallFileEntries"] + B --> J["Memory File-System Export (INDEX/MEMORY/SKILL .md)"] +``` + +## Core runtime components + +### `MemoryService` as composition root + +`src/memu/app/service.py` constructs and owns: + +- typed configs (`LLMProfilesConfig`, `DatabaseConfig`, `MemorizeConfig`, + `RetrieveConfig`, `RetrieveWorkspaceConfig`, `MemoryFilesConfig`, `UserConfig`, + `EmbeddingProfilesConfig`) +- storage backend (`build_database(...)`) +- resource filesystem fetcher (`LocalFS`) +- per-capability client pools (one `ClientPool` each for chat/LLM, VLM and + embedding; see `src/memu/app/client_pool.py`) +- workflow and LLM interceptor registries +- workflow runner (`local` by default, pluggable) +- named workflow pipelines via `PipelineManager` +- the markdown memory file-system builder (`MemoryFilesBuilder`, + `src/memu/app/memory_files.py`), which owns the exporter/synthesizer/lock + +The three client pools (`ClientPool[Config, Client]`) replace hand-rolled +per-capability caches: each lazily builds and caches a client per profile name +through the matching gateway (`build_llm_client` / `build_vlm_client` / +`build_embedding_client`), so adding a new capability is one more pool rather +than another copy of the get/cache/build dance. + +Public APIs are assembled by mixins: + +- `MemorizeMixin`: `memorize(...)`, `memorize_workspace(...)` +- `RetrieveMixin`: `retrieve(...)`, `retrieve_workspace(...)` +- `CRUDMixin`: list/clear/create/update/delete memory operations + +### Workflow engine + +All major operations execute as workflows (`WorkflowStep`) with: + +- explicit required/produced state keys (`requires` / `produces`) +- declared capability tags (`llm`, `vector`, `db`, `io`, `vision`) +- per-step config (for profile selection) + +`PipelineManager` validates step dependencies at registration/mutation time and +supports runtime pipeline revisioning (`config_step`, `insert_before/after`, +`replace_step`, `remove_step`). + +`WorkflowRunner` is a protocol; the default `LocalWorkflowRunner` executes +sequentially with `run_steps(...)`. Workflow state is a plain `dict[str, Any]`, +so step contracts are validated by key names rather than static types. + +Registered pipelines (see `MemoryService._register_pipelines`): + +| Pipeline | Entry point | +| ------------------------ | ---------------------------- | +| `memorize` | `memorize(...)` | +| `memorize_workspace` | `memorize_workspace(...)` | +| `retrieve_rag` | `retrieve(...)` (method=rag) | +| `retrieve_llm` | `retrieve(...)` (method=llm) | +| `retrieve_workspace` | `retrieve_workspace(...)` | +| `patch_create` / `patch_update` / `patch_delete` | `CRUDMixin` create/update/delete | +| `crud_list_recall_entries` / `crud_list_recall_files` / `crud_clear_memory` | `CRUDMixin` list/clear | + +### Interception and observability hooks + +Two interceptor systems exist: + +- workflow step interceptors: before/after/on_error around each step +- LLM call interceptors: before/after/on_error around `chat/summarize/vision/embed/transcribe` + +LLM wrappers also extract best-effort usage metadata from raw provider responses. + +## Ingestion architecture (`memorize`) + +`memorize(...)` executes the `memorize` pipeline: + +1. `ingest_resource`: fetch local/remote resource into `blob_config.resources_dir` + via `LocalFS`. For the `document` modality, rich formats (PDF, Word, + PowerPoint, Excel, HTML, EPub) are converted to Markdown text via + [MarkItDown](https://github.com/microsoft/markitdown) (`memu.blob.document_text`), + while plain text (`.txt`/`.md`/...) is read directly. MarkItDown is an optional + extra: `pip install 'memu-py[document]'`. +2. `preprocess_multimodal`: modality-specific preprocessing for + conversation/document/audio (text-oriented path) and image/video + (vision-oriented path) +3. `extract_entries`: per-memory-type LLM extraction into structured `RecallEntry` + records (`memory_type` ∈ profile/event/knowledge/behavior/skill/tool by default) +4. `dedupe_merge`: placeholder stage (currently pass-through) +5. `categorize_entries`: persist resource + recall entries + entry-file relations + and embeddings (memory track) +6. `persist_index`: update recall-file summaries; optionally persist entry references +7. `build_response`: return resource(s), entries, files, relations + +Recall-file (category) bootstrap is lazy and scoped: files are initialized when +needed with embeddings, and mapped by normalized name. New-category creation can +be gated behind `allow_new_categories`. + +### Workspace ingestion (`memorize_workspace`) + +`memorize_workspace(...)` syncs a folder and, per changed file, runs the +**`memorize_workspace`** pipeline: the standard `memorize` steps plus a final +`generate_skills` step (ADR 0006). That step: + +- runs per file, after the memory persist step, taking `preprocessed_resources` + as input (no data dependency on the memory categorize/persist outputs); +- reads existing `track="skill"` `RecallFile`s from the DB as the "existing skills" + context, so file *N* sees skills created by files *1..N-1*; +- emits `(name, description, body)` skills and persists each **directly** as a + `RecallFile(track="skill", content=body)`, embedding `name + description` and + bypassing the `RecallEntry` plane entirely; +- is gated behind `memory_files_config.synthesize`. + +Single-file `memorize(...)` is intentionally left untouched and does **not** run +skill generation or drive the exporter. + +## Retrieval architecture (`retrieve`) + +`retrieve(...)` chooses one of two pipelines from `retrieve_config.method`: + +- `retrieve_rag` (embedding-driven ranking) +- `retrieve_llm` (LLM-driven ranking) + +Both share the same staged pattern (step ids): + +1. `route_intention` — route intention + optional query rewrite +2. `route_file` — recall-file (category) recall +3. `sufficiency_after_file` — optional early stop +4. `recall_entries` — entry recall +5. `sufficiency_after_entries` — optional early stop +6. `recall_resources` — raw resource recall +7. `build_context` — response build + +Key behavior: + +- `where` filters are validated against `user_model` fields before querying +- file readers scope to `track="memory"` so skills stay out of memory RAG scoring +- RAG path uses vector similarity (and optional salience ranking for entries) +- LLM path ranks IDs from formatted file/entry/resource context +- each stage can stop early if sufficiency check decides context is enough + +### Workspace retrieval (`retrieve_workspace`) + +`retrieve_workspace(...)` is the LLM-free, embedding-only counterpart of the +routing-heavy `retrieve` (ADR 0006). Driven by `RetrieveWorkspaceConfig` +(`method="rag"` only), it embeds the query once and ranks the file/entry/resource +layers by vector similarity, with **no** intention routing, query rewrite, +sufficiency checks, or summarization. Steps: `recall_files` → `recall_entries` +→ `recall_resources` → `build_response`. The file layer scopes by track via +`RetrieveFileConfig.tracks`, so skill-track files can be retrieved (or excluded) +independently of the memory track — the read-side mirror of the ingestion track +isolation. + +## Data and storage architecture + +### Repository contracts + +Storage is abstracted through a `Database` protocol (`src/memu/database/interfaces.py`) +with four repositories: + +- `ResourceRepo` (incl. `vector_search_resources`) +- `RecallEntryRepo` (incl. `vector_search_items`, similarity/salience) +- `RecallFileRepo` +- `RecallFileEntryRepo` + +Vector ranking over **stored** embeddings is a repository responsibility, which +keeps the retrieval layer from reaching into any concrete backend. The pure +cosine/salience math lives in the storage-neutral `memu.vector` module (not under +any backend), so the app layer and every backend depend on it instead of on each +other. (File recall still ranks freshly re-embedded summaries at query time in +the retrieval layer, since that is query-time policy rather than search over +stored vectors.) + +### Backends + +`build_database(...)` selects backend by `database_config.metadata_store.provider`: + +- `inmemory`: in-process dict/list state +- `sqlite`: SQLModel persistence, embeddings stored as JSON text, brute-force cosine search +- `postgres`: SQLModel persistence with pgvector support (when enabled), local fallback ranking when needed + +`DatabaseConfig` auto-derives `vector_index`: `pgvector` for postgres, `bruteforce` +otherwise. For Postgres, startup runs migration bootstrap and attempts +`CREATE EXTENSION IF NOT EXISTS vector` in `ddl_mode="create"`. + +### Track column and scope model + +`RecallFile.track` (`"memory" | "skill"`, default `"memory"`) splits the two +tracks within one table. `get_or_create_category` keys on `(name, track, *scope)`, +and the Postgres unique-with-scope index is `(name, track, *scope)`, so a skill +and a memory category may share a name without colliding. Every `RecallFile` +reader scopes by track through the `where` filter rather than ad-hoc +post-filtering. + +`UserConfig.model` is merged into record/table models so scope fields (for example +`user_id`) become first-class columns/attributes across resources, entries, files, +and relations. This is why `where` filters and `user_data` writes are consistently +available across APIs. + +## LLM/provider architecture + +LLM access is profile-based (`llm_profiles`): + +- `default` profile for chat-like tasks +- `embedding` profile for embedding tasks (auto-derived from default if not set) + +Per-step profile routing happens through step config (`chat_llm_profile`, +`embed_llm_profile`, or `llm_profile`). + +Client backends (selected per profile via `client_backend`): + +- `sdk`: official OpenAI SDK wrapper +- `anthropic`: official Anthropic/Claude SDK wrapper +- `httpx`: provider-adapted HTTP backend (OpenAI, Claude, Grok, DeepSeek, Kimi, + MiniMax, Doubao, OpenRouter — see `memu.llm.backends`) +- `lazyllm_backend`: LazyLLM adapter + +### VLM (vision-language) architecture + +Image/video understanding uses a dedicated `memu.vlm` package, a sibling of +`memu.llm` mirroring its layout (`backends/`, `*_client.py`, `gateway.py`), +scoped to the multimodal `vision` capability. Only providers whose first-party +API offers native image understanding are included (`memu.vlm.backends`: +OpenAI, Claude, Grok, Kimi, MiniMax, Doubao, OpenRouter); text-only providers +such as DeepSeek are intentionally excluded. + +VLM profiles are **derived** from the LLM profiles (`vlm_config_from_llm`): each +VLM client reuses the matching LLM profile's provider, credentials, and +`client_backend`, swapping only the model for that provider's latest VLM +(`memu.vlm.VLM_PROVIDER_DEFAULTS`), falling back to the LLM chat model when the +provider has no known VLM. This makes vision work with zero extra configuration. + +During `preprocess_multimodal`, `MemoryService` routes by modality: `image` and +`video` use the VLM client (`_get_vlm_client`, profile from +`memorize_config.vlm_profile`), while `conversation`/`document`/`audio` use the +chat LLM client. VLM clients are cached per profile and wrapped by the same +`LLMClientWrapper`, so interceptors and usage metadata behave identically. + +### Embedding (vectorization) architecture + +Vectorization uses a dedicated `memu.embedding` package, a sibling of `memu.llm` +and `memu.vlm` mirroring their layout (`backends/`, `*_client.py`/`openai_sdk.py`, +`gateway.py`, `defaults.py`), scoped to the `embed` capability used by vector +search. `memu.embedding.backends` covers OpenAI, Jina, Voyage, Doubao and +OpenRouter; the HTTP client falls back to an OpenAI-compatible backend for any +other provider. + +Embedding is **fully decoupled** from the text/chat clients: `OpenAIClient`, +`HTTPLLMClient` and `AnthropicClient` no longer expose `embed()`. All +vectorization — query embedding, file/entry embedding, RAG ranking — flows +through `_get_step_embedding_client` / `_get_embedding_client`, which build +dedicated `memu.embedding` clients. (`LazyLLMClient` remains multi-capability, as +it is the embedding transport for the `lazyllm_backend`.) + +Embedding profiles (`EmbeddingConfig`) are **derived** from the LLM profiles +(`embedding_config_from_llm`) by default, reusing each profile's provider, +credentials and transport. Prefer passing an explicit `embedding_profiles` to +`MemoryService` to point vectorization at a dedicated embedding provider (e.g. +`jina`/`voyage`) independently of the chat provider. Embedding clients are built +via `build_embedding_client` (`client_backend`: `sdk`/`httpx`/`lazyllm_backend`; +`anthropic` raises, as Claude has no embeddings API), cached per profile, and +wrapped by the same `LLMClientWrapper`. `_get_step_embedding_client` resolves the +profile from step config (`embed_llm_profile`, default `embedding`). + +## Integration surfaces + +- `memu.integrations.langgraph`: LangChain/LangGraph tool adapter + (`save_memory`, `search_memory`) + +There is no built-in HTTP server or CLI in this repository; usage is programmatic +via `MemoryService` (the hosted Cloud API is a separate product). + +## Memory file system export (`memu.memory_fs`) + +`MemoryService.export_memory_files(...)` renders the structured store into a +browsable markdown tree. Each root index sits beside a sibling payload directory: + +```txt +/ +├── INDEX.md ← index of the raw files under resource/ +├── MEMORY.md ← overview + index of memory/ +├── SKILL.md ← overview + index of skill/ +├── resource/ +│ └── ← one copied raw source file (verbatim bytes) +├── memory/ +│ └── .md ← one memory-track RecallFile (description + content) +├── skill/ +│ └── .md ← one skill-track RecallFile (description + content) +└── .memufs_manifest.json ← per-artifact content hashes (diff detection) +``` + +- `resource/` holds the raw source files copied verbatim out of the blob store + (`Resource.local_path`); `INDEX.md` indexes them (name, modality, description, + link). +- `memory/.md` is one file per `track="memory"` `RecallFile`; `MEMORY.md` + links to each one. +- `skill/.md` is one file per `track="skill"` `RecallFile`, rendered by the + **same** track-agnostic renderer as `memory/` (ADR 0006). Skills are now + first-class rows, not an export-time synthesis artifact — the old + `skill//SKILL.md` folder layout is gone. + +### Deterministic vs. synthesized + +`INDEX.md`, the `resource/` copies, and the per-file `memory/.md` / +`skill/.md` documents are **always deterministic** — recomputed from the +current store, no LLM. Rendered content avoids volatile values so an unchanged +store re-exports as a no-op. + +The two root overviews are deterministic link indexes by default. When +`memory_files_config.synthesize=True`, `MemoryFilesBuilder` synthesizes their +bodies with the `synthesis_llm_profile` LLM: + +- `MEMORY.md` is synthesized from the per-source multimodal descriptions +- `SKILL.md` is synthesized from the skill-track files' bodies (the mirror of + `MEMORY.md`) + +### Initialize vs. incremental update + +`MemoryFilesBuilder.build(database, where, changed=...)` decides between two paths: + +- **Initialization** (no prior tree on disk, or `changed is None`): rebuild from + the full scoped store. `export_memory_files(user=...)` always takes this path. +- **Incremental update** (a tree already exists and a changed set is supplied): + read existing overview bodies back off disk and merge only the changed sources' + descriptions into them. + +`memorize_workspace(...)` drives this builder after each folder sync — passing the +just-created resources as the changed set, or forcing a full rebuild when files +were modified/deleted. That refresh is best-effort: an export failure is logged +and never fails the sync, since the structured memory is already persisted. The +exporter is read-only against the database, disabled by default +(`memory_files_config.enabled`), and serialized through a per-service lock. + +## Current constraints and tradeoffs + +- workflow state is dict-based, so step contracts are validated by key names + rather than static types +- SQLite/inmemory vector search is brute-force (portable but less scalable) +- file/entry summary and extraction quality are prompt/LLM dependent +- the `dedupe_merge` stage is a placeholder (currently pass-through) +- **skill provenance/deletion is a known, deferred gap** (ADR 0006): skill + `RecallFile`s bypass the `RecallEntry` plane and have no link back to their + source resources, so they do not invalidate on source change/delete. The skill + track is currently append/merge-only. + +## Related ADRs + +- `docs/adr/0001-workflow-pipeline-architecture.md` +- `docs/adr/0002-pluggable-storage-and-vector-strategy.md` +- `docs/adr/0003-user-scope-in-data-model.md` +- `docs/adr/0004-workspace-memorize-and-memory-file-system.md` +- `docs/adr/0005-dedicated-embedding-package.md` +- `docs/adr/0006-from-memory-item-category-to-tracked-workspace-memorization.md` diff --git a/docs/integrations/grok.md b/docs/integrations/grok.md new file mode 100644 index 0000000..db50183 --- /dev/null +++ b/docs/integrations/grok.md @@ -0,0 +1,53 @@ +# Grok (xAI) Integration + +MemU supports **Grok**, the AI model from xAI, as a first-class LLM provider. + +## Prerequisites + +1. **xAI Account:** You need an active account with [xAI](https://x.ai/). +2. **API Key:** Obtain an API key from the [xAI Console](https://console.x.ai/). + +## Configuration + +To enable Grok, you need to set the `XAI_API_KEY` environment variable. + +### Environment Variable + +```bash +export XAI_API_KEY="your-xai-api-key-here" +``` + +PowerShell: + +```powershell +$env:XAI_API_KEY="your-xai-api-key-here" +``` + +## Usage + +To use Grok as your LLM provider, switch the `provider` setting to `grok`. This can be done in your configuration file or when initializing the application. + +### Python Example + +```python +from memu.app.settings import LLMConfig + +# Configure MemU to use Grok +config = LLMConfig( + provider="grok", + # The default API key env var is XAI_API_KEY + # The default model is grok-2-latest +) + +print(f"Using provider: {config.provider}") +print(f"Base URL: {config.base_url}") +print(f"Chat Model: {config.chat_model}") +``` + +## Models Supported + +We currently support the following Grok models: + +* **grok-2-latest** (Default) + +The integration automatically sets the base URL to `https://api.x.ai/v1`. diff --git a/docs/langgraph_integration.md b/docs/langgraph_integration.md new file mode 100644 index 0000000..005851e --- /dev/null +++ b/docs/langgraph_integration.md @@ -0,0 +1,97 @@ +# MemU LangGraph Integration + +The MemU LangGraph Integration provides a seamless adapter to expose MemU's powerful memory capabilities (`memorize` and `retrieve`) as standard [LangChain](https://python.langchain.com/) / [LangGraph](https://langchain-ai.github.io/langgraph/) tools. This allows your agents to persist information and recall it across sessions using MemU as the long-term memory backend. + +## Overview + +This integration wraps the `MemoryService` and exposes two key tools: +- **`save_memory`**: Persists text, conversation snippets, or facts associated with a user. +- **`search_memory`**: Retrieves relevant memories based on semantic search queries. + +These tools are fully typed and compatible with LangGraph's `prebuilt.ToolNode` and LangChain's agents. + +## Installation + +To use this integration, you need to install the optional dependencies: + +```bash +uv add langgraph langchain-core +``` + +## Quick Start + +Here is a complete example of how to initialize the MemU memory service and bind it to a LangGraph agent. + +```python +import asyncio +import os +from memu.app.service import MemoryService +from memu.integrations.langgraph import MemULangGraphTools + +# Ensure you have your configuration set (e.g., env vars for DB connection) +# os.environ["MEMU_DATABASE_URL"] = "..." + +async def main(): + # 1. Initialize MemoryService + memory_service = MemoryService() + # If your service requires async init (check your specific implementation): + # await memory_service.initialize() + + # 2. Instantiate MemULangGraphTools + memu_tools = MemULangGraphTools(memory_service) + + # Get the list of tools (BaseTool compatible) + tools = memu_tools.tools() + + # 3. Example Usage: Manually invoking a tool + # In a real app, you would pass 'tools' to your LangGraph agent or StateGraph. + + # Save a memory + save_tool = memu_tools.save_memory_tool() + print("Saving memory...") + result = await save_tool.ainvoke({ + "content": "The user prefers dark mode.", + "user_id": "user_123", + "metadata": {"category": "preferences"} + }) + print(f"Save Result: {result}") + + # Search for a memory + search_tool = memu_tools.search_memory_tool() + print("\nSearching memory...") + search_result = await search_tool.ainvoke({ + "query": "What are the user's preferences?", + "user_id": "user_123" + }) + print(f"Search Result:\n{search_result}") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## API Reference + +### `MemULangGraphTools` + +The main adapter class. + +```python +class MemULangGraphTools(memory_service: MemoryService) +``` + +#### `save_memory_tool() -> StructuredTool` +Returns a tool named `save_memory`. +- **Inputs**: `content` (str), `user_id` (str), `metadata` (dict, optional). +- **Description**: Save a piece of information, conversation snippet, or memory for a user. + +#### `search_memory_tool() -> StructuredTool` +Returns a tool named `search_memory`. +- **Inputs**: `query` (str), `user_id` (str), `limit` (int, default=5), `metadata_filter` (dict, optional), `min_relevance_score` (float, default=0.0). +- **Description**: Search for relevant memories or information for a user based on a query. + +## Troubleshooting + +### Import Errors +If you see an `ImportError` regarding `langchain_core` or `langgraph`: +1. Ensure you have installed the extras: `uv add langgraph langchain-core` (or `pip install langgraph langchain-core`). +2. Verify your virtual environment is active. diff --git a/docs/providers/grok.md b/docs/providers/grok.md new file mode 100644 index 0000000..4265b5c --- /dev/null +++ b/docs/providers/grok.md @@ -0,0 +1,70 @@ +# Grok (xAI) Provider + +memU includes first-class support for [Grok](https://grok.x.ai/), allowing you to leverage xAI's powerful language models directly within your application. + +## Prerequisites + +To use this provider, you must have an active xAI account. + +1. Navigate to the [xAI Console](https://console.x.ai/). +2. Sign up or log in. +3. Create a new **API Key** in the API Keys section. + +## Configuration + +The integration is designed to work out-of-the-box with minimal configuration. + +### Environment Variables + +Set the following environment variable in your `.env` file or system environment: + +```bash +GROK_API_KEY=xai-YOUR_API_KEY_HERE +``` + +### Defaults + +When you select the `grok` provider, memU automatically configures the following defaults: + +* **Base URL**: `https://api.x.ai/v1` +* **Model**: `grok-2-latest` + +## Usage Example + +You can enable the Grok provider by setting the `provider` field to `"grok"` in your application configuration. + +### Using Python Configuration + +```python +import os +from memu.app.settings import LLMConfig +from memu.app.service import MemoryService + +# Configure the LLM provider to use Grok +llm_config = LLMConfig( + provider="grok", + api_key=os.environ.get("GROK_API_KEY", ""), +) + +# Initialize the service — pass config via llm_profiles, not llm_config +service = MemoryService(llm_profiles={"default": llm_config}) +print(f"Service initialized with model: {llm_config.chat_model}") +# Output: Service initialized with model: grok-2-latest +``` + +## Troubleshooting + +### Connection Issues +If you are unable to connect to the xAI API: +1. Verify that your `GROK_API_KEY` is set correctly and has not expired. +2. Ensure that the `base_url` is resolving to `https://api.x.ai/v1`. If you have manual overrides in your settings, they might be conflicting with the default. + +### Model Availability +If you receive a `404` or "Model not found" error, xAI may have updated their model names. You can override the model manually in the config if needed: + +```python +config = LLMConfig( + provider="grok", + chat_model="grok-beta" # Example override +) +``` diff --git a/docs/sealos-devbox-guide.md b/docs/sealos-devbox-guide.md new file mode 100644 index 0000000..4740ddc --- /dev/null +++ b/docs/sealos-devbox-guide.md @@ -0,0 +1,410 @@ +# Deploying MemU on Sealos DevBox + +This guide demonstrates how to build and deploy a **Personal AI Assistant with Long-Term Memory** using MemU on [Sealos DevBox](https://sealos.io/products/devbox). + +## Overview + +MemU enables AI agents to maintain persistent, structured memory across conversations. Combined with Sealos DevBox's 1-click cloud development environment, you can quickly build and deploy memory-enabled AI applications. + +**What we'll build:** +- A FastAPI-based AI assistant that remembers user preferences and past conversations +- Persistent memory storage using MemU's in-memory or PostgreSQL backend +- Simple REST API for chat interactions +- One-click deployment to production + +**Time to complete:** ~15 minutes + +## Prerequisites + +- [Sealos account](https://sealos.io) (free tier available) +- OpenAI API key (or compatible provider like Nebius, Groq) + +## Step 1: Create a DevBox Environment + +1. Log in to [Sealos Dashboard](https://cloud.sealos.io) +2. Navigate to **DevBox** module +3. Click **Create New Project** +4. Select **Python 3.11+** template +5. Configure resources (recommended: 2 vCPU, 4GB RAM) +6. Click **Create** - your environment will be ready in ~60 seconds + +## Step 2: Connect Your IDE + +1. In the DevBox project list, click the **VS Code** or **Cursor** button +2. Your local IDE will open with a secure SSH connection to the cloud environment +3. All code runs in the cloud, keeping your local machine free + +## Step 3: Set Up the Project + +Open the terminal in your connected IDE and run: + +```bash +# Clone or create project directory +mkdir memu-assistant && cd memu-assistant + +# Create virtual environment +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install dependencies +pip install memu fastapi uvicorn python-dotenv +``` + +## Step 4: Create the Application + +Create the following files in your project: + +### `.env` + +```env +# LLM Provider Configuration +OPENAI_API_KEY=your_api_key_here +OPENAI_BASE_URL=https://api.openai.com/v1 + +# Or use Nebius (OpenAI-compatible) +# OPENAI_API_KEY=your_nebius_key +# OPENAI_BASE_URL=https://api.tokenfactory.nebius.com/v1/ + +# Model Configuration +CHAT_MODEL=gpt-4o-mini +EMBED_MODEL=text-embedding-3-small + +# Server Configuration +HOST=0.0.0.0 +PORT=8000 +``` + +### `main.py` + +```python +""" +Personal AI Assistant with Long-Term Memory +Powered by MemU + FastAPI on Sealos DevBox +""" + +import os +from contextlib import asynccontextmanager +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +load_dotenv() + +# MemU imports +from memu.app import MemoryService + +# Global memory service +memory_service: MemoryService | None = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Initialize MemU on startup.""" + global memory_service + + llm_profiles = { + "default": { + "provider": "openai", + "base_url": os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1"), + "api_key": os.getenv("OPENAI_API_KEY"), + "chat_model": os.getenv("CHAT_MODEL", "gpt-4o-mini"), + "client_backend": "sdk", + }, + "embedding": { + "provider": "openai", + "base_url": os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1"), + "api_key": os.getenv("OPENAI_API_KEY"), + "embed_model": os.getenv("EMBED_MODEL", "text-embedding-3-small"), + "client_backend": "sdk", + }, + } + + memory_service = MemoryService(llm_profiles=llm_profiles) + print("✓ MemU Memory Service initialized") + yield + print("Shutting down...") + + +app = FastAPI( + title="MemU Assistant", + description="AI Assistant with Long-Term Memory", + lifespan=lifespan, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +class ChatRequest(BaseModel): + message: str + user_id: str = "default" + + +class ChatResponse(BaseModel): + response: str + memories_used: int + memories_stored: int + + +class MemorizeRequest(BaseModel): + content: str + user_id: str = "default" + + +@app.get("/") +async def root(): + return { + "service": "MemU Assistant", + "status": "running", + "endpoints": ["/chat", "/memorize", "/recall", "/health"], + } + + +@app.get("/health") +async def health(): + return {"status": "healthy", "memory_service": memory_service is not None} + + +@app.post("/chat", response_model=ChatResponse) +async def chat(request: ChatRequest): + """ + Chat with the AI assistant. The assistant will: + 1. Retrieve relevant memories from past conversations + 2. Generate a response using those memories as context + 3. Store new information from the conversation + """ + if not memory_service: + raise HTTPException(status_code=503, detail="Memory service not initialized") + + # Step 1: Retrieve relevant memories + retrieve_result = await memory_service.retrieve( + queries=[{"role": "user", "content": request.message}] + ) + + memories = retrieve_result.get("items", []) + memories_context = "" + if memories: + memories_context = "\n\nRelevant memories from past conversations:\n" + for mem in memories[:5]: # Limit to top 5 memories + if isinstance(mem, dict): + memories_context += f"- {mem.get('summary', str(mem))}\n" + + # Step 2: Generate response (simplified - in production, use full LLM call) + # For demo, we'll create a simple response acknowledging the memories + response_text = f"I received your message: '{request.message}'" + if memories: + response_text += f"\n\nI found {len(memories)} relevant memories that might help." + + # Step 3: Store the conversation as a new memory + import tempfile + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(f"User ({request.user_id}): {request.message}") + temp_file = f.name + + try: + memorize_result = await memory_service.memorize( + resource_url=temp_file, + modality="text", + ) + memories_stored = len(memorize_result.get("items", [])) + finally: + os.unlink(temp_file) + + return ChatResponse( + response=response_text, + memories_used=len(memories), + memories_stored=memories_stored, + ) + + +@app.post("/memorize") +async def memorize(request: MemorizeRequest): + """Store information in long-term memory.""" + if not memory_service: + raise HTTPException(status_code=503, detail="Memory service not initialized") + + import tempfile + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(request.content) + temp_file = f.name + + try: + result = await memory_service.memorize( + resource_url=temp_file, + modality="text", + ) + return { + "status": "stored", + "items_created": len(result.get("items", [])), + "categories": len(result.get("categories", [])), + } + finally: + os.unlink(temp_file) + + +@app.get("/recall") +async def recall(query: str, limit: int = 5): + """Recall memories related to a query.""" + if not memory_service: + raise HTTPException(status_code=503, detail="Memory service not initialized") + + result = await memory_service.retrieve( + queries=[{"role": "user", "content": query}] + ) + + items = result.get("items", [])[:limit] + return { + "query": query, + "memories_found": len(items), + "memories": [ + {"summary": item.get("summary", str(item)) if isinstance(item, dict) else str(item)} + for item in items + ], + } + + +if __name__ == "__main__": + import uvicorn + uvicorn.run( + "main:app", + host=os.getenv("HOST", "0.0.0.0"), + port=int(os.getenv("PORT", 8000)), + reload=True, + ) +``` + +### `requirements.txt` + +``` +memu>=0.1.0 +fastapi>=0.100.0 +uvicorn[standard]>=0.23.0 +python-dotenv>=1.0.0 +``` + +### `entrypoint.sh` + +```bash +#!/bin/bash +source venv/bin/activate +uvicorn main:app --host 0.0.0.0 --port 8000 +``` + +## Step 5: Test Locally in DevBox + +```bash +# Run the application +python main.py +``` + +Use the DevBox preview feature to access your running application, or test with curl: + +```bash +# Health check +curl http://localhost:8000/health + +# Store a memory +curl -X POST http://localhost:8000/memorize \ + -H "Content-Type: application/json" \ + -d '{"content": "User prefers dark mode and uses Python for AI development"}' + +# Chat with memory +curl -X POST http://localhost:8000/chat \ + -H "Content-Type: application/json" \ + -d '{"message": "What programming language do I use?"}' + +# Recall memories +curl "http://localhost:8000/recall?query=programming%20preferences" +``` + +## Step 6: Deploy to Production + +1. In the Sealos Dashboard, go to your DevBox project +2. Click **Create Release** to package your application +3. Click **Deploy** next to your release +4. Configure environment variables (OPENAI_API_KEY, etc.) +5. Click **Deploy** - your app will be live in minutes! + +Your application will receive a public URL like: `https://your-app.cloud.sealos.io` + +## Using with PostgreSQL (Optional) + +For production deployments with persistent storage: + +1. In Sealos Dashboard, go to **Database** module +2. Create a PostgreSQL instance +3. Update your `.env` with the connection string: + +```env +DATABASE_URL=postgresql://user:password@host:5432/memu +``` + +4. Update `main.py` to use PostgreSQL backend (see MemU documentation) + +## API Reference + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/` | GET | Service info | +| `/health` | GET | Health check | +| `/chat` | POST | Chat with memory-aware AI | +| `/memorize` | POST | Store information in memory | +| `/recall` | GET | Query stored memories | + +## Architecture + +``` +┌─────────────────────────────────────────────────────────┐ +│ Sealos DevBox │ +│ ┌─────────────────────────────────────────────────┐ │ +│ │ FastAPI Application │ │ +│ │ ┌─────────┐ ┌─────────┐ ┌─────────────────┐ │ │ +│ │ │ /chat │ │/memorize│ │ /recall │ │ │ +│ │ └────┬────┘ └────┬────┘ └────────┬────────┘ │ │ +│ │ │ │ │ │ │ +│ │ └────────────┼────────────────┘ │ │ +│ │ │ │ │ +│ │ ┌───────▼───────┐ │ │ +│ │ │ MemU Service │ │ │ +│ │ │ (Memory Mgmt)│ │ │ +│ │ └───────┬───────┘ │ │ +│ │ │ │ │ +│ │ ┌────────────┼────────────┐ │ │ +│ │ │ │ │ │ │ +│ │ ┌────▼────┐ ┌────▼────┐ ┌───▼────┐ │ │ +│ │ │ Vector │ │ LLM │ │Postgres│ │ │ +│ │ │ Store │ │ API │ │(opt.) │ │ │ +│ │ └─────────┘ └─────────┘ └────────┘ │ │ +│ └─────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + +## Benefits of This Setup + +- **Zero Infrastructure Management**: Sealos handles Kubernetes complexity +- **Instant Environment**: Ready-to-code in 60 seconds +- **Persistent Memory**: MemU maintains context across sessions +- **Scalable**: Easily scale resources as needed +- **Cost-Effective**: Pay only for what you use + +## Next Steps + +- Add authentication for multi-user support +- Integrate with Slack, Discord, or other platforms +- Use PostgreSQL for production-grade persistence +- Add conversation history UI + +## Resources + +- [MemU Documentation](https://github.com/NevaMind-AI/memU) +- [Sealos DevBox Guide](https://sealos.io/blog/how-to-setup-devbox) +- [FastAPI Documentation](https://fastapi.tiangolo.com) + +--- + +*This guide was created for the MemU PR Hackathon - 2026 New Year Challenge (Issue #228)* diff --git a/docs/sealos_use_case.md b/docs/sealos_use_case.md new file mode 100644 index 0000000..5ef9edb --- /dev/null +++ b/docs/sealos_use_case.md @@ -0,0 +1,54 @@ +# 🛡️ Context-Aware Support Agent (Sealos Edition) + +## Overview +This use case demonstrates how **MemU** enables a support agent to remember user history across sessions, deployed on a **Sealos Devbox** environment. + +Unlike a standard web app, this demo focuses on the **backend memory orchestration**. It runs as a **CLI (Command Line Interface)** tool to transparently show the internal memory logs, retrieval process, and state persistence without the abstraction layer of a UI. + +## 🚀 Quick Start + +### Prerequisites +- Sealos Devbox Environment +- Python 3.13+ +- MemU Library (installed via `make install`) + +### How to Run the Demo +Since this is a backend demonstration, you will run the agent directly in the terminal to observe the memory cycle. + +```bash +uv run python examples/sealos_support_agent.py +``` + +## 📸 Live Demo Output (Proof of Concept) + +Below is the actual output captured from the Sealos terminal. This serves as verification of the "Demonstration Quality" requirement. + +```plaintext +🚀 Starting Sealos Support Agent Demo (Offline Mode) + +📝 --- Phase 1: Ingesting Conversation History --- +👤 Captain: "I'm getting a 502 Bad Gateway error on port 3000." +🤖 Agent: (Memorizing this interaction...) +✅ Memory stored! extracted 2 items. + - [profile] Captain reported a 502 Bad Gateway error on port 3000. + +🔍 --- Phase 2: Retrieval on New Interaction --- +👤 Captain: "Hello" +🤖 Agent: (Searching memory for context...) + +💡 Retrieved Context: + Found Memory: Captain reported a 502 Bad Gateway error on port 3000. + +💬 --- Phase 3: Agent Response --- +🤖 Agent: "Welcome back, Captain. I see you had a 502 error on port 3000 recently. Is that resolved?" + +✨ Demo Completed Successfully +``` + +## 💡 Code Highlights & Justification + +- **CLI vs Web**: We chose a CLI implementation to provide clear visibility into the memory ingestion and retrieval logs, which are often hidden in web implementations. + +- **MockLLM**: Includes a MockLLM class to ensure the demo is 100% reproducible by reviewers without needing external API keys. + +- **Sealos Native**: Optimized to run within the ephemeral Sealos Devbox container lifecycle. diff --git a/docs/sqlite.md b/docs/sqlite.md new file mode 100644 index 0000000..176b7ad --- /dev/null +++ b/docs/sqlite.md @@ -0,0 +1,230 @@ +# SQLite Database Integration + +MemU supports SQLite as a lightweight, file-based database backend for memory storage. This is ideal for: + +- **Local development** and testing +- **Single-user applications** with persistent storage +- **Portable deployments** where you need a simple database solution +- **Offline-capable applications** that can't rely on external databases + +## Quick Start + +### Basic Configuration + +```python +from memu.app import MemoryService + +# Using default SQLite file (memu.db in current directory) +service = MemoryService( + llm_profiles={"default": {"api_key": "your-api-key"}}, + database_config={ + "metadata_store": { + "provider": "sqlite", + }, + }, +) + +# Or specify a custom database path +service = MemoryService( + llm_profiles={"default": {"api_key": "your-api-key"}}, + database_config={ + "metadata_store": { + "provider": "sqlite", + "dsn": "sqlite:///path/to/your/memory.db", + }, + }, +) +``` + +### In-Memory SQLite (No Persistence) + +For testing or temporary storage, you can use an in-memory SQLite database: + +```python +service = MemoryService( + llm_profiles={"default": {"api_key": "your-api-key"}}, + database_config={ + "metadata_store": { + "provider": "sqlite", + "dsn": "sqlite:///:memory:", + }, + }, +) +``` + +## Configuration Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `provider` | `str` | `"inmemory"` | Set to `"sqlite"` to use SQLite backend | +| `dsn` | `str` | `"sqlite:///memu.db"` | SQLite connection string | + +### DSN Format + +SQLite DSN follows this format: +- **File-based**: `sqlite:///path/to/database.db` +- **In-memory**: `sqlite:///:memory:` +- **Relative path**: `sqlite:///./data/memu.db` +- **Absolute path**: `sqlite:////home/user/data/memu.db` (note the 4 slashes) + +## Vector Search + +SQLite doesn't have native vector support like PostgreSQL's pgvector. MemU uses **brute-force cosine similarity** for vector search when using SQLite: + +```python +service = MemoryService( + llm_profiles={"default": {"api_key": "your-api-key"}}, + database_config={ + "metadata_store": { + "provider": "sqlite", + "dsn": "sqlite:///memu.db", + }, + "vector_index": { + "provider": "bruteforce", # This is the default for SQLite + }, + }, +) +``` + +**Note**: Brute-force search loads all embeddings into memory and computes similarity for each. This works well for moderate dataset sizes (up to ~100k items) but may be slow for larger datasets. + +## Database Schema + +SQLite creates the following tables automatically: + +- `sqlite_resources` - Multimodal resource records (images, documents, etc.) +- `sqlite_memory_items` - Extracted memory items with embeddings +- `sqlite_memory_categories` - Memory categories with summaries +- `sqlite_category_items` - Relationships between items and categories + +Embeddings are stored as JSON-serialized text in SQLite since there's no native vector type. + +## Data Import/Export + +### Export Data + +You can export your SQLite database for backup or migration: + +```python +import shutil + +# Simply copy the database file +shutil.copy("memu.db", "memu_backup.db") +``` + +### Import from SQLite to PostgreSQL + +To migrate data from SQLite to PostgreSQL: + +```python +import json +from memu.database.sqlite import build_sqlite_database +from memu.database.postgres import build_postgres_database +from memu.app.settings import DatabaseConfig +from pydantic import BaseModel + +class UserScope(BaseModel): + user_id: str + +# Load from SQLite +sqlite_config = DatabaseConfig( + metadata_store={"provider": "sqlite", "dsn": "sqlite:///memu.db"} +) +sqlite_db = build_sqlite_database(config=sqlite_config, user_model=UserScope) +sqlite_db.load_existing() + +# Connect to PostgreSQL +postgres_config = DatabaseConfig( + metadata_store={"provider": "postgres", "dsn": "postgresql://..."} +) +postgres_db = build_postgres_database(config=postgres_config, user_model=UserScope) + +# Migrate resources +for res_id, resource in sqlite_db.resources.items(): + postgres_db.resource_repo.create_resource( + url=resource.url, + modality=resource.modality, + local_path=resource.local_path, + caption=resource.caption, + embedding=resource.embedding, + user_data={"user_id": getattr(resource, "user_id", None)}, + ) + +# Similar for categories, items, and relations... +``` + +## Performance Considerations + +| Aspect | SQLite | PostgreSQL | +|--------|--------|------------| +| Setup | Zero configuration | Requires server setup | +| Concurrency | Single writer, multiple readers | Full concurrent access | +| Vector Search | Brute-force (in-memory) | Native pgvector (indexed) | +| Scale | Up to ~100k items | Millions of items | +| Deployment | Single file, portable | External service | + +## Example: Full Workflow + +```python +import asyncio +from memu.app import MemoryService + +async def main(): + # Initialize with SQLite + service = MemoryService( + llm_profiles={"default": {"api_key": "your-api-key"}}, + database_config={ + "metadata_store": { + "provider": "sqlite", + "dsn": "sqlite:///my_memories.db", + }, + }, + ) + + # Memorize a conversation + result = await service.memorize( + resource_url="conversation.json", + modality="conversation", + user={"user_id": "alice"}, + ) + print(f"Created {len(result['categories'])} categories") + + # Retrieve relevant memories + memories = await service.retrieve( + queries=[ + {"role": "user", "content": {"text": "What are my preferences?"}} + ], + where={"user_id": "alice"}, + ) + + for item in memories.get("items", []): + print(f"- {item['summary']}") + +asyncio.run(main()) +``` + +## Troubleshooting + +### Database Locked Error + +SQLite only allows one writer at a time. If you see "database is locked" errors: + +1. Ensure you're not running multiple processes writing to the same database +2. Consider using PostgreSQL for concurrent access needs +3. Use connection pooling with appropriate timeouts + +### Permission Denied + +Make sure the directory containing the SQLite file is writable: + +```bash +chmod 755 /path/to/data/directory +``` + +### Slow Vector Search + +If vector search is slow with large datasets: + +1. Consider migrating to PostgreSQL with pgvector +2. Use more selective `where` filters to reduce the search space +3. Reduce `top_k` parameters in your retrieve configuration diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md new file mode 100644 index 0000000..f198664 --- /dev/null +++ b/docs/tutorials/getting_started.md @@ -0,0 +1,181 @@ +# Quickstart: Adding Long-Term Memory to Python Agents + +Welcome to MemU! This guide will help you add robust long-term memory capabilities to your Python agents in just a few minutes. Without MemU, LLMs are limited by their context window. MemU solves this by providing an intelligent, persistent memory layer. + +## Prerequisites + +Before we begin, ensure you have the following: + +- **Python 3.13+**: MemU takes advantage of modern Python features. +- **OpenAI API Key**: This quickstart uses OpenAI's models (`gpt-4o-mini`). You will need a valid API key. + +## Step-by-Step Guide + +### 1. Installation + +Install MemU using `pip` or `uv`: + +```bash +pip install memu +# OR +uv add memu +``` + +### 2. Configuration + +MemU requires an LLM backend to function. By default, it looks for the `OPENAI_API_KEY` environment variable. + +**Linux / macOS / Git Bash:** +```bash +export OPENAI_API_KEY=sk-proj-your-api-key +``` + +**Windows (PowerShell):** +```powershell +$env:OPENAI_API_KEY="sk-proj-your-api-key" +``` + +### 3. The Robust Starter Script + +Below is a complete, production-ready script that demonstrates the full lifecycle of a memory-enabled agent: **Initialization**, **Injection** (adding memory), and **Retrieval** (searching memory). + +Create a file named `getting_started.py` and paste the following code: + +```python +""" +Getting Started with MemU: A Robust Example. + +This script demonstrates the core lifecycle of MemU: +1. **Initialization**: Setting up the client with secure API key handling. +2. **Memory Injection**: Adding a specific memory with metadata. +3. **Retrieval**: Searching for that memory using natural language. +4. **Error Handling**: Catching common configuration issues. + +Usage: + export OPENAI_API_KEY=your_api_key_here + python getting_started.py +""" + +import asyncio +import logging +import os +import sys + +from memu.app import MemoryService + +# Configure logging to show info but suppress noisy libraries +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logging.getLogger("httpx").setLevel(logging.WARNING) + + +async def main() -> None: + """Run the MemU lifecycle demonstration.""" + print(">>> MemU Getting Started Example") + print("-" * 30) + + # 1. API Key Handling + # MemU relies on an LLM backend (defaulting to OpenAI). + # We ensure the API key is present before proceeding. + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + print("[!] Error: OPENAI_API_KEY environment variable is not set.") + print("Please export it: export OPENAI_API_KEY=sk-...") + return + + try: + # 2. Initialization + # We initialize the MemoryService with: + # - llm_profiles: Configuration for the LLM (model, api_key). + # - memorize_config: Pre-defining a memory category ensures we can organize memories efficiently. + print(f"[*] Initializing MemoryService with model: gpt-4o-mini...") + service = MemoryService( + llm_profiles={ + "default": { + "api_key": api_key, + "chat_model": "gpt-4o-mini", + }, + }, + memorize_config={ + "memory_categories": [ + { + "name": "User Facts", + "description": "General and specific facts known about the user preference and identity.", + } + ] + }, + ) + print("[OK] Service initialized successfully.\n") + + # 3. Memory Injection + # We manually inject a memory into the system. + # This is useful for bootstrapping a user profile or adding explicit knowledge. + print("[*] Injecting memory...") + memory_content = "The user is a senior Python architect who loves clean code and type hints." + + # We use 'create_memory_item' to insert a single memory record. + # memory_type='profile' indicates this is an attribute of the user. + result = await service.create_memory_item( + memory_type="profile", + memory_content=memory_content, + memory_categories=["User Facts"], + ) + print(f"[OK] Memory created! ID: {result.get('memory_item', {}).get('id')}\n") + + # 4. Retrieval + # Now we query the system naturally to see if it recalls the information. + query_text = "What kind of code does the user like?" + print(f"[*] Querying: '{query_text}'") + + search_results = await service.retrieve( + queries=[{"role": "user", "content": query_text}] + ) + + # 5. Display Results + items = search_results.get("items", []) + if items: + print(f"[OK] Found {len(items)} relevant memory item(s):") + for idx, item in enumerate(items, 1): + print(f" {idx}. {item.get('summary')} (Type: {item.get('memory_type')})") + else: + print("[!] No relevant memories found.") + + except Exception as e: + print(f"\n[!] An error occurred during execution: {e}") + logging.exception("Detailed traceback:") + finally: + print("\n[=] Example execution finished.") + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Understanding the Code + +1. **Initialization**: We configure `MemoryService` with specific `llm_profiles`. This tells MemU which model to use. We also define a `memorize_config` with a "User Facts" category. Categories help the LLM organize and retrieve information more effectively. +2. **Memory Injection**: `create_memory_item` is used to explicitly add a piece of knowledge. We tag it with `memory_type="profile"` to semantically indicate this is a user attribute. +3. **Retrieval**: We use `retrieve` with a natural language query. MemU's internal workflow ("RAG" or "LLM" based) will determine the best way to find relevant memories. + +## Troubleshooting + +### `[!] Error: OPENAI_API_KEY environment variable is not set.` + +This is the most common issue. It means the script cannot find your API key which is required to communicate with OpenAI. + +**Solution:** +Ensure you have exported the key in your **current terminal session**. +- **Windows PowerShell**: `$env:OPENAI_API_KEY="sk-..."` +- **Linux/Mac**: `export OPENAI_API_KEY=sk-...` + +Also, verify that you didn't accidentally include spaces around the `=` sign in bash. + +## Next Steps + +Now that you have the basics running, consider exploring: +- **Core Concepts**: Learn about `MemoryService`, `MemoryItem`, and `MemoryCategory`. +- **Advanced Configuration**: Switch to local LLMs or use different vector stores. +- **Integrations**: Connect MemU to your existing agent framework. + +## Community Resources + +This tutorial was created as part of the MemU 2026 Challenge. For a summary of the architectural analysis, see the author's [LinkedIn Post](https://www.linkedin.com/posts/david-a-mamani-c_github-nevamind-aimemu-memory-infrastructure-activity-7418493617482207232-_MtG?utm_source=share&utm_medium=member_desktop&rcm=ACoAAFdc0CIB__DJovR2t1BOxxJ6tgEeOqVEgx4). diff --git a/examples/example_1_conversation_memory.py b/examples/example_1_conversation_memory.py new file mode 100644 index 0000000..e023fae --- /dev/null +++ b/examples/example_1_conversation_memory.py @@ -0,0 +1,117 @@ +""" +Example 1: Multiple Conversations -> Memory Category File + +This example demonstrates how to process multiple conversation files +and generate a memory category JSON file. + +Usage: + export OPENAI_API_KEY=your_api_key + python examples/example_1_conversation_memory.py +""" + +import asyncio +import os +import sys + +from memu.app import MemoryService + +# Add src to sys.path +src_path = os.path.abspath("src") +sys.path.insert(0, src_path) + + +async def generate_memory_md(categories, output_dir): + """Generate concise markdown files for each memory category.""" + + os.makedirs(output_dir, exist_ok=True) + + generated_files = [] + + for cat in categories: + name = cat.get("name", "unknown") + summary = cat.get("summary", "") + + filename = f"{name}.md" + filepath = os.path.join(output_dir, filename) + + with open(filepath, "w", encoding="utf-8") as f: + # Title + # Content - concise version + if summary: + cleaned_summary = summary.replace("", "").replace("", "").strip() + f.write(f"{cleaned_summary}\n") + else: + f.write("*No content available*\n") + + generated_files.append(filename) + + return generated_files + + +async def main(): + """ + Process multiple conversation files and generate memory categories. + + This example: + 1. Initializes MemoryService with OpenAI API + 2. Processes conversation JSON files + 3. Extracts memory categories from conversations + 4. Outputs the categories to files + """ + print("Example 1: Conversation Memory Processing") + print("-" * 50) + + # Get OpenAI API key from environment + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + msg = "Please set OPENAI_API_KEY environment variable" + raise ValueError(msg) + + # Initialize service with OpenAI using llm_profiles + # The "default" profile is required and used as the primary LLM configuration + service = MemoryService( + llm_profiles={ + "default": { + "api_key": api_key, + "chat_model": "gpt-4o-mini", + }, + }, + ) + + # Conversation files to process + conversation_files = [ + "examples/resources/conversations/conv1.json", + "examples/resources/conversations/conv2.json", + "examples/resources/conversations/conv3.json", + ] + + # Process each conversation + print("\nProcessing conversations...") + total_items = 0 + categories = [] + for conv_file in conversation_files: + if not os.path.exists(conv_file): + continue + + try: + result = await service.memorize(resource_url=conv_file, modality="conversation") + total_items += len(result.get("items", [])) + # Categories are returned in the result and updated after each memorize call + categories = result.get("categories", []) + except Exception as e: + print(f"Error: {e}") + + # Write to output files + output_dir = "examples/output/conversation_example" + os.makedirs(output_dir, exist_ok=True) + + # 1. Generate individual Markdown files for each category + await generate_memory_md(categories, output_dir) + + print(f"\n✓ Processed {len(conversation_files)} files, extracted {total_items} items") + print(f"✓ Generated {len(categories)} categories") + print(f"✓ Output: {output_dir}/") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/example_2_skill_extraction.py b/examples/example_2_skill_extraction.py new file mode 100644 index 0000000..3ca7580 --- /dev/null +++ b/examples/example_2_skill_extraction.py @@ -0,0 +1,274 @@ +""" +Example 2: Workflow & Agent Logs -> Skill Extraction + +This example demonstrates how to extract skills from workflow descriptions +and agent runtime logs, then output them to a Markdown file. + +Usage: + export OPENAI_API_KEY=your_api_key + python examples/example_2_skill_extraction.py +""" + +import asyncio +import os +import sys + +from openai import AsyncOpenAI + +from memu.app import MemoryService + +# Add src to sys.path +src_path = os.path.abspath("src") +sys.path.insert(0, src_path) + + +async def generate_skill_md( + all_skills, service, output_file, attempt_number, total_attempts, categories=None, is_final=False +): + """ + Use LLM to generate a concise task execution guide (skill.md). + + This creates a production-ready guide incorporating lessons learned from deployment attempts. + """ + + os.makedirs(os.path.dirname(output_file), exist_ok=True) + + # Prepare context for LLM + skills_text = "\n\n".join([f"### From {skill_data['source']}\n{skill_data['skill']}" for skill_data in all_skills]) + + # Get category summaries if available + categories_text = "" + if categories: + categories_with_content = [cat for cat in categories if cat.get("summary") and cat.get("summary").strip()] + if categories_with_content: + categories_text = "\n\n".join([ + f"**{cat.get('name', 'unknown')}**:\n{cat.get('summary', '')}" for cat in categories_with_content + ]) + + # Construct prompt for LLM + prompt = f"""Generate a concise production-ready task execution guide. + +**Context**: +- Task: Production Microservice Deployment with Blue-Green Strategy +- Progress: {attempt_number}/{total_attempts} attempts +- Status: {"Complete" if is_final else f"v0.{attempt_number}"} + +**Skills Learned**: +{skills_text} + +{f"**Categories**:\n{categories_text}" if categories_text else ""} + +**Required Structure**: + +1. **Frontmatter** (YAML): + - name: production-microservice-deployment + - description: Brief description + - version: {"1.0.0" if is_final else f"0.{attempt_number}.0"} + - status: {"Production-Ready" if is_final else "Evolving"} + +2. **Introduction**: What this guide does and when to use it + +3. **Deployment Context**: Strategy, environment, goals + +4. **Pre-Deployment Checklist**: + - Actionable checks from lessons learned + - Group by category (Database, Monitoring, etc.) + - Mark critical items + +5. **Deployment Procedure**: + - Step-by-step instructions with commands + - Include monitoring points + +6. **Rollback Procedure**: + - When to rollback (thresholds) + - Exact commands + - Expected recovery time + +7. **Common Pitfalls & Solutions**: + - Failures/issues encountered + - Root cause, symptoms, solution + +8. **Best Practices**: + - What works well + - Expected timelines + +9. **Key Takeaways**: 3-5 most important lessons + +**Style**: +- Use markdown with clear hierarchy +- Be specific and concise +- Technical and production-grade tone +- Focus on PRACTICAL steps + +**CRITICAL**: +- ONLY use information from provided skills/lessons +- DO NOT make assumptions or add generic advice +- Extract ACTUAL experiences from the logs + +Generate the complete markdown document now:""" + + client = AsyncOpenAI(api_key=service.llm_config.api_key) + + response = await client.chat.completions.create( + model=service.llm_config.chat_model, + messages=[ + { + "role": "system", + "content": "You are an expert technical writer creating concise, production-grade deployment guides from real experiences.", + }, + {"role": "user", "content": prompt}, + ], + temperature=0.7, + max_tokens=3000, + ) + + generated_content = response.choices[0].message.content + + # Write to file + with open(output_file, "w", encoding="utf-8") as f: + f.write(generated_content) + + return True + + +async def main(): + """ + Extract skills from agent logs using incremental memory updates. + + This example demonstrates INCREMENTAL LEARNING: + 1. Process files ONE BY ONE + 2. Each file UPDATES existing memory + 3. Category summaries EVOLVE with each new file + 4. Final output shows accumulated knowledge + """ + print("Example 2: Incremental Skill Extraction") + print("-" * 50) + + # Get OpenAI API key from environment + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + msg = "Please set OPENAI_API_KEY environment variable" + raise ValueError(msg) + + # Custom config for skill extraction + skill_prompt = """ + You are analyzing an agent execution log. Extract the key actions taken, their outcomes, and lessons learned. + + For each significant action or phase: + + 1. **Action/Phase**: What was being attempted? + 2. **Status**: SUCCESS ✅ or FAILURE ❌ + 3. **What Happened**: What was executed + 4. **Outcome**: What worked/failed, metrics + 5. **Root Cause** (for failures): Why did it fail? + 6. **Lesson**: What did we learn? + 7. **Action Items**: Concrete steps for next time + + **IMPORTANT**: + - Focus on ACTIONS and outcomes + - Be specific: include actual metrics, errors, timing + - ONLY extract information explicitly stated + - DO NOT infer or assume information + + Extract ALL significant actions from the text: + + Text: {resource} + """ + + # Define custom categories + skill_categories = [ + {"name": "deployment_execution", "description": "Deployment actions, traffic shifting, environment management"}, + { + "name": "pre_deployment_validation", + "description": "Capacity validation, configuration checks, readiness verification", + }, + { + "name": "incident_response_rollback", + "description": "Incident response, error detection, rollback procedures", + }, + { + "name": "performance_monitoring", + "description": "Metrics monitoring, performance analysis, bottleneck detection", + }, + {"name": "database_management", "description": "Database capacity planning, optimization, schema changes"}, + {"name": "testing_verification", "description": "Testing, smoke tests, load tests, verification"}, + {"name": "infrastructure_setup", "description": "Kubernetes, containers, networking configuration"}, + {"name": "lessons_learned", "description": "Key reflections, root cause analyses, action items"}, + ] + + memorize_config = { + "memory_types": ["skill"], + "memory_type_prompts": {"skill": skill_prompt}, + "memory_categories": skill_categories, + } + + # Initialize service with OpenAI using llm_profiles + # The "default" profile is required and used as the primary LLM configuration + service = MemoryService( + llm_profiles={ + "default": { + "api_key": api_key, + "chat_model": "gpt-4o-mini", + }, + }, + memorize_config=memorize_config, + ) + + # Resources to process + resources = [ + ("examples/resources/logs/log1.txt", "document"), + ("examples/resources/logs/log2.txt", "document"), + ("examples/resources/logs/log3.txt", "document"), + ] + + # Process each resource sequentially + print("\nProcessing files...") + all_skills = [] + categories = [] + + for idx, (resource_file, modality) in enumerate(resources, 1): + if not os.path.exists(resource_file): + continue + + try: + result = await service.memorize(resource_url=resource_file, modality=modality) + + # Extract skill items + for item in result.get("items", []): + if item.get("memory_type") == "skill": + all_skills.append({"skill": item.get("summary", ""), "source": os.path.basename(resource_file)}) + + # Categories are returned in the result and updated after each memorize call + categories = result.get("categories", []) + + # Generate intermediate skill.md + await generate_skill_md( + all_skills=all_skills, + service=service, + output_file=f"examples/output/skill_example/log_{idx}.md", + attempt_number=idx, + total_attempts=len(resources), + categories=categories, + ) + + except Exception as e: + print(f"Error: {e}") + + # Generate final comprehensive skill.md + await generate_skill_md( + all_skills=all_skills, + service=service, + output_file="examples/output/skill_example/skill.md", + attempt_number=len(resources), + total_attempts=len(resources), + categories=categories, + is_final=True, + ) + + print(f"\n✓ Processed {len(resources)} files, extracted {len(all_skills)} skills") + print(f"✓ Generated {len(categories)} categories") + print("✓ Output: examples/output/skill_example/") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/example_3_multimodal_memory.py b/examples/example_3_multimodal_memory.py new file mode 100644 index 0000000..83aba74 --- /dev/null +++ b/examples/example_3_multimodal_memory.py @@ -0,0 +1,137 @@ +""" +Example 3: Multimodal Processing -> Memory Category File + +This example demonstrates how to process multiple modalities (images, documents) +and generate a unified memory category JSON file. + +Usage: + export OPENAI_API_KEY=your_api_key + python examples/example_3_multimodal_memory.py +""" + +import asyncio +import os +import sys + +from memu.app import MemoryService + +# Add src to sys.path +src_path = os.path.abspath("src") +sys.path.insert(0, src_path) + + +async def generate_memory_md(categories, output_dir): + """Generate concise markdown files for each memory category.""" + + os.makedirs(output_dir, exist_ok=True) + + generated_files = [] + + for cat in categories: + name = cat.get("name", "unknown") + description = cat.get("description", "") + summary = cat.get("summary", "") + + filename = f"{name}.md" + filepath = os.path.join(output_dir, filename) + + with open(filepath, "w", encoding="utf-8") as f: + # Title + formatted_name = name.replace("_", " ").title() + f.write(f"# {formatted_name}\n\n") + + if description: + f.write(f"*{description}*\n\n") + + # Content - full version + if summary: + cleaned_summary = summary.replace("", "").replace("", "").strip() + f.write(f"{cleaned_summary}\n") + else: + f.write("*No content available*\n") + + generated_files.append(filename) + + return generated_files + + +async def main(): + """ + Process multiple modalities (images and documents) to generate memory categories. + + This example: + 1. Initializes MemoryService with OpenAI API + 2. Processes documents and images + 3. Extracts unified memory categories across modalities + 4. Outputs the categories to files + """ + print("Example 3: Multimodal Memory Processing") + print("-" * 50) + + # Get OpenAI API key from environment + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + msg = "Please set OPENAI_API_KEY environment variable" + raise ValueError(msg) + + # Define custom categories for multimodal content + multimodal_categories = [ + {"name": "technical_documentation", "description": "Technical documentation, guides, and tutorials"}, + { + "name": "architecture_concepts", + "description": "System architecture, design patterns, and structural concepts", + }, + {"name": "best_practices", "description": "Best practices, recommendations, and guidelines"}, + {"name": "code_examples", "description": "Code snippets, examples, and implementation details"}, + {"name": "visual_diagrams", "description": "Visual concepts, diagrams, charts, and illustrations from images"}, + ] + + # Initialize service with OpenAI using llm_profiles + # The "default" profile is required and used as the primary LLM configuration + service = MemoryService( + llm_profiles={ + "default": { + "api_key": api_key, + "chat_model": "gpt-4o-mini", + }, + }, + memorize_config={"memory_categories": multimodal_categories}, + ) + + # Resources to process (file_path, modality) + resources = [ + ("examples/resources/docs/doc1.txt", "document"), + ("examples/resources/docs/doc2.txt", "document"), + ("examples/resources/images/image1.png", "image"), + ] + + # Process each resource + print("\nProcessing resources...") + total_items = 0 + categories = [] + for resource_file, modality in resources: + if not os.path.exists(resource_file): + continue + + try: + result = await service.memorize(resource_url=resource_file, modality=modality) + total_items += len(result.get("items", [])) + # Categories are returned in the result and updated after each memorize call + categories = result.get("categories", []) + except Exception as e: + print(f"Error: {e}") + + # Write to output files + output_dir = "examples/output/multimodal_example" + os.makedirs(output_dir, exist_ok=True) + + # 1. Generate individual Markdown files for each category + await generate_memory_md(categories, output_dir) + + print(f"\n✓ Processed {len(resources)} files, extracted {total_items} items") + print(f"✓ Generated {len(categories)} categories") + print(f"✓ Output: {output_dir}/") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/example_4_openrouter_memory.py b/examples/example_4_openrouter_memory.py new file mode 100644 index 0000000..f5a8daa --- /dev/null +++ b/examples/example_4_openrouter_memory.py @@ -0,0 +1,112 @@ +""" +Example 4: Multiple Conversations -> Memory Category File (Using OpenRouter) + +This example demonstrates how to process multiple conversation files +and generate memory categories using OpenRouter as the LLM backend. + +Usage: + export OPENROUTER_API_KEY=your_api_key + python examples/example_4_openrouter_memory.py +""" + +import asyncio +import os +import sys + +from memu.app import MemoryService + +src_path = os.path.abspath("src") +sys.path.insert(0, src_path) + + +async def generate_memory_md(categories, output_dir): + """Generate concise markdown files for each memory category.""" + os.makedirs(output_dir, exist_ok=True) + generated_files = [] + + for cat in categories: + name = cat.get("name", "unknown") + summary = cat.get("summary", "") + + filename = f"{name}.md" + filepath = os.path.join(output_dir, filename) + + with open(filepath, "w", encoding="utf-8") as f: + if summary: + cleaned_summary = summary.replace("", "").replace("", "").strip() + f.write(f"{cleaned_summary}\n") + else: + f.write("*No content available*\n") + + generated_files.append(filename) + + return generated_files + + +async def main(): + """ + Process multiple conversation files and generate memory categories using OpenRouter. + + This example: + 1. Initializes MemoryService with OpenRouter API + 2. Processes conversation JSON files + 3. Extracts memory categories from conversations + 4. Outputs the categories to files + """ + print("Example 4: Conversation Memory Processing (OpenRouter)") + print("-" * 50) + + api_key = os.getenv("OPENROUTER_API_KEY") + if not api_key: + msg = "Please set OPENROUTER_API_KEY environment variable" + raise ValueError(msg) + + # Initialize service with OpenRouter + service = MemoryService( + llm_profiles={ + "default": { + "provider": "openrouter", + "client_backend": "httpx", + "base_url": "https://openrouter.ai", + "api_key": api_key, + "chat_model": "anthropic/claude-3.5-sonnet", # you can use any model from openrouter.ai + "embed_model": "openai/text-embedding-3-small", # you can use any model from openrouter.ai + }, + }, + ) + + conversation_files = [ + "examples/resources/conversations/conv1.json", + "examples/resources/conversations/conv2.json", + "examples/resources/conversations/conv3.json", + ] + + print("\nProcessing conversations...") + total_items = 0 + categories = [] + + for conv_file in conversation_files: + if not os.path.exists(conv_file): + print(f"Skipped: {conv_file} not found") + continue + + try: + print(f"Processing: {conv_file}") + result = await service.memorize(resource_url=conv_file, modality="conversation") + total_items += len(result.get("items", [])) + categories = result.get("categories", []) + except Exception as e: + print(f"Error processing {conv_file}: {e}") + + output_dir = "examples/output/openrouter_example" + os.makedirs(output_dir, exist_ok=True) + + await generate_memory_md(categories, output_dir) + + print(f"\nProcessed {len(conversation_files)} files, extracted {total_items} items") + print(f"Generated {len(categories)} categories") + print(f"Output: {output_dir}/") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/example_5_with_lazyllm_client.py b/examples/example_5_with_lazyllm_client.py new file mode 100644 index 0000000..3b30029 --- /dev/null +++ b/examples/example_5_with_lazyllm_client.py @@ -0,0 +1,250 @@ +""" +Unified Example: LazyLLM Integration Demo +========================================= + +This example merges functionalities from: +1. Example 1: Conversation Memory Processing +2. Example 2: Skill Extraction +3. Example 3: Multimodal Processing + +It demonstrates how to use the LazyLLM backend for: +- Processing conversation history +- Extracting technical skills from logs +- Handling multimodal content (images + text) +- defaut source and model are from qwen + +Usage: + export MEMU_QWEN_API_KEY=your_api_key + python examples/example_5_with_lazyllm_client.py +""" + +import asyncio +import os +import sys +from pathlib import Path + +# Add src to sys.path FIRST before importing memu +project_root = Path(__file__).parent.parent +src_path = str(project_root / "src") +if src_path not in sys.path: + sys.path.insert(0, src_path) + +from memu.app import MemoryService + +# ========================================== +# PART 1: Conversation Memory Processing +# ========================================== + + +async def run_conversation_memory_demo(service): + print("\n" + "=" * 60) + print("PART 1: Conversation Memory Processing") + print("=" * 60) + + conversation_files = [ + "examples/resources/conversations/conv1.json", + "examples/resources/conversations/conv2.json", + "examples/resources/conversations/conv3.json", + ] + + total_items = 0 + categories = [] + + for conv_file in conversation_files: + if not os.path.exists(conv_file): + print(f"⚠ File not found: {conv_file}") + continue + + try: + print(f" Processing: {conv_file}") + result = await service.memorize(resource_url=conv_file, modality="conversation") + total_items += len(result.get("items", [])) + categories = result.get("categories", []) + print(f" ✓ Extracted {len(result.get('items', []))} items") + except Exception as e: + print(f" ✗ Error processing {conv_file}: {e}") + + # Output generation + output_dir = "examples/output/lazyllm_example/conversation" + os.makedirs(output_dir, exist_ok=True) + await generate_markdown_output(categories, output_dir) + print(f"✓ Conversation processing complete. Output: {output_dir}") + + +# ========================================== +# PART 2: Skill Extraction +# ========================================== + + +async def run_skill_extraction_demo(service): + print("\n" + "=" * 60) + print("PART 2: Skill Extraction from Logs") + print("=" * 60) + + # Configure prompt for skill extraction + skill_prompt = """ + You are analyzing an agent execution log. Extract the key actions taken, their outcomes, and lessons learned. + + Output MUST be valid XML wrapped in tags. + Format: + + + + [Action] Description... + [Lesson] Key lesson... + + + Category Name + + + + + Text: {resource} + """ + + # Update service config for skill extraction + service.memorize_config.memory_types = ["skill"] + service.memorize_config.memory_type_prompts = {"skill": skill_prompt} + + logs = ["examples/resources/logs/log1.txt", "examples/resources/logs/log2.txt", "examples/resources/logs/log3.txt"] + + all_skills = [] + for log_file in logs: + if not os.path.exists(log_file): + continue + + print(f" Processing log: {log_file}") + try: + result = await service.memorize(resource_url=log_file, modality="document") + for item in result.get("items", []): + if item.get("memory_type") == "skill": + all_skills.append(item.get("summary", "")) + print(f" ✓ Extracted {len(result.get('items', []))} skills") + except Exception as e: + print(f" ✗ Error: {e}") + + # Generate summary guide + if all_skills: + output_file = "examples/output/lazyllm_example/skills/skill_guide.md" + await generate_skill_guide(all_skills, service, output_file) + print(f"✓ Skill guide generated: {output_file}") + + +# ========================================== +# PART 3: Multimodal Memory +# ========================================== + + +async def run_multimodal_demo(service): + print("\n" + "=" * 60) + print("PART 3: Multimodal Memory Processing") + print("=" * 60) + + # Configure for knowledge extraction + xml_prompt = """ + Analyze content and extract key information. + Output MUST be valid XML wrapped in tags. + Format: + + + Extracted content... + category_name + + + + Content: {resource} + """ + + service.memorize_config.memory_types = ["knowledge"] + service.memorize_config.memory_type_prompts = {"knowledge": xml_prompt} + + resources = [ + ("examples/resources/docs/doc1.txt", "document"), + ("examples/resources/images/image1.png", "image"), + ] + + categories = [] + for res_file, modality in resources: + if not os.path.exists(res_file): + continue + + print(f" Processing {modality}: {res_file}") + try: + result = await service.memorize(resource_url=res_file, modality=modality) + categories = result.get("categories", []) + print(f" ✓ Extracted {len(result.get('items', []))} items") + except Exception as e: + print(f" ✗ Error: {e}") + + output_dir = "examples/output/lazyllm_example/multimodal" + os.makedirs(output_dir, exist_ok=True) + await generate_markdown_output(categories, output_dir) + print(f"✓ Multimodal processing complete. Output: {output_dir}") + + +# ========================================== +# Helpers +# ========================================== + + +async def generate_markdown_output(categories, output_dir): + for cat in categories: + name = cat.get("name", "unknown") + summary = cat.get("summary", "") + if not summary: + continue + + with open(os.path.join(output_dir, f"{name}.md"), "w", encoding="utf-8") as f: + f.write(f"# {name.replace('_', ' ').title()}\n\n") + cleaned = summary.replace("", "").replace("", "").strip() + f.write(cleaned) + + +async def generate_skill_guide(skills, service, output_file): + os.makedirs(os.path.dirname(output_file), exist_ok=True) + skills_text = "\n\n".join(skills) + prompt = f"Summarize these skills into a guide:\n\n{skills_text}" + + # Use LazyLLM via service + summary = await service.llm_client.chat(text=prompt) + + with open(output_file, "w", encoding="utf-8") as f: + f.write(summary) + + +# ========================================== +# Main Entry +# ========================================== + + +async def main(): + print("Unified LazyLLM Example") + print("=" * 60) + # 1. Initialize Shared Service + service = MemoryService( + llm_profiles={ + "default": { + "client_backend": "lazyllm_backend", + "chat_model": "qwen3-max", + "embed_model": "text-embedding-v3", + "lazyllm_source": { + "source": "qwen", + "llm_source": "qwen", + "vlm_source": "qwen", + "embed_source": "qwen", + "stt_source": "qwen", + "vlm_model": "qwen-vl-plus", + "stt_model": "qwen-audio-turbo", + }, + }, + } + ) + + # 2. Run Demos + await run_conversation_memory_demo(service) + # await run_skill_extraction_demo(service) + # await run_multimodal_demo(service) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/example_6_preprocess_modalities.py b/examples/example_6_preprocess_modalities.py new file mode 100644 index 0000000..53748d4 --- /dev/null +++ b/examples/example_6_preprocess_modalities.py @@ -0,0 +1,140 @@ +""" +Example 6: Multimodal Preprocessing -> Extracted Text + Caption + +This example exercises memU's preprocessing stage for every supported modality +(conversation, document, image, audio, video) and prints what each preprocessor +extracts. It is a quick way to verify that multimodal extraction works end to end +against a real provider: + +- conversation / document : chat LLM (text understanding) +- image / video : VLM vision (video uses a sampled mid-frame) +- audio : speech-to-text transcription + chat LLM cleanup + +Requirements: +- An OpenAI API key with access to a chat model, a vision model, and the + transcription model (``gpt-4o-mini-transcribe``). +- ``ffmpeg``/``ffprobe`` on PATH for the video frame extraction. +- Optional document extras for PDF/Office input: ``pip install 'memu-py[document]'``. + +Usage: + export OPENAI_API_KEY=your_api_key + # Optional model overrides (defaults to the library defaults): + # export MEMU_CHAT_MODEL=gpt-4o-mini + # export MEMU_VLM_MODEL=gpt-4o-mini + python examples/example_6_preprocess_modalities.py +""" + +from __future__ import annotations + +import asyncio +import os +import sys + +from memu.app import MemoryService + +# Allow running from a source checkout without installing the package. +sys.path.insert(0, os.path.abspath("src")) + +RESOURCES = "examples/resources" + +# (modality, file). Vision modalities are routed to the VLM client automatically. +CASES: list[tuple[str, str]] = [ + ("conversation", f"{RESOURCES}/conversations/conv1.json"), + ("document", f"{RESOURCES}/docs/doc1.txt"), + ("document", f"{RESOURCES}/docs/doc_sample.pdf"), + ("image", f"{RESOURCES}/images/image1.png"), + ("audio", f"{RESOURCES}/audio/audio_intro.mp3"), + ("video", f"{RESOURCES}/video/video_test.mp4"), +] + +VISION_MODALITIES = {"image", "video"} + + +def build_service() -> MemoryService: + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + msg = "Please set OPENAI_API_KEY environment variable" + raise ValueError(msg) + + default_profile: dict[str, str] = {"api_key": api_key} + chat_model = os.getenv("MEMU_CHAT_MODEL") + if chat_model: + default_profile["chat_model"] = chat_model + + service = MemoryService(llm_profiles={"default": default_profile}) + + # VLM profiles are derived from the LLM profile; allow an explicit override of + # the vision model (handy when a default vision model is unavailable). + vlm_model = os.getenv("MEMU_VLM_MODEL") + if vlm_model: + for cfg in service.vlm_profiles.values(): + cfg.vlm_model = vlm_model + + return service + + +def _preview(value: str | None, limit: int = 500) -> str: + if not value: + return "" + value = value.strip() + return value if len(value) <= limit else value[:limit] + f"... [+{len(value) - limit} chars]" + + +async def preprocess_one(service: MemoryService, modality: str, path: str) -> bool: + """Run the preprocessing stage for a single resource and print the result.""" + print("\n" + "=" * 72) + print(f"MODALITY: {modality} ({os.path.basename(path)})") + print("-" * 72) + + if not os.path.exists(path): + print(f" SKIP: missing sample file {path}") + return False + + # Ingest (copies the file into the resources dir and extracts inline text for + # text modalities), then run the modality-specific preprocessor. Vision + # modalities use the VLM client; everything else uses the chat LLM client. + local_path, raw_text = await service.fs.fetch(path, modality) + if modality in VISION_MODALITIES: + client = service._get_vlm_client(service.memorize_config.vlm_profile) + else: + client = service._get_llm_client("default") + + segments = await service._preprocess_resource_url( + local_path=local_path, + text=raw_text, + modality=modality, + llm_client=client, + ) + + extracted = False + for i, seg in enumerate(segments): + print(f" [segment {i}] caption: {_preview(seg.get('caption'), 200)}") + print(f" [segment {i}] text : {_preview(seg.get('text'))}") + if seg.get("text"): + extracted = True + return extracted + + +async def main() -> None: + print("Example 6: Multimodal Preprocessing") + print("-" * 50) + + service = build_service() + results: list[tuple[str, str, bool]] = [] + for modality, path in CASES: + try: + ok = await preprocess_one(service, modality, path) + except Exception as e: + print(f" ERROR: {e}") + ok = False + results.append((modality, os.path.basename(path), ok)) + + print("\n" + "#" * 72) + print("SUMMARY") + print("#" * 72) + for modality, name, ok in results: + print(f" [{'OK ' if ok else 'FAIL'}] {modality:<12} {name}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/getting_started_robust.py b/examples/getting_started_robust.py new file mode 100644 index 0000000..883af99 --- /dev/null +++ b/examples/getting_started_robust.py @@ -0,0 +1,107 @@ +""" +Getting Started with MemU: A Robust Example. + +This script demonstrates the core lifecycle of MemU: +1. **Initialization**: Setting up the client with secure API key handling. +2. **Memory Injection**: Adding a specific memory with metadata. +3. **Retrieval**: Searching for that memory using natural language. +4. **Error Handling**: Catching common configuration issues. + +Usage: + export OPENAI_API_KEY=your_api_key_here + python examples/getting_started_robust.py +""" + +import asyncio +import logging +import os +import sys + +# Ensure src is in the path for local usage if custom installing +sys.path.insert(0, os.path.abspath("src")) + +from memu.app import MemoryService + +# Configure logging to show info but suppress noisy libraries +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logging.getLogger("httpx").setLevel(logging.WARNING) + + +async def main() -> None: + """Run the MemU lifecycle demonstration.""" + print(">>> MemU Getting Started Example") + print("-" * 30) + + # 1. API Key Handling + # MemU relies on an LLM backend (defaulting to OpenAI). + # We ensure the API key is present before proceeding. + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + print("[!] Error: OPENAI_API_KEY environment variable is not set.") + print("Please export it: export OPENAI_API_KEY=sk-...") + return + + try: + # 2. Initialization + # We initialize the MemoryService with: + # - llm_profiles: Configuration for the LLM (model, api_key). + # - memorize_config: Pre-defining a memory category ensures we can organize memories efficiently. + print("[*] Initializing MemoryService with model: gpt-4o-mini...") + service = MemoryService( + llm_profiles={ + "default": { + "api_key": api_key, + "chat_model": "gpt-4o-mini", + }, + }, + memorize_config={ + "memory_categories": [ + { + "name": "User Facts", + "description": "General and specific facts known about the user preference and identity.", + } + ] + }, + ) + print("[OK] Service initialized successfully.\n") + + # 3. Memory Injection + # We manually inject a memory into the system. + # This is useful for bootstrapping a user profile or adding explicit knowledge. + print("[*] Injecting memory...") + memory_content = "The user is a senior Python architect who loves clean code and type hints." + + # We use 'create_memory_item' to insert a single memory record. + # memory_type='profile' indicates this is an attribute of the user. + result = await service.create_memory_item( + memory_type="profile", + memory_content=memory_content, + memory_categories=["User Facts"], + ) + print(f"[OK] Memory created! ID: {result.get('memory_item', {}).get('id')}\n") + + # 4. Retrieval + # Now we query the system naturally to see if it recalls the information. + query_text = "What kind of code does the user like?" + print(f"[*] Querying: '{query_text}'") + + search_results = await service.retrieve(queries=[{"role": "user", "content": query_text}]) + + # 5. Display Results + items = search_results.get("items", []) + if items: + print(f"[OK] Found {len(items)} relevant memory item(s):") + for idx, item in enumerate(items, 1): + print(f" {idx}. {item.get('summary')} (Type: {item.get('memory_type')})") + else: + print("[!] No relevant memories found.") + + except Exception as e: + print(f"\n[!] An error occurred during execution: {e}") + logging.exception("Detailed traceback:") + finally: + print("\n[=] Example execution finished.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/langgraph_demo.py b/examples/langgraph_demo.py new file mode 100644 index 0000000..4107452 --- /dev/null +++ b/examples/langgraph_demo.py @@ -0,0 +1,76 @@ +"""Demo script for MemU LangGraph Integration.""" + +import asyncio +import logging +import os +import sys + +# Try imports and fail proactively if missing +try: + import langgraph # noqa: F401 + from langchain_core.tools import BaseTool + + from memu.app.service import MemoryService + from memu.integrations.langgraph import MemULangGraphTools +except ImportError: + print("Missing dependencies. Please run: uv sync --extra langgraph") + sys.exit(1) + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("langgraph_demo") + + +async def initialize_infrastructure() -> MemULangGraphTools: + """Initialize the MemoryService and the LangGraph adapter.""" + # Ensure OPENAI_API_KEY is present + if not os.environ.get("OPENAI_API_KEY"): + logger.warning("OPENAI_API_KEY not found in environment variables.") + + # In a real scenario, you might load config from file or env + service = MemoryService() + return MemULangGraphTools(service) + + +async def process_conversation(tools: list[BaseTool], user_id: str) -> None: + """Simulate a conversation where memory is saved.""" + save_tool = next(t for t in tools if t.name == "save_memory") + + logger.info("--- Simulating Save Memory ---") + inputs = { + "content": "The user prefers dark mode and likes Python programming.", + "user_id": user_id, + "metadata": {"source": "demo_script"}, + } + # Invoke the tool (async execution) + result = await save_tool.ainvoke(inputs) + logger.info("Save Result: %s", result) + + +async def process_retrieval(tools: list[BaseTool], user_id: str) -> None: + """Simulate retrieving memory.""" + search_tool = next(t for t in tools if t.name == "search_memory") + + logger.info("--- Simulating Search Memory ---") + inputs = {"query": "What are the user's preferences?", "user_id": user_id, "limit": 3} + result = await search_tool.ainvoke(inputs) + logger.info("Search Result:\n%s", result) + + +async def main() -> None: + """Main entry point.""" + logger.info("Starting LangGraph Demo...") + + adapter = await initialize_infrastructure() + tools = adapter.tools() + + user_id = "demo_user_123" + + await process_conversation(tools, user_id) + await process_retrieval(tools, user_id) + + logger.info("Demo completed.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/output/conversation_example/activities.md b/examples/output/conversation_example/activities.md new file mode 100644 index 0000000..68a8fb6 --- /dev/null +++ b/examples/output/conversation_example/activities.md @@ -0,0 +1,9 @@ +# activities +## Open Source Contributions +- The user enjoys contributing to open source projects, specifically a Python CLI tool used for automating deployment tasks. +## Running +- The user usually goes for a run every morning and is interested in running routes near downtown San Francisco. +## Dining Plans +- The user plans to try several vegetarian restaurants in San Francisco for their partner, who is vegetarian. +## Gym +- The user goes to the gym 3-4 times a week, usually after work around 7 PM. diff --git a/examples/output/conversation_example/experiences.md b/examples/output/conversation_example/experiences.md new file mode 100644 index 0000000..d209faa --- /dev/null +++ b/examples/output/conversation_example/experiences.md @@ -0,0 +1,5 @@ +# experiences +## User Experiences +- The user has been programming for about 5 years. +- The user is leading a big product launch next month, which is causing work-related stress that is affecting their sleep schedule. +- The user usually goes to bed around 11 PM but finds themselves awake thinking about work projects. diff --git a/examples/output/conversation_example/goals.md b/examples/output/conversation_example/goals.md new file mode 100644 index 0000000..eee4596 --- /dev/null +++ b/examples/output/conversation_example/goals.md @@ -0,0 +1,17 @@ +# goals +## Learning Objectives +- The user is interested in learning more about system design and scalability patterns. +- The user is learning about event-driven architecture and message queues, specifically using Apache Kafka for event streaming. + +## Product Development Goals +- The user wants to build their own SaaS product related to developer tools or automation. +- The user is currently in the research phase for their product idea, reading about OpenAPI specifications and exploring tools like Swagger and Postman. +- The user plans to start prototyping their SaaS product idea in the next few months. + +## Tool Development Ideas +- The user is considering building a tool for API testing and documentation that automatically generates test cases based on API specifications. +- The user is researching tools for developing the API tool. + +## Work-Life Balance Goals +- The user plans to talk to their team about response time expectations for emails, as they believe they are putting pressure on themselves. +- The user expresses a desire to establish better work-life boundaries and has been considering ways to limit work-related activities during the evening. diff --git a/examples/output/conversation_example/habits.md b/examples/output/conversation_example/habits.md new file mode 100644 index 0000000..d238387 --- /dev/null +++ b/examples/output/conversation_example/habits.md @@ -0,0 +1,13 @@ +# habits +## Eating Habits +- The user is trying to eat less meat. +## Exercise Routine +- The user usually goes for a run every morning. +- The user exercises regularly, going to the gym 3-4 times a week. +## Sleep Habits +- The user has been having trouble sleeping due to work stress. +- The user usually tries to go to bed around 11 PM but struggles to fall asleep due to stress from work projects. +- The user checks their phone before bed, which may impact their sleep quality. +- The user used to read before bed but stopped because they were always checking work emails. +## Caffeine Consumption +- The user usually drinks coffee throughout the day to stay alert, typically having their last coffee around 3-4 PM. diff --git a/examples/output/conversation_example/knowledge.md b/examples/output/conversation_example/knowledge.md new file mode 100644 index 0000000..15abeba --- /dev/null +++ b/examples/output/conversation_example/knowledge.md @@ -0,0 +1,29 @@ +# knowledge +## Reading Interests +- The user has been reading about OpenAPI specifications and exploring tools like Swagger and Postman. +## Technology Stack +- Alex's technology stack includes Django and FastAPI for Python services, Kubernetes for orchestration, Redis for caching, and Apache Kafka for event streaming. +## Interests +- Alex has an interest in system design and scalability patterns and is learning about event-driven architecture and message queues. +- For food in San Francisco, check out the Ferry Building Marketplace for gourmet options and local produce. +- Golden Gate Park, Lands End, and Muir Woods are recommended for nature activities in San Francisco. +- A self-guided tour of tech headquarters in Silicon Valley and the Computer History Museum are good tech-related activities in San Francisco. +- The de Young Museum and SFMOMA are suitable for photography and museum interests in San Francisco. +## Monitoring and Observability +- For monitoring and observability, Alex works with Prometheus and Grafana. +## Sleep and Wellness +- Work stress can negatively affect sleep quality, especially when one is in a leadership role. +- Using a phone before bed can disrupt sleep due to blue light, which suppresses melatonin production. +- Regular exercise can improve sleep, but working out close to bedtime may be stimulating. +- Caffeine consumption, particularly in the afternoon, can adversely affect sleep quality. +- Establishing a wind-down routine, such as reading or gentle stretching before bed, can help improve sleep. +- Keeping a phone out of the bedroom can create a boundary between work and sleep, improving sleep quality. +- Setting specific 'no work' hours can help transition from work mode to rest mode, reducing stress. +## Activities and Recommendations +- The best time to visit Muir Woods is early in the morning around 8 AM to avoid crowds and enjoy peacefulness. +- Parking reservations at Muir Woods are required and should be booked in advance. +- Recommended vegetarian restaurants in San Francisco include Greens Restaurant, Nourish Cafe, Shizen, Gracias Madre, and Ike's Love & Sandwiches. +- The Embarcadero is a great flat waterfront path for morning runs in San Francisco. +- Crissy Field offers stunning views of the Golden Gate Bridge and is another good running spot. +- Running to Coit Tower from North Beach provides a more challenging route with hills. +- San Francisco weather can be changeable, particularly across different neighborhoods, so packing layers is advisable. diff --git a/examples/output/conversation_example/opinions.md b/examples/output/conversation_example/opinions.md new file mode 100644 index 0000000..e9f7c82 --- /dev/null +++ b/examples/output/conversation_example/opinions.md @@ -0,0 +1 @@ +*No content available* diff --git a/examples/output/conversation_example/personal_info.md b/examples/output/conversation_example/personal_info.md new file mode 100644 index 0000000..273cfd1 --- /dev/null +++ b/examples/output/conversation_example/personal_info.md @@ -0,0 +1,3 @@ +# personal_info +## Basic Information +- The user is named Alex and works as a software engineer at TechCorp. diff --git a/examples/output/conversation_example/preferences.md b/examples/output/conversation_example/preferences.md new file mode 100644 index 0000000..52f34bb --- /dev/null +++ b/examples/output/conversation_example/preferences.md @@ -0,0 +1,6 @@ +# preferences +## Interests +- The user loves food and nature. +- The user enjoys exploring tech companies due to their background in software development. +- The user is interested in vegetarian options as their partner is vegetarian and the user is trying to eat less meat. +- The user likes reading and used to read before bed. diff --git a/examples/output/conversation_example/relationships.md b/examples/output/conversation_example/relationships.md new file mode 100644 index 0000000..579fa47 --- /dev/null +++ b/examples/output/conversation_example/relationships.md @@ -0,0 +1,3 @@ +# relationships +## partner +- The user's partner enjoys photography and museums. diff --git a/examples/output/conversation_example/work_life.md b/examples/output/conversation_example/work_life.md new file mode 100644 index 0000000..9ba3e90 --- /dev/null +++ b/examples/output/conversation_example/work_life.md @@ -0,0 +1,12 @@ +# work_life +## Professional Background +- The user is a software engineer at TechCorp. +- Alex is a software engineer with approximately 5 years of programming experience. +- The user works in software development. +## Current Responsibilities +- The user primarily works on backend systems using Python and Go for an e-commerce platform. +- At TechCorp, Alex's team is building a distributed microservices architecture for their e-commerce platform. +- The user uses Django and FastAPI for Python services and is migrating to Go for better performance. +- The user works with Kubernetes for orchestration and Redis for caching. +- The user is responsible for monitoring and observability, using Prometheus and Grafana. +- The user is leading a big product launch next month. diff --git a/examples/proactive/memory/config.py b/examples/proactive/memory/config.py new file mode 100644 index 0000000..622d5e1 --- /dev/null +++ b/examples/proactive/memory/config.py @@ -0,0 +1,66 @@ +memorize_config = { + "memory_types": [ + "record", + ], + "memory_type_prompts": { + "record": { + "objective": { + "ordinal": 10, + "prompt": "# Task Objective\nYou will be given a conversation between a user and an coding agent. Your goal is to extract detailed records for what are planed to do, and what have been done.", + }, + "workflow": { + "ordinal": 20, + "prompt": "# Workflow\nRead through the conversation and extract records. You should expecially focus on:\n- What the user ask the agent to do\n- What plan does the agent suggest\n- What the agent has done", + }, + "rules": { + "ordinal": -1, + "prompt": None, + }, + "examples": { + "ordinal": 60, + "prompt": "# Example\n## Output\n\n \n The user ask the agent to generate a code example for fastapi\n \n todo\n \n \n \n The agent suggest to use the code example from the document\n \n todo\n \n \n \n The agent ask the user to specify the response type\n \n todo\n \n \n", + }, + } + }, + "memory_categories": [ + { + "name": "todo", + "description": "This file traces the latest status of the task. All records should be included in this file.", + "target_length": None, + "custom_prompt": { + "objective": { + "ordinal": 10, + "prompt": "# Task Objective\nYou are a specialist in task management. You should update the markdown file to reflect the latest status of the task.", + }, + "workflow": { + "ordinal": 20, + "prompt": "# Workflow\nRead through the existing markdown file and the new records. Then update the markdown file to reflect:\n- What existing tasks are completed\n- What new tasks are added\n- What tasks are still in progress", + }, + "rules": { + "ordinal": 30, + "prompt": "# Rules\nFor each action-like record, explictly mark it as [Done] or [Todo].", + }, + "examples": { + "ordinal": 50, + "prompt": "# Example\n## Output\n```markdown\n# Task\n## Task Objective\nThe user ask the agent to generate a code example for fastapi\n## Breakdown\n- [Done] The agent suggest to use the code example from the document\n- [Todo] The agent ask the user to specify the response type\n```", + }, + }, + } + ], +} + +retrieve_config = { + "method": "rag", + "route_intention": False, + "sufficiency_check": False, + "category": { + "enabled": False, + }, + "item": { + "enabled": True, + "top_k": 10, + }, + "resource": { + "enabled": False, + }, +} diff --git a/examples/proactive/memory/local/__init__.py b/examples/proactive/memory/local/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/proactive/memory/local/common.py b/examples/proactive/memory/local/common.py new file mode 100644 index 0000000..8f6394c --- /dev/null +++ b/examples/proactive/memory/local/common.py @@ -0,0 +1,31 @@ +import os + +from memu.app import MemoryService + +from ..config import memorize_config, retrieve_config + +USER_ID = "claude_user" +SHARED_MEMORY_SERVICE = None + + +def get_memory_service() -> MemoryService: + global SHARED_MEMORY_SERVICE + if SHARED_MEMORY_SERVICE is not None: + return SHARED_MEMORY_SERVICE + + api_key = os.getenv("OPENAI_API_KEY") + if not api_key: + msg = "Please set OPENAI_API_KEY environment variable" + raise ValueError(msg) + + SHARED_MEMORY_SERVICE = MemoryService( + llm_profiles={ + "default": { + "api_key": api_key, + "chat_model": "gpt-4o-mini", + }, + }, + memorize_config=memorize_config, + retrieve_config=retrieve_config, + ) + return SHARED_MEMORY_SERVICE diff --git a/examples/proactive/memory/local/memorize.py b/examples/proactive/memory/local/memorize.py new file mode 100644 index 0000000..796a44b --- /dev/null +++ b/examples/proactive/memory/local/memorize.py @@ -0,0 +1,38 @@ +import json +from collections.abc import Awaitable +from pathlib import Path +from typing import Any + +import pendulum + +from .common import get_memory_service + +USER_ID = "claude_user" + + +def dump_conversation_resource( + conversation_messages: list[dict[str, Any]], +) -> str: + resource_data = { + "content": [ + { + "role": message.get("role", "system"), + "content": {"text": message.get("content", "")}, + "created_at": message.get("timestamp", pendulum.now().isoformat()), + } + for message in conversation_messages + ] + } + time_string = pendulum.now().format("YYYYMMDD_HHmmss") + resource_url = Path(__file__).parent / "data" / f"conv_{time_string}.json" + resource_url.parent.mkdir(parents=True, exist_ok=True) + with open(resource_url, "w") as f: + json.dump(resource_data, f, indent=4, ensure_ascii=False) + return resource_url.as_posix() + + +def memorize(conversation_messages: list[dict[str, Any]]) -> Awaitable[dict[str, Any]]: + memory_service = get_memory_service() + + resource_url = dump_conversation_resource(conversation_messages) + return memory_service.memorize(resource_url=resource_url, modality="conversation", user={"user_id": USER_ID}) diff --git a/examples/proactive/memory/local/tools.py b/examples/proactive/memory/local/tools.py new file mode 100644 index 0000000..7062c2c --- /dev/null +++ b/examples/proactive/memory/local/tools.py @@ -0,0 +1,42 @@ +from typing import Any + +from claude_agent_sdk import create_sdk_mcp_server, tool + +from .common import get_memory_service + +USER_ID = "claude_user" + + +@tool("memu_memory", "Retrieve memory based on a query", {"query": str}) +async def get_memory(args: dict[str, Any]) -> dict[str, Any]: + """Retrieve memory from the memory API based on the provided query.""" + query = {"role": "user", "content": args["query"]} + + memory_service = get_memory_service() + + result = await memory_service.retrieve(query, where={"user_id": USER_ID}) + + return {"content": [{"type": "text", "text": str(result)}]} + + +async def _get_todos() -> str: + memory_service = get_memory_service() + + result = await memory_service.list_memory_categories(where={"user_id": USER_ID}) + + categories = result["categories"] + todos = "" + for category in categories: + if category["name"] == "todo": + todos = category["summary"] + return todos + + +@tool("memu_todos", "Retrieve todos for the user", {}) +async def get_todos() -> dict[str, Any]: + """Retrieve todos from the memory API.""" + todos = await _get_todos() + return {"content": [{"type": "text", "text": str(todos)}]} + + +memu_server = create_sdk_mcp_server(name="memu", version="1.0.0", tools=[get_memory, get_todos]) diff --git a/examples/proactive/memory/platform/__init__.py b/examples/proactive/memory/platform/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/proactive/memory/platform/memorize.py b/examples/proactive/memory/platform/memorize.py new file mode 100644 index 0000000..9fc1722 --- /dev/null +++ b/examples/proactive/memory/platform/memorize.py @@ -0,0 +1,31 @@ +from typing import Any + +import aiohttp + +from ..config import memorize_config + +BASE_URL = "https://api.memu.so" +API_KEY = "your memu api key" +USER_ID = "claude_user" +AGENT_ID = "claude_agent" + + +async def memorize(conversation_messages: list[dict[str, Any]]) -> str | None: + payload = { + "conversation": conversation_messages, + "user_id": USER_ID, + "agent_id": AGENT_ID, + "override_config": memorize_config, + } + + async with ( + aiohttp.ClientSession() as session, + session.post( + f"{BASE_URL}/api/v3/memory/memorize", + headers={"Authorization": f"Bearer {API_KEY}"}, + json=payload, + ) as response, + ): + result = await response.json() + task_id = result["task_id"] + return task_id diff --git a/examples/proactive/memory/platform/tools.py b/examples/proactive/memory/platform/tools.py new file mode 100644 index 0000000..5ed859d --- /dev/null +++ b/examples/proactive/memory/platform/tools.py @@ -0,0 +1,52 @@ +from typing import Any + +import aiohttp +from claude_agent_sdk import create_sdk_mcp_server, tool + +BASE_URL = "https://api.memu.so" +API_KEY = "your memu api key" +USER_ID = "claude_user" +AGENT_ID = "claude_agent" + + +@tool("memu_memory", "Retrieve memory based on a query", {"query": str}) +async def get_memory(args: dict[str, Any]) -> dict[str, Any]: + """Retrieve memory from the memory API based on the provided query.""" + query = args["query"] + url = f"{BASE_URL}/api/v3/memory/retrieve" + headers = {"Authorization": f"Bearer {API_KEY}"} + data = {"user_id": USER_ID, "agent_id": AGENT_ID, "query": query} + + async with aiohttp.ClientSession() as session, session.post(url, headers=headers, json=data) as response: + result = await response.json() + + return {"content": [{"type": "text", "text": str(result)}]} + + +async def _get_todos() -> str: + url = f"{BASE_URL}/api/v3/memory/categories" + headers = {"Authorization": f"Bearer {API_KEY}"} + data = { + "user_id": USER_ID, + "agent_id": AGENT_ID, + } + async with aiohttp.ClientSession() as session, session.post(url, headers=headers, json=data) as response: + result = await response.json() + + categories = result["categories"] + todos = "" + for category in categories: + if category["name"] == "todo": + todos = category["summary"] + return todos + + +@tool("memu_todos", "Retrieve todos for the user", {}) +async def get_todos() -> dict[str, Any]: + """Retrieve todos from the memory API.""" + todos = await _get_todos() + return {"content": [{"type": "text", "text": str(todos)}]} + + +# Create the MCP server with the tool +memu_server = create_sdk_mcp_server(name="memu", version="1.0.0", tools=[get_memory, get_todos]) diff --git a/examples/proactive/proactive.py b/examples/proactive/proactive.py new file mode 100644 index 0000000..7ffe10c --- /dev/null +++ b/examples/proactive/proactive.py @@ -0,0 +1,198 @@ +import asyncio + +from claude_agent_sdk import ( + AssistantMessage, + ClaudeAgentOptions, + ClaudeSDKClient, + ResultMessage, + TextBlock, +) +from memory.local.memorize import memorize +from memory.local.tools import _get_todos, memu_server + +# Set your Anthropic API key here if it's not set in the environment variables +# os.environ["ANTHROPIC_API_KEY"] = "" + +N_MESSAGES_MEMORIZE = 2 +RUNNING_MEMORIZATION: asyncio.Task | None = None + + +async def trigger_memorize(messages: list[dict[str, any]]) -> bool: + """Create a background task to memorize conversation messages. + + Returns True if the task was successfully created and registered. + """ + global RUNNING_MEMORIZATION + try: + memorize_awaitable = memorize(messages) + RUNNING_MEMORIZATION = asyncio.create_task(memorize_awaitable) + except Exception as e: + print(f"\n[Memory] Memorization initialization failed: {e!r}") + return False + else: + print("\n[Memory] Memorization task submitted.") + return True + + +async def get_next_input(iteration: int) -> tuple[str | None, bool]: + """ + Get the next input for the conversation. + + Returns: + tuple of (input_text, should_break) + - input_text: The user input or todo-based input, None if should continue + - should_break: True if the loop should break + """ + if iteration == 0: + return await get_user_input() + + todos = await _get_todos() + + print(f">>> Todos:\n{todos}\n") + print("-" * 40) + + if todos and "[todo]" in todos.lower(): + return f"Please continue with the following todos:\n{todos}", False + + return await get_user_input() + + +async def get_user_input() -> tuple[str | None, bool]: + """ + Get input from the user. + + Returns: + tuple of (input_text, should_break) + """ + try: + user_input = input("\nYou: ").strip() + except EOFError: + return None, True + + if not user_input: + return None, False + + if user_input.lower() in ("quit", "exit"): + return None, True + + return user_input, False + + +async def process_response(client: ClaudeSDKClient) -> list[str]: + """Process the assistant response and return collected text parts.""" + assistant_text_parts: list[str] = [] + + async for message in client.receive_response(): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + print(f"Claude: {block.text}") + assistant_text_parts.append(block.text) + elif isinstance(message, ResultMessage): + print(f"Result: {message.result}") + + return assistant_text_parts + + +async def check_and_memorize(conversation_messages: list[dict[str, any]]) -> None: + """Check if memorization threshold is reached and trigger if needed. + + Skips triggering if a previous memorization task is still running. + """ + global RUNNING_MEMORIZATION + + if len(conversation_messages) < N_MESSAGES_MEMORIZE: + return + + # Check if there's a running memorization task + if RUNNING_MEMORIZATION is not None: + if not RUNNING_MEMORIZATION.done(): + print("\n[Info] Have running memorization, skipping...") + return + # Previous task completed, check for exceptions + try: + RUNNING_MEMORIZATION.result() + except Exception as e: + print(f"\n[Memory] Memorization failed: {e!r}") + RUNNING_MEMORIZATION = None + + print(f"\n[Info] Reached {N_MESSAGES_MEMORIZE} messages, triggering memorization...") + success = await trigger_memorize(conversation_messages.copy()) + if success: + conversation_messages.clear() + + +async def run_conversation_loop(client: ClaudeSDKClient) -> list[dict[str, any]]: + """Run the main conversation loop.""" + conversation_messages: list[dict[str, any]] = [] + iteration = 0 + + while True: + user_input, should_break = await get_next_input(iteration) + + if should_break: + break + if user_input is None: + continue + + conversation_messages.append({"role": "user", "content": user_input}) + await client.query(user_input) + + assistant_text_parts = await process_response(client) + + if assistant_text_parts: + conversation_messages.append({ + "role": "assistant", + "content": "\n".join(assistant_text_parts), + }) + + await check_and_memorize(conversation_messages) + iteration += 1 + + return conversation_messages + + +async def main(): + options = ClaudeAgentOptions( + mcp_servers={"memu": memu_server}, + allowed_tools=[ + # "mcp__memu__memu_memory", + "mcp__memu__memu_todos", + ], + ) + + print("Claude Autorun") + print("Type 'quit' or 'exit' to end the session.") + print("-" * 40) + + async with ClaudeSDKClient(options=options) as client: + remaining_messages = await run_conversation_loop(client) + + # Wait for any running memorization task to complete + global RUNNING_MEMORIZATION + if RUNNING_MEMORIZATION is not None and not RUNNING_MEMORIZATION.done(): + print("\n[Info] Waiting for running memorization task to complete...") + try: + await RUNNING_MEMORIZATION + print("\n[Memory] Running memorization completed successfully.") + except Exception as e: + print(f"\n[Memory] Running memorization failed: {e!r}") + RUNNING_MEMORIZATION = None + + # Memorize remaining messages and wait for completion + if remaining_messages: + print("\n[Info] Session ended, memorizing remaining messages...") + success = await trigger_memorize(remaining_messages.copy()) + if success and RUNNING_MEMORIZATION is not None: + print("\n[Info] Waiting for final memorization to complete...") + try: + await RUNNING_MEMORIZATION + print("\n[Memory] Final memorization completed successfully.") + except Exception as e: + print(f"\n[Memory] Final memorization failed: {e!r}") + + print("\nDone") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/resources/audio/audio_intro.mp3 b/examples/resources/audio/audio_intro.mp3 new file mode 100644 index 0000000..e0598ef Binary files /dev/null and b/examples/resources/audio/audio_intro.mp3 differ diff --git a/examples/resources/conversations/conv1.json b/examples/resources/conversations/conv1.json new file mode 100644 index 0000000..9454071 --- /dev/null +++ b/examples/resources/conversations/conv1.json @@ -0,0 +1,54 @@ +[ + { + "role": "user", + "content": "Hi! I'm Alex. I'm a software engineer working at TechCorp. I've been programming for about 5 years now." + }, + { + "role": "assistant", + "content": "Nice to meet you, Alex! It's great to have you here. What kind of software development do you primarily work on at TechCorp?" + }, + { + "role": "user", + "content": "I mainly work on backend systems using Python and Go. We're building a distributed microservices architecture for our e-commerce platform. I'm particularly interested in learning more about system design and scalability patterns." + }, + { + "role": "assistant", + "content": "That sounds like an exciting project! Distributed systems and microservices certainly present interesting challenges. Are you working with any specific frameworks or technologies for the microservices?" + }, + { + "role": "user", + "content": "Yes, we use Django and FastAPI for Python services, and we're migrating some critical services to Go for better performance. We use Kubernetes for orchestration and Redis for caching. I've been learning about event-driven architecture and message queues lately." + }, + { + "role": "assistant", + "content": "Excellent technology stack! Event-driven architecture is a great pattern for microservices. Are you using any specific message broker like RabbitMQ or Kafka?" + }, + { + "role": "user", + "content": "We're using Apache Kafka for our event streaming. I'm also responsible for monitoring and observability, so I work with Prometheus and Grafana. Outside of work, I like to contribute to open source projects. I recently started contributing to a Python CLI tool." + }, + { + "role": "assistant", + "content": "That's wonderful! Open source contributions are a great way to learn and give back to the community. What made you interested in contributing to that particular CLI tool?" + }, + { + "role": "user", + "content": "I actually use it in my daily workflow for automating deployment tasks. I found a few bugs and thought I could help improve it. I also want to eventually build my own SaaS product, maybe something related to developer tools or automation." + }, + { + "role": "assistant", + "content": "That's a great goal! Having a SaaS product idea is exciting. Do you have any specific problem you're looking to solve with your developer tools idea?" + }, + { + "role": "user", + "content": "I'm thinking about building a tool for API testing and documentation. Something that can automatically generate test cases based on API specifications and maintain up-to-date documentation. I notice this is a pain point in my current team." + }, + { + "role": "assistant", + "content": "That's a valuable problem to solve! Many teams struggle with keeping API documentation synchronized with their codebase. Have you done any market research or prototyping yet?" + }, + { + "role": "user", + "content": "Not yet, I'm still in the research phase. I've been reading about OpenAPI specifications and exploring tools like Swagger and Postman. I think there's room for a more automated and AI-powered solution. I'm planning to start prototyping in the next few months." + } +] diff --git a/examples/resources/conversations/conv2.json b/examples/resources/conversations/conv2.json new file mode 100644 index 0000000..1f0b082 --- /dev/null +++ b/examples/resources/conversations/conv2.json @@ -0,0 +1,67 @@ +[ + { + "role": "user", + "created_at": "2025-02-01T09:00:00Z", + "content": "I need help planning a weekend trip to San Francisco. Can you suggest some activities?" + }, + { + "role": "assistant", + "created_at": "2025-02-01T09:00:08Z", + "content": "I'd be happy to help you plan your San Francisco trip! What are your main interests? Are you into tech, food, nature, art, or something else?" + }, + { + "role": "user", + "created_at": "2025-02-01T09:00:25Z", + "content": "I love food and nature! I'm also interested in visiting some tech companies since I work in software development. My partner enjoys photography and museums." + }, + { + "role": "assistant", + "created_at": "2025-02-01T09:00:40Z", + "content": "Perfect! For food, you should definitely check out the Ferry Building Marketplace for gourmet options and local produce. For nature, I'd recommend Golden Gate Park, hiking at Lands End, or visiting Muir Woods for the redwoods. For tech, you could do a self-guided tour of tech headquarters in Silicon Valley or visit the Computer History Museum. For your partner, there's the de Young Museum in Golden Gate Park and SFMOMA for modern art." + }, + { + "role": "user", + "created_at": "2025-02-01T09:01:20Z", + "content": "Those sound great! I'm particularly excited about Muir Woods. I've never seen redwoods before. What's the best time to visit?" + }, + { + "role": "assistant", + "created_at": "2025-02-01T09:01:35Z", + "content": "Muir Woods is magical! I'd recommend going early in the morning, ideally right when it opens around 8 AM. This way you'll avoid the crowds and experience the peaceful atmosphere of the forest. The morning light filtering through the redwoods is also spectacular for photography. Make sure to book parking reservations in advance as they're required." + }, + { + "role": "user", + "created_at": "2025-02-01T09:02:10Z", + "content": "Good to know about the parking! Are there any good vegetarian restaurants you'd recommend? My partner is vegetarian and I'm trying to eat less meat." + }, + { + "role": "assistant", + "created_at": "2025-02-01T09:02:25Z", + "content": "San Francisco has amazing vegetarian options! I'd recommend Greens Restaurant for upscale vegetarian with a view, Nourish Cafe for healthy bowls, and Shizen for vegan sushi. If you're in the Mission District, try Gracias Madre for organic Mexican food. For something casual, Ike's Love & Sandwiches has great veggie options." + }, + { + "role": "user", + "created_at": "2025-02-01T09:03:05Z", + "content": "Perfect! I'll add these to my list. One more thing - I usually go for a run every morning. Are there good running routes near downtown?" + }, + { + "role": "assistant", + "created_at": "2025-02-01T09:03:20Z", + "content": "Absolutely! The Embarcadero is perfect for morning runs - it's a flat waterfront path with beautiful bay views. You can run from the Ferry Building towards Fisherman's Wharf, which is about 3 miles one way. Alternatively, Crissy Field offers stunning views of the Golden Gate Bridge. If you want something more challenging with hills, try running up to Coit Tower from North Beach." + }, + { + "role": "user", + "created_at": "2025-02-01T09:03:55Z", + "content": "The Embarcadero sounds perfect for my morning routine. Thanks for all the suggestions! I'm getting really excited about this trip now." + }, + { + "role": "assistant", + "created_at": "2025-02-01T09:04:05Z", + "content": "You're welcome! It sounds like you have a great mix of activities planned. Don't forget to bring layers - San Francisco weather can be quite changeable, especially if you're visiting different neighborhoods. Have a wonderful trip!" + }, + { + "role": "user", + "created_at": "2025-02-01T09:04:30Z", + "content": "Will do! I always forget how different the weather can be in different parts of the city. I'll pack accordingly." + } +] diff --git a/examples/resources/conversations/conv3.json b/examples/resources/conversations/conv3.json new file mode 100644 index 0000000..4b62998 --- /dev/null +++ b/examples/resources/conversations/conv3.json @@ -0,0 +1,67 @@ +[ + { + "role": "user", + "created_at": "2025-03-10T22:00:00Z", + "content": "I've been having trouble sleeping lately. I think work stress might be affecting my sleep schedule." + }, + { + "role": "assistant", + "created_at": "2025-03-10T22:00:12Z", + "content": "I'm sorry to hear you're having trouble sleeping. Work stress can definitely impact sleep quality. Can you tell me more about your current sleep routine and what's been going on at work?" + }, + { + "role": "user", + "created_at": "2025-03-10T22:00:45Z", + "content": "I usually try to go to bed around 11 PM, but I find myself lying awake thinking about work projects. I'm leading a big product launch next month and there's a lot of pressure. I also tend to check my phone before bed." + }, + { + "role": "assistant", + "created_at": "2025-03-10T22:01:05Z", + "content": "It's common for work pressure to interfere with sleep, especially when you're in a leadership role. The phone usage before bed could also be contributing to the problem - the blue light can suppress melatonin production. Have you tried any relaxation techniques or sleep hygiene practices?" + }, + { + "role": "user", + "created_at": "2025-03-10T22:01:40Z", + "content": "I tried meditation once but didn't stick with it. I do exercise regularly - I go to the gym 3-4 times a week, usually after work around 7 PM. I also drink coffee throughout the day to stay alert." + }, + { + "role": "assistant", + "created_at": "2025-03-10T22:01:55Z", + "content": "Exercise is great for sleep, but working out too close to bedtime can actually be stimulating. The caffeine consumption throughout the day could also be affecting your sleep, especially if you're having coffee in the afternoon. Could you try moving your workouts to the morning and cutting off caffeine after 2 PM?" + }, + { + "role": "user", + "created_at": "2025-03-10T22:02:25Z", + "content": "That makes sense. I usually have my last coffee around 3-4 PM. Maybe I should switch to decaf in the afternoon. Morning workouts might be tough though - I'm not really a morning person." + }, + { + "role": "assistant", + "created_at": "2025-03-10T22:02:40Z", + "content": "Starting with the caffeine cutoff is a good first step! As for morning workouts, you don't have to make a drastic change immediately. Even a 15-minute walk in the morning can help regulate your circadian rhythm. For the evening, you might want to establish a wind-down routine - maybe some light reading, gentle stretching, or breathing exercises about an hour before bed." + }, + { + "role": "user", + "created_at": "2025-03-10T22:03:20Z", + "content": "I like reading, actually. I used to read before bed but stopped because I was always checking work emails. Maybe I should keep my phone out of the bedroom entirely." + }, + { + "role": "assistant", + "created_at": "2025-03-10T22:03:35Z", + "content": "That's an excellent idea! Keeping your phone out of the bedroom can make a big difference. You could get a regular alarm clock if you use your phone for that. Creating a clear boundary between work and sleep time is important, especially with the product launch stress." + }, + { + "role": "user", + "created_at": "2025-03-10T22:04:10Z", + "content": "You're right. I think I need to be better about work-life boundaries in general. I often answer emails late at night, which probably doesn't help my stress levels." + }, + { + "role": "assistant", + "created_at": "2025-03-10T22:04:25Z", + "content": "Those late-night emails can definitely keep your mind in 'work mode' when it should be winding down. Have you considered setting specific 'no work' hours, maybe from 8 PM onwards? That way you give your brain time to transition from work mode to rest mode." + }, + { + "role": "user", + "created_at": "2025-03-10T22:04:55Z", + "content": "That sounds reasonable. I'll talk to my team about response time expectations. I don't think anyone expects instant replies at night anyway - I'm probably putting that pressure on myself." + } +] diff --git a/examples/resources/docs/doc1.txt b/examples/resources/docs/doc1.txt new file mode 100644 index 0000000..ff28017 --- /dev/null +++ b/examples/resources/docs/doc1.txt @@ -0,0 +1,330 @@ +MemU: Agentic Memory Framework - Technical Documentation + +Introduction +============ +MemU is a sophisticated agentic memory framework designed to provide AI agents and applications with human-like memory capabilities. Unlike traditional RAG (Retrieval-Augmented Generation) systems that simply store and retrieve information, MemU organizes, categorizes, and maintains memories in a structured, semantically meaningful way. + +Core Concepts +============= + +1. Memory Organization + MemU organizes information into several layers: + - Memory Items: Individual pieces of information extracted from inputs + - Memory Categories: Semantic groupings of related memories + - Memory Types: Classifications of memory content (profile, event, knowledge, behavior) + +2. Multi-Modal Support + The framework supports various input modalities: + - Text documents (PDF, TXT, DOC) + - Conversations (JSON chat logs) + - Images (PNG, JPG, with vision model integration) + - Audio (transcription and processing) + - Video (frame extraction and analysis) + +3. Intelligent Processing Pipeline + Each input goes through several processing stages: + a. Preprocessing: Content extraction and normalization + b. Summarization: Key information extraction + c. Embedding: Vector representation generation + d. Classification: Memory type identification + e. Categorization: Semantic category assignment + f. Storage: Persistent storage with metadata + +Architecture Components +======================= + +1. MemoryService (Core Service Layer) + The main entry point for all memory operations: + - memorize(): Process and store new information + - retrieve(): Query and fetch relevant memories + - update(): Modify existing memories + - delete(): Remove memories + + Configuration options: + - LLM provider settings (OpenAI, Azure, custom) + - Embedding model selection + - Memory type definitions + - Category templates + - Retrieval methods (RAG, LLM-based) + +2. Storage Layer + Multiple storage backends supported: + - SQLite (default, local development) + - PostgreSQL (production deployments) + - In-memory (testing and temporary storage) + + Data persistence: + - Memory items with metadata + - Category definitions and summaries + - Vector embeddings for similarity search + - Resource references and URLs + +3. Vector Search Engine + Semantic search capabilities powered by: + - Dense embeddings (OpenAI text-embedding-3-small/large) + - Similarity metrics (cosine similarity, dot product) + - Efficient indexing for fast retrieval + - Hybrid search combining semantic and keyword matching + +4. LLM Integration Layer + Flexible LLM backend support: + - OpenAI SDK client (primary) + - HTTP-based client (custom endpoints) + - Configurable model selection + - Prompt template system + - Streaming response support + +Memory Types +============ + +1. Profile Memory + Stores persistent information about entities: + - Personal attributes (name, age, occupation) + - Preferences and interests + - Relationships and connections + - Identity and characteristics + + Example: "Alex is a software engineer at TechCorp, specializing in backend development" + +2. Event Memory + Records discrete occurrences and activities: + - Temporal events with timestamps + - Actions and experiences + - Milestones and achievements + - Incidents and interactions + + Example: "Completed the deployment pipeline implementation on November 15, 2024" + +3. Knowledge Memory + Captures factual information and learnings: + - Facts and concepts + - Skills and capabilities + - Domain expertise + - Technical knowledge + + Example: "Proficient in Python, Go, Kubernetes, and microservices architecture" + +4. Behavior Memory + Tracks patterns and tendencies: + - Habits and routines + - Decision patterns + - Behavioral preferences + - Interaction styles + + Example: "Prefers morning workouts, typically exercises 3-4 times per week" + +Category Management +=================== + +Dynamic Categorization: +MemU automatically assigns memories to semantic categories based on content similarity. Categories are created and maintained dynamically as new memories are added. + +Default Categories: +- personal_info: Personal details and identity +- preferences: Likes, dislikes, and choices +- relationships: Connections with others +- activities: Hobbies and interests +- goals: Aspirations and objectives +- experiences: Past events and learnings +- knowledge: Facts and information +- opinions: Views and perspectives +- habits: Routines and patterns +- work_life: Professional information + +Custom Categories: +Users can define custom categories with: +- Name and description +- Embedding vector for semantic matching +- Assignment threshold for automatic categorization +- Summary generation for category overview + +Category Summaries: +MemU maintains auto-generated summaries for each category that: +- Provide overview of category contents +- Get updated as new memories are added +- Help with high-level information retrieval +- Support category-level search + +Retrieval Strategies +==================== + +1. RAG-Based Retrieval (Default) + Vector similarity search approach: + - Query embedding generation + - Similarity calculation with stored memories + - Top-K selection per category + - Ranking by relevance score + - Context window assembly + + Advantages: + - Fast and efficient + - Deterministic results + - Lower LLM costs + - Good for factual recall + +2. LLM-Based Retrieval + AI-powered search and ranking: + - Query understanding and expansion + - Semantic relevance judgment + - Context-aware ranking + - Multi-hop reasoning support + - Natural language result explanation + + Advantages: + - Better semantic understanding + - Handles complex queries + - Context-aware results + - Flexible interpretation + +Retrieval Pipeline: +1. Pre-retrieval decision (should we retrieve?) +2. Query rewriting (optimize for search) +3. Category ranking (which categories are relevant?) +4. Item retrieval (fetch top-K items) +5. Item ranking (rerank by relevance) +6. Resource retrieval (fetch original sources) +7. Result assembly (format for output) + +Best Practices +============== + +1. Memory Quality + - Provide detailed, contextual inputs + - Include timestamps for events + - Maintain consistent terminology + - Regular memory consolidation + - Remove outdated information + +2. Configuration Optimization + - Tune embedding models for your domain + - Adjust category assignment thresholds + - Customize memory type prompts + - Set appropriate top-K values + - Configure LLM parameters + +3. Performance Optimization + - Batch memory operations when possible + - Use appropriate storage backend + - Index frequently queried fields + - Cache embeddings when reusing + - Monitor memory growth + +4. Privacy and Security + - Implement access controls + - Encrypt sensitive memories + - Regular data audits + - Compliance with data regulations + - User consent for memory storage + +Use Cases +========= + +1. Personal AI Assistants + - Remember user preferences and context + - Maintain conversation history + - Learn from interactions + - Personalize responses + +2. Customer Support Systems + - Track customer history and issues + - Remember preferences and complaints + - Build customer profiles + - Improve service quality + +3. Educational Applications + - Track learning progress + - Remember concepts learned + - Adapt to learning style + - Provide personalized content + +4. Knowledge Management + - Organize organizational knowledge + - Track project information + - Build expertise databases + - Enable knowledge discovery + +5. Agent Workflows + - Maintain task context + - Remember tool usage patterns + - Learn from execution history + - Optimize decision making + +API Reference +============= + +Basic Usage Example: + +```python +from memu.app import MemoryService + +# Initialize service +service = MemoryService( + llm_config={ + "api_key": "your-api-key", + "chat_model": "gpt-4o-mini" + } +) + +# Store a memory +result = await service.memorize( + resource_url="conversation.json", + modality="conversation" +) + +# Retrieve memories +memories = await service.retrieve( + query="What programming languages does Alex know?", + top_k=5 +) + +# Access categories +categories = service.store.categories +``` + +Advanced Configuration: + +```python +# Custom memory types +memorize_config = { + "memory_types": ["profile", "knowledge", "custom"], + "memory_type_prompts": { + "custom": "Extract specific information: {resource}" + }, + "memory_categories": [ + {"name": "technical_skills", "description": "Programming and technical abilities"}, + {"name": "soft_skills", "description": "Communication and interpersonal skills"} + ] +} + +service = MemoryService( + llm_config=llm_config, + memorize_config=memorize_config +) +``` + +Roadmap +======= + +Upcoming Features: +- Long-term memory consolidation +- Federated memory systems +- Memory importance scoring +- Automatic memory pruning +- Cross-user memory sharing +- Memory versioning and history +- Enhanced temporal reasoning +- Graph-based memory relationships +- Memory export and import +- Advanced privacy controls + +Contributing +============ + +MemU is open source and welcomes contributions: +- Bug reports and feature requests +- Documentation improvements +- Code contributions +- Example applications +- Performance optimizations + +For more information, visit: https://github.com/mem-labs/memU diff --git a/examples/resources/docs/doc2.txt b/examples/resources/docs/doc2.txt new file mode 100644 index 0000000..a6f7700 --- /dev/null +++ b/examples/resources/docs/doc2.txt @@ -0,0 +1,433 @@ +Building Production-Ready AI Agents: A Comprehensive Guide + +Executive Summary +================= + +This document provides a comprehensive guide to building, deploying, and maintaining production-ready AI agents. It covers architecture patterns, best practices, common pitfalls, and real-world implementation strategies based on experiences from deploying agents at scale. + +What is an AI Agent? +==================== + +An AI agent is an autonomous system that can: +- Perceive its environment through sensors or data inputs +- Make decisions based on goals and constraints +- Take actions using available tools and APIs +- Learn and adapt from experience +- Operate with minimal human intervention + +Key characteristics that differentiate agents from simple LLM applications: +1. Goal-directed behavior (not just reactive) +2. Multi-step reasoning and planning +3. Tool use and external API integration +4. Memory and state management +5. Error handling and recovery +6. Continuous operation + +Agent Architecture Patterns +============================ + +1. ReAct (Reasoning + Acting) Pattern + The most common agent architecture: + - Thought: Reason about the current situation + - Action: Choose and execute a tool/action + - Observation: Observe the results + - Repeat until goal is achieved + + Advantages: + - Interpretable decision-making process + - Easy to debug with visible reasoning + - Works well with modern LLMs + + Challenges: + - Can be verbose and slow + - May get stuck in reasoning loops + - Token costs accumulate quickly + +2. Plan-and-Execute Pattern + Separates planning from execution: + - Generate high-level plan upfront + - Execute steps sequentially + - Replan if execution fails + + Advantages: + - More efficient for complex tasks + - Better resource allocation + - Clearer progress tracking + + Challenges: + - Less adaptable to changing conditions + - Planning failures cascade + - Requires good task decomposition + +3. Hierarchical Agent Systems + Multiple agents working together: + - Coordinator agent manages workflow + - Specialist agents handle specific domains + - Memory shared across agents + + Advantages: + - Scalable to complex domains + - Parallel execution possible + - Clear separation of concerns + + Challenges: + - Coordination overhead + - Complex error propagation + - Harder to debug + +4. Autonomous Agent Pattern + Continuous operation without explicit tasks: + - Monitor environment for triggers + - Self-generate tasks and goals + - Learn from outcomes + - Adjust behavior over time + + Advantages: + - Truly autonomous operation + - Proactive rather than reactive + - Continuous improvement + + Challenges: + - Harder to control and bound + - Safety and alignment concerns + - Resource management critical + +Essential Components +==================== + +1. Language Model Integration + Choosing the right LLM: + - GPT-4 family: Best reasoning, highest cost + - Claude: Strong safety, good reasoning + - Open source models: Lower cost, self-hosted + + Optimization strategies: + - Use smaller models for simple tasks + - Implement prompt caching + - Batch similar requests + - Fine-tune for specific domains + +2. Tool Registry and Execution + Managing agent capabilities: + - Dynamic tool discovery + - Type-safe tool schemas + - Sandboxed execution environment + - Rate limiting and quotas + - Error handling and retries + + Tool design principles: + - Single responsibility per tool + - Clear input/output contracts + - Idempotent when possible + - Detailed error messages + - Comprehensive documentation + +3. Memory Systems + Types of memory agents need: + - Working memory: Current task context + - Short-term memory: Recent interactions + - Long-term memory: Persistent knowledge + - Episodic memory: Past experiences + - Semantic memory: General knowledge + + Implementation approaches: + - Vector databases for semantic search + - Graph databases for relationships + - Key-value stores for fast lookup + - SQL databases for structured data + +4. Planning and Decision Making + Strategies for complex tasks: + - Breadth-first vs depth-first search + - Heuristic-guided planning + - Monte Carlo tree search + - Reinforcement learning + - Symbolic reasoning integration + +5. Monitoring and Observability + Critical metrics to track: + - Task completion rate + - Average execution time + - Token usage and costs + - Error rates by type + - Tool usage patterns + - User satisfaction scores + + Logging and tracing: + - Structured logging (JSON format) + - Distributed tracing (trace IDs) + - Action replay capabilities + - Performance profiling + - Anomaly detection + +Production Deployment Considerations +==================================== + +1. Safety and Control + Implementing guardrails: + - Input validation and sanitization + - Output filtering and moderation + - Action confirmation for critical operations + - Rollback mechanisms + - Circuit breakers for cascading failures + - Human-in-the-loop for high-risk decisions + +2. Cost Management + Strategies to control expenses: + - Budget limits per agent/user + - Automatic degradation to cheaper models + - Request batching and caching + - Usage analytics and alerts + - Prompt optimization for token efficiency + +3. Latency Optimization + Reducing response time: + - Parallel tool execution when possible + - Streaming responses to users + - Pre-warming model connections + - Edge deployment for low latency + - Async processing for non-critical tasks + +4. Scalability + Handling growth: + - Stateless agent design + - Horizontal scaling with load balancers + - Queue-based task distribution + - Database sharding strategies + - Caching layers (Redis, Memcached) + +5. Reliability + Building fault-tolerant systems: + - Graceful degradation + - Automatic retry with exponential backoff + - Dead letter queues for failed tasks + - Health checks and auto-recovery + - Multi-region deployment + +Common Pitfalls and Solutions +============================= + +1. Infinite Loops + Problem: Agent gets stuck repeating same actions + Solutions: + - Implement maximum iteration limits + - Track state to detect loops + - Add randomization to break patterns + - Use reflection to detect stuck states + +2. Context Window Overflow + Problem: Conversation history exceeds model limits + Solutions: + - Implement context summarization + - Use sliding window approach + - Store full history, send summaries + - Priority-based context selection + +3. Tool Hallucination + Problem: Agent tries to use non-existent tools + Solutions: + - Provide clear tool documentation in prompt + - Validate tool names before execution + - Use structured output formats (JSON) + - Fine-tune on correct tool usage + +4. Inconsistent Behavior + Problem: Agent gives different results for same input + Solutions: + - Set temperature to 0 for determinism + - Implement result caching + - Add explicit reasoning chain requirements + - Use voting/consensus from multiple runs + +5. Poor Error Recovery + Problem: Single failures cause complete task abandonment + Solutions: + - Implement retry logic with backoff + - Fallback to alternative approaches + - Graceful degradation to partial results + - Clear error messages and recovery suggestions + +Testing Strategies +================== + +1. Unit Testing + - Test individual tool functions + - Mock LLM responses + - Validate prompt templates + - Test parsing and formatting logic + +2. Integration Testing + - End-to-end task completion + - Tool chain execution + - Memory persistence + - API integration points + +3. Evaluation Benchmarks + - Task success rate + - Response quality (human eval) + - Reasoning coherence + - Tool usage efficiency + - Cost per task + +4. Adversarial Testing + - Malicious input handling + - Edge case scenarios + - Resource exhaustion attacks + - Prompt injection attempts + +5. A/B Testing + - Compare prompt variations + - Test different model versions + - Evaluate architecture changes + - Measure user satisfaction + +Real-World Use Cases +==================== + +1. Customer Support Agent + Capabilities: + - Answer common questions using knowledge base + - Create support tickets for complex issues + - Schedule appointments and callbacks + - Escalate to human agents when needed + + Key challenges: + - Maintaining empathy in responses + - Handling frustrated customers + - Accurate issue classification + - Privacy and data security + +2. Research Assistant Agent + Capabilities: + - Search academic databases + - Summarize research papers + - Identify trends and gaps + - Generate literature reviews + + Key challenges: + - Source credibility verification + - Citation accuracy + - Handling conflicting information + - Domain expertise requirements + +3. DevOps Automation Agent + Capabilities: + - Monitor system metrics + - Diagnose performance issues + - Execute remediation actions + - Generate incident reports + + Key challenges: + - High stakes decision-making + - Complex system dependencies + - Security and access control + - Audit trail requirements + +4. Sales Prospecting Agent + Capabilities: + - Research potential customers + - Personalize outreach messages + - Schedule meetings + - Track engagement and follow-ups + + Key challenges: + - Avoiding spam-like behavior + - Personalization at scale + - CRM integration complexity + - Compliance with regulations + +Performance Optimization +======================== + +1. Prompt Engineering + - Use clear, structured instructions + - Provide relevant examples (few-shot) + - Include constraints and boundaries + - Optimize token usage + - Version control prompts + +2. Model Selection + - Match model capability to task complexity + - Use smaller models for simple tasks + - Consider latency requirements + - Balance cost vs performance + - Evaluate fine-tuning benefits + +3. Caching Strategies + - Cache LLM responses for common queries + - Cache tool results when appropriate + - Implement embeddings cache + - Use CDN for static resources + - Cache database queries + +4. Parallel Execution + - Identify independent tool calls + - Execute in parallel where possible + - Use async/await patterns + - Implement concurrent request limits + - Handle partial failures gracefully + +Future Trends +============= + +1. Multi-Modal Agents + - Vision, audio, and text integration + - Video understanding capabilities + - Embodied AI for robotics + - Mixed reality interactions + +2. Improved Planning + - Better long-term reasoning + - Hierarchical task decomposition + - Probabilistic planning under uncertainty + - Resource-constrained optimization + +3. Enhanced Memory + - Better long-term retention + - Efficient memory consolidation + - Personalization and adaptation + - Cross-agent knowledge sharing + +4. Tool Learning + - Automatic tool discovery + - Tool composition and chaining + - Learning from tool usage patterns + - Generating new tools dynamically + +5. Human-Agent Collaboration + - Natural delegation interfaces + - Explainable decision-making + - Interactive planning and refinement + - Shared mental models + +Conclusion +========== + +Building production-ready AI agents requires careful attention to architecture, safety, performance, and user experience. Success comes from: +- Starting with clear, well-defined use cases +- Iterating based on real-world feedback +- Investing in monitoring and observability +- Prioritizing safety and controllability +- Continuously optimizing costs and performance + +The agent paradigm represents a significant leap from traditional applications, but with thoughtful design and implementation, agents can deliver tremendous value while operating reliably at scale. + +References and Resources +======================== + +Key papers: +- "ReAct: Synergizing Reasoning and Acting in Language Models" (Yao et al., 2022) +- "Toolformer: Language Models Can Teach Themselves to Use Tools" (Schick et al., 2023) +- "Reflexion: Language Agents with Verbal Reinforcement Learning" (Shinn et al., 2023) + +Frameworks and tools: +- LangChain: Popular agent framework +- AutoGPT: Autonomous agent implementation +- BabyAGI: Task-driven autonomous agent +- AgentGPT: Web-based agent platform + +Communities: +- LangChain Discord +- AI Agent subreddit +- AutoGPT GitHub discussions +- Agent research papers on ArXiv diff --git a/examples/resources/docs/doc_sample.pdf b/examples/resources/docs/doc_sample.pdf new file mode 100644 index 0000000..7faacff Binary files /dev/null and b/examples/resources/docs/doc_sample.pdf differ diff --git a/examples/resources/images/image1.png b/examples/resources/images/image1.png new file mode 100644 index 0000000..98a1160 Binary files /dev/null and b/examples/resources/images/image1.png differ diff --git a/examples/resources/logs/log1.txt b/examples/resources/logs/log1.txt new file mode 100644 index 0000000..66d00ff --- /dev/null +++ b/examples/resources/logs/log1.txt @@ -0,0 +1,212 @@ +Agent Runtime Log - Production Microservice Deployment (ATTEMPT 1 - FAILURE) +================================================================================ +Session ID: deploy-agent-20241120-1430-attempt1 +Task: "Deploy recommendation microservice to production with zero downtime" +Agent: DeploymentAgent v3.2 +Start Time: 2024-11-20 14:30:00 UTC +Attempt: 1/3 + +[14:30:00] [INIT] Agent initialization started +[14:30:00] [INFO] Task: Deploy recommendation-service v2.5.0 to production +[14:30:00] [INFO] Target environment: production-cluster-us-east +[14:30:01] [INFO] Deployment strategy: Blue-green with gradual traffic shift +[14:30:01] [SUCCESS] Agent initialized + +[14:30:01] [PLANNING] Creating deployment plan +[14:30:02] [THOUGHT] Deployment steps should include: + 1. Validate service configuration and dependencies + 2. Build and push Docker image + 3. Deploy green environment alongside blue + 4. Run health checks and smoke tests + 5. Gradually shift traffic from blue to green + 6. Monitor metrics and rollback if needed + +[14:30:02] [ACTION] Step 1: Validating service configuration +[14:30:02] [TOOL] Executing: validate_config(service="recommendation-service", version="v2.5.0") +[14:30:05] [OBSERVATION] Configuration validation results: + - Service manifest: Valid + - Environment variables: 12 variables defined + - Resource limits: CPU 2 cores, Memory 4Gi + - Dependencies: PostgreSQL, Redis, Kafka + - Health check endpoint: /health + +[14:30:05] [THOUGHT] Configuration looks good. Proceeding with Docker build. + +[14:30:05] [ACTION] Step 2: Building Docker image +[14:30:05] [TOOL] Executing: build_docker_image( + dockerfile="services/recommendation/Dockerfile", + tag="recommendation-service:v2.5.0" +) +[14:30:55] [OBSERVATION] Docker build completed: + - Image: recommendation-service:v2.5.0 + - Size: 856 MB + - Build time: 50 seconds + - Base image: python:3.11-slim + - Layers: 12 + +[14:30:55] [ACTION] Pushing image to registry +[14:30:55] [TOOL] Executing: push_image(image="recommendation-service:v2.5.0") +[14:31:20] [OBSERVATION] Image pushed successfully to gcr.io/company/recommendation-service:v2.5.0 + +[14:31:20] [ACTION] Step 3: Deploying green environment +[14:31:20] [TOOL] Executing: deploy_kubernetes( + service="recommendation-service", + version="v2.5.0", + environment="green", + replicas=3 +) +[14:31:25] [OBSERVATION] Kubernetes deployment initiated: + - Deployment: recommendation-service-green + - Replicas: 3 + - Namespace: production + - Pods starting... + +[14:31:45] [ACTION] Monitoring pod startup +[14:31:46] [OBSERVATION] Pod status: + - recommendation-green-0: Running + - recommendation-green-1: Running + - recommendation-green-2: Running + - All pods ready and passing health checks + +[14:31:46] [ACTION] Step 4: Running smoke tests on green environment +[14:31:46] [TOOL] Executing: run_smoke_tests(target="recommendation-green") +[14:32:00] [OBSERVATION] Smoke test results: + - Health check: PASSED (200 OK) + - Database connectivity: PASSED + - Redis connectivity: PASSED + - Kafka connectivity: PASSED + - Sample recommendation request: PASSED (response time: 120ms) + +[14:32:00] [THOUGHT] Green environment healthy. Ready to shift traffic. + +[14:32:00] [ACTION] Step 5: Starting gradual traffic shift (Blue → Green) +[14:32:00] [TOOL] Executing: traffic_shift( + from_env="blue", + to_env="green", + strategy="gradual", + initial_percentage=10 +) +[14:32:05] [OBSERVATION] Traffic shift initiated: + - Blue environment: 90% traffic + - Green environment: 10% traffic + - Monitoring for 5 minutes before next increment + +[14:32:05] [ACTION] Monitoring metrics during traffic shift +[14:33:00] [OBSERVATION] Metrics at 10% traffic (1 minute): + - Request rate: 150 req/s to green (1500 req/s total) + - Average latency: 125ms (baseline: 110ms) + - Error rate: 0.1% + - All metrics within acceptable range + +[14:35:00] [ACTION] Increasing traffic to 25% +[14:35:01] [TOOL] Executing: traffic_shift(percentage=25) +[14:35:05] [OBSERVATION] Traffic updated: + - Blue: 75%, Green: 25% + +[14:36:00] [OBSERVATION] Metrics at 25% traffic (1 minute): + - Request rate: 375 req/s to green + - Average latency: 130ms + - Error rate: 0.2% + - Metrics stable + +[14:38:00] [ACTION] Increasing traffic to 50% +[14:38:01] [TOOL] Executing: traffic_shift(percentage=50) +[14:38:05] [OBSERVATION] Traffic updated: + - Blue: 50%, Green: 50% + +[14:39:00] [OBSERVATION] Metrics at 50% traffic (1 minute): + - Request rate: 750 req/s to green + - Average latency: 135ms + - Error rate: 0.3% + +[14:40:30] [ERROR] Alert triggered: Error rate spike detected! +[14:40:31] [TOOL] Executing: check_error_logs() +[14:40:32] [OBSERVATION] ERROR ANALYSIS: + - Error rate jumped from 0.3% to 4.2% (14x increase) + - Error type: "DatabaseConnectionError: too many connections" + - Affected pods: All 3 green pods + - Pattern: Started at 50% traffic mark + +[14:40:32] [THOUGHT] CRITICAL: Connection pool exhausted at 50% traffic! + +[14:40:32] [ACTION] Investigating database connection configuration +[14:40:32] [TOOL] Executing: check_db_config(service="recommendation-green") +[14:40:35] [OBSERVATION] Database configuration: + - Max connections per pod: 20 + - Total pods: 3 + - Total connection capacity: 60 connections + - Current active connections: 58 (approaching limit) + - Database max_connections: 100 + - Blue environment using: 40 connections + - Green environment using: 58 connections + - TOTAL: 98/100 connections (98% utilization!) + +[14:40:35] [THOUGHT] Root cause identified: Insufficient connection pool size! + - At 50% traffic, green needs ~60 connections + - At 100% traffic, would need ~120 connections + - Database max_connections=100 is insufficient + - We're already at 98% capacity with both environments running + +[14:40:35] [ACTION] Initiating emergency rollback +[14:40:35] [TOOL] Executing: rollback_traffic(to_env="blue") +[14:40:40] [OBSERVATION] Traffic rollback initiated: + - Shifting all traffic back to blue environment + - Green: 50% → 25% → 10% → 0% + +[14:41:00] [OBSERVATION] Rollback progress: + - Traffic: 100% on blue, 0% on green + - Error rate: Dropping (4.2% → 2.1% → 0.4%) + - Service stabilizing + +[14:41:30] [OBSERVATION] Rollback complete: + - Blue environment: 100% traffic, stable + - Green environment: 0% traffic, shutting down + - Error rate: Back to 0.2% (normal baseline) + - Service restored + +[14:41:30] [FAILURE] Deployment failed - Root cause analysis: + +PRIMARY ISSUE: Database connection pool misconfiguration + - Database max_connections: 100 (too low for blue+green running simultaneously) + - Each environment needs ~60 connections at 50% traffic + - During blue-green deployment, total needed: 100+ connections + - Database cannot support both environments concurrently + +CONTRIBUTING FACTORS: + 1. Did not account for connection pool sizing during dual environment operation + 2. No load testing performed with both blue and green running simultaneously + 3. Connection pool monitoring alerts not in place + 4. Gradual rollout exposed issue only at 50% mark + +LESSONS LEARNED: + 1. ⚠️ CRITICAL: Database connection pools must accommodate BOTH environments during deployment + 2. ⚠️ Connection pool size = (peak connections per pod) × (total pods in blue + green) + 3. ⚠️ Must load test with full capacity of both environments before production + 4. ⚠️ Set up connection pool monitoring and alerts BEFORE deployment + 5. ⚠️ Have emergency rollback procedure ready (which saved us here) + +REMEDIATION PLAN: + 1. Increase database max_connections from 100 to 250 + 2. Adjust connection pool per pod from 20 to 30 + 3. Add connection pool utilization monitoring + 4. Set alert threshold at 70% connection utilization + 5. Document connection pool requirements in deployment checklist + +DEPLOYMENT STATUS: FAILED +ROLLBACK STATUS: SUCCESSFUL (no customer impact beyond brief error spike) +TIME TO DETECT: 8 minutes +TIME TO ROLLBACK: 1.5 minutes +CUSTOMER IMPACT: 1.5 minutes of elevated errors (4.2% error rate) + +[14:41:30] [EXPORT] Saving failure report: reports/deployment-attempt1-failure.json +[14:41:31] [NOTIFICATION] Alerting team about deployment failure and lessons learned +[14:41:31] [SHUTDOWN] Agent session terminated + +================================================================================= +KEY TAKEAWAYS FOR NEXT ATTEMPT: +================================================================================= +1. Fix database connection pool sizing BEFORE retry +2. Calculate total connections needed: (pods_blue + pods_green) × connections_per_pod +3. Add connection monitoring to detect issues early +4. Test with both environments running at full capacity +5. Verify database can handle combined load before production deployment diff --git a/examples/resources/logs/log2.txt b/examples/resources/logs/log2.txt new file mode 100644 index 0000000..5a27886 --- /dev/null +++ b/examples/resources/logs/log2.txt @@ -0,0 +1,226 @@ +Agent Runtime Log - Production Microservice Deployment (ATTEMPT 2 - FAILURE) +================================================================================ +Session ID: deploy-agent-20241121-0930-attempt2 +Task: "Deploy recommendation microservice to production with zero downtime" +Agent: DeploymentAgent v3.2 +Start Time: 2024-11-21 09:30:00 UTC +Attempt: 2/3 +Previous Attempt: FAILED (database connection pool exhaustion) + +[09:30:00] [INIT] Agent initialization started +[09:30:00] [INFO] Task: Deploy recommendation-service v2.5.0 to production (RETRY) +[09:30:00] [INFO] Applying lessons from Attempt 1 +[09:30:01] [INFO] Fixed issues: + - ✓ Database max_connections increased: 100 → 250 + - ✓ Connection pool per pod increased: 20 → 30 + - ✓ Added connection pool monitoring alerts + - ✓ Verified capacity calculation: (3 blue + 3 green) × 30 = 180 connections (72% of 250) +[09:30:01] [SUCCESS] Agent initialized with fixes applied + +[09:30:01] [PLANNING] Creating deployment plan (improved from attempt 1) +[09:30:02] [THOUGHT] Key improvements for this attempt: + 1. Pre-deployment connection pool validation + 2. Monitor connection utilization during traffic shift + 3. More gradual traffic ramp (10% → 25% → 50% → 75% → 100%) + 4. Longer monitoring windows between increments (5 min instead of 3 min) + +[09:30:02] [ACTION] Step 1: Pre-deployment validation +[09:30:02] [TOOL] Executing: validate_database_capacity() +[09:30:05] [OBSERVATION] Database capacity check: + - Current connections (blue only): 42/250 (17% utilization) + - Projected at 100% blue + 100% green: ~180/250 (72% utilization) + - Safety margin: 28% + - ✓ PASS: Sufficient capacity for blue-green deployment + +[09:30:05] [THOUGHT] Connection pool capacity validated. Safe to proceed. + +[09:30:05] [ACTION] Step 2: Building and pushing Docker image +[09:30:05] [TOOL] Executing: build_and_push_image(tag="v2.5.0") +[09:30:55] [OBSERVATION] Image ready: gcr.io/company/recommendation-service:v2.5.0 + +[09:30:55] [ACTION] Step 3: Deploying green environment +[09:30:55] [TOOL] Executing: deploy_kubernetes(environment="green", replicas=3) +[09:31:15] [OBSERVATION] Green environment deployed: + - All 3 pods running and healthy + - Health checks passing + - Ready for traffic + +[09:31:15] [ACTION] Step 4: Starting gradual traffic shift with enhanced monitoring +[09:31:15] [TOOL] Executing: traffic_shift(percentage=10, monitor_connections=true) +[09:31:20] [OBSERVATION] Traffic at 10%: + - Blue: 90%, Green: 10% + - Monitoring connection pools... + +[09:32:20] [OBSERVATION] Metrics after 1 minute at 10%: + - Request rate: 150 req/s to green + - Average latency: 118ms (good) + - Error rate: 0.1% (normal) + - DB connections: Blue 40, Green 15, Total 55/250 (22%) + - ✓ All metrics healthy + +[09:36:20] [ACTION] Increasing traffic to 25% (after 5 minute monitoring) +[09:36:21] [TOOL] Executing: traffic_shift(percentage=25) +[09:36:25] [OBSERVATION] Traffic at 25%: + - Blue: 75%, Green: 25% + +[09:37:25] [OBSERVATION] Metrics after 1 minute at 25%: + - Request rate: 375 req/s to green + - Average latency: 122ms + - Error rate: 0.1% + - DB connections: Blue 35, Green 28, Total 63/250 (25%) + - ✓ All metrics healthy + +[09:41:25] [ACTION] Increasing traffic to 50% +[09:41:26] [TOOL] Executing: traffic_shift(percentage=50) +[09:41:30] [OBSERVATION] Traffic at 50%: + - Blue: 50%, Green: 50% + +[09:42:30] [OBSERVATION] Metrics after 1 minute at 50%: + - Request rate: 750 req/s to green + - Average latency: 125ms + - Error rate: 0.2% + - DB connections: Blue 58, Green 62, Total 120/250 (48%) + - ✓ No connection issues! (Fixed from Attempt 1) + +[09:43:30] [OBSERVATION] Metrics after 2 minutes at 50%: + - Latency: 128ms + - Error rate: 0.2% + - DB connections: 125/250 (50%) + - ✓ Stable performance + +[09:46:30] [ACTION] Increasing traffic to 75% +[09:46:31] [TOOL] Executing: traffic_shift(percentage=75) +[09:46:35] [OBSERVATION] Traffic at 75%: + - Blue: 25%, Green: 75% + +[09:47:35] [OBSERVATION] Metrics after 1 minute at 75%: + - Request rate: 1125 req/s to green + - Average latency: 132ms + - Error rate: 0.3% + +[09:48:35] [OBSERVATION] Metrics after 2 minutes at 75%: + - Request rate: 1125 req/s + - Average latency: 145ms (increasing trend) + - Error rate: 0.4% (slight increase) + +[09:49:35] [OBSERVATION] Metrics after 3 minutes at 75%: + - Request rate: 1125 req/s + - Average latency: 178ms (significant increase!) + - Error rate: 0.6% + - DB connections: 155/250 (62%) + +[09:50:00] [WARNING] Latency threshold exceeded: 178ms (threshold: 150ms) +[09:50:01] [ACTION] Investigating latency spike +[09:50:01] [TOOL] Executing: analyze_performance_bottleneck() +[09:50:05] [OBSERVATION] Performance analysis: + - CPU utilization: 45% (normal) + - Memory utilization: 58% (normal) + - Network I/O: Normal + - Database query time: 85ms (elevated from baseline 40ms) + - Database CPU: 78% (high!) + - Database connections: 155 (healthy) + +[09:50:05] [THOUGHT] Database performance degrading under load, but not connection related. + +[09:51:00] [OBSERVATION] Continued monitoring: + - Average latency: 215ms (getting worse) + - P95 latency: 450ms + - P99 latency: 780ms (SLO violation: >500ms) + - Error rate: 1.2% (increasing) + +[09:51:00] [ERROR] SLO violation: P99 latency exceeds 500ms threshold + +[09:51:00] [ACTION] Deep dive into database performance +[09:51:00] [TOOL] Executing: analyze_database_queries() +[09:51:10] [OBSERVATION] Query performance analysis: + - Slow query detected: recommendation_candidates lookup + - Query time: 200-400ms (baseline: 30-50ms) + - Cause: Missing database index on new query pattern + - New v2.5.0 feature uses "user_segment" field in WHERE clause + - Index exists on "user_id" but NOT on "user_segment" + - Full table scan on 50M row table causing slowdown + +[09:51:10] [THOUGHT] ROOT CAUSE: Missing database index for new query pattern! + - V2.5.0 introduced new recommendation algorithm + - Uses user_segment field not previously indexed + - Works fine at low traffic but degrades at high volume + - This was not caught in staging (smaller data volume) + +[09:51:10] [ERROR] CRITICAL: Cannot fix index during deployment (requires DB downtime or long rebuild) + +[09:51:10] [ACTION] Initiating rollback to prevent further SLO violations +[09:51:10] [TOOL] Executing: rollback_traffic(to_env="blue") +[09:51:15] [OBSERVATION] Rolling back traffic: + - Green: 75% → 50% → 25% → 10% → 0% + +[09:52:00] [OBSERVATION] Rollback progress: + - Traffic: 100% on blue + - Latency dropping: 215ms → 145ms → 110ms + - Error rate: 1.2% → 0.5% → 0.2% + - Service stabilizing + +[09:52:30] [OBSERVATION] Rollback complete: + - Blue environment: 100% traffic, stable + - Green environment: 0% traffic, shutting down + - Average latency: 108ms (back to normal) + - Error rate: 0.2% (baseline) + - Service fully restored + +[09:52:30] [FAILURE] Deployment failed - Root cause analysis: + +PRIMARY ISSUE: Missing database index for new query pattern + - V2.5.0 code uses new WHERE clause on "user_segment" field + - No index exists for this field + - Full table scan on 50M rows causes 200-400ms queries + - Acceptable at low traffic, degrades at high volume + +CONTRIBUTING FACTORS: + 1. Code review did not flag new query pattern requiring index + 2. Staging environment has smaller dataset (5M rows vs 50M in prod) + 3. Performance testing in staging didn't reveal the issue + 4. Database indexing not part of deployment checklist + 5. No query performance monitoring during deployment + +WHY ATTEMPT 1 FIX WASN'T ENOUGH: + - Fixed connection pool issue ✓ + - But didn't discover the index issue ✗ + - Different root causes require different solutions + +LESSONS LEARNED (New from Attempt 2): + 1. ⚠️ CRITICAL: Analyze ALL new query patterns for index requirements + 2. ⚠️ Run EXPLAIN ANALYZE on new queries before deployment + 3. ⚠️ Performance test with production-scale data volumes + 4. ⚠️ Monitor query execution time during deployment + 5. ⚠️ Create required indexes BEFORE deploying code that needs them + 6. ⚠️ Staging must have production-like data volume for realistic testing + +REMEDIATION PLAN: + 1. Create index: CREATE INDEX idx_user_segment ON users(user_segment) + 2. Wait for index build to complete (~2 hours on 50M rows) + 3. Add index creation to deployment prerequisites checklist + 4. Update staging database to match production size + 5. Add query performance monitoring to deployment pipeline + +DEPLOYMENT STATUS: FAILED +ROLLBACK STATUS: SUCCESSFUL +TIME TO DETECT: 5 minutes at 75% traffic +TIME TO ROLLBACK: 1.5 minutes +CUSTOMER IMPACT: 5 minutes of degraded performance (P99 latency 500-780ms, 1.2% errors) + +[09:52:30] [EXPORT] Saving failure report: reports/deployment-attempt2-failure.json +[09:52:31] [NOTIFICATION] Alerting team: Index required before retry +[09:52:31] [SHUTDOWN] Agent session terminated + +================================================================================= +KEY TAKEAWAYS FOR NEXT ATTEMPT: +================================================================================= +1. Database index on user_segment MUST be created before retry +2. Verify index exists: SELECT * FROM pg_indexes WHERE indexname = 'idx_user_segment' +3. Add query performance monitoring to catch slow queries early +4. Test with production-scale data before deploying +5. Include "Database Schema Changes" section in deployment checklist + +CUMULATIVE LESSONS (From Both Attempts): +- Connection pool sizing for dual environments (Attempt 1) +- Database index requirements for new queries (Attempt 2) +- Both are critical and both must be fixed for success diff --git a/examples/resources/logs/log3.txt b/examples/resources/logs/log3.txt new file mode 100644 index 0000000..882b629 --- /dev/null +++ b/examples/resources/logs/log3.txt @@ -0,0 +1,374 @@ +Agent Runtime Log - Production Microservice Deployment (ATTEMPT 3 - SUCCESS) +================================================================================ +Session ID: deploy-agent-20241122-1000-attempt3 +Task: "Deploy recommendation microservice to production with zero downtime" +Agent: DeploymentAgent v3.2 +Start Time: 2024-11-22 10:00:00 UTC +Attempt: 3/3 +Previous Attempts: + - Attempt 1: FAILED (database connection pool exhaustion) + - Attempt 2: FAILED (missing database index) + +[10:00:00] [INIT] Agent initialization started +[10:00:00] [INFO] Task: Deploy recommendation-service v2.5.0 to production (FINAL ATTEMPT) +[10:00:00] [INFO] Applying ALL lessons from previous attempts +[10:00:01] [INFO] Pre-deployment checklist completed: + - ✓ Database max_connections: 250 (increased from 100) + - ✓ Connection pool per pod: 30 (increased from 20) + - ✓ Connection pool monitoring: ACTIVE + - ✓ Database index idx_user_segment: CREATED and READY + - ✓ Index verified: SELECT * FROM pg_indexes → Found + - ✓ Query performance tested with EXPLAIN ANALYZE → 35ms (excellent) + - ✓ Staging tests passed with production-scale data + - ✓ All monitoring alerts configured +[10:00:01] [SUCCESS] Agent initialized - Ready for deployment with all fixes applied + +[10:00:01] [PLANNING] Creating comprehensive deployment plan +[10:00:02] [THOUGHT] This attempt incorporates: + FROM ATTEMPT 1: Connection pool sizing for dual environment operation + FROM ATTEMPT 2: Database index requirements and query performance monitoring + NEW FOR ATTEMPT 3: Enhanced pre-deployment validation and longer monitoring windows + +[10:00:02] [ACTION] Step 1: Comprehensive pre-deployment validation +[10:00:02] [TOOL] Executing: run_predeploy_checks() +[10:00:10] [OBSERVATION] Pre-deployment validation results: + + ✓ Database Capacity Check: + - Current connections: 45/250 (18%) + - Projected peak: 180/250 (72%) + - Safety margin: 28% ✓ + + ✓ Database Index Check: + - idx_user_segment exists: YES + - Index size: 1.2 GB + - Index scan cost: Low (35ms query time) + + ✓ Query Performance Check: + - EXPLAIN ANALYZE user_segment query: 35ms ✓ + - All queries under 100ms threshold ✓ + + ✓ Resource Availability: + - Kubernetes nodes: 8 available + - CPU capacity: 45% utilization + - Memory capacity: 52% utilization + + ✓ Dependency Health: + - PostgreSQL: Healthy + - Redis: Healthy + - Kafka: Healthy + + ✓ Monitoring Systems: + - Prometheus: Active + - Grafana: Active + - Alert manager: Active + - All deployment alerts configured + +[10:00:10] [THOUGHT] All validation checks passed. Proceeding with high confidence. + +[10:00:10] [ACTION] Step 2: Building and pushing Docker image +[10:00:10] [TOOL] Executing: build_and_push_image(tag="v2.5.0") +[10:01:00] [OBSERVATION] Image ready: gcr.io/company/recommendation-service:v2.5.0 + +[10:01:00] [ACTION] Step 3: Deploying green environment +[10:01:00] [TOOL] Executing: deploy_kubernetes( + environment="green", + replicas=3, + resource_limits={"cpu": "2", "memory": "4Gi"} +) +[10:01:05] [OBSERVATION] Kubernetes deployment started... + +[10:01:25] [OBSERVATION] Green environment status: + - recommendation-green-0: Running, Ready + - recommendation-green-1: Running, Ready + - recommendation-green-2: Running, Ready + - Health checks: All passing + - Database connections: 15 (healthy) + +[10:01:25] [ACTION] Step 4: Running comprehensive smoke tests +[10:01:25] [TOOL] Executing: run_smoke_tests(environment="green", comprehensive=true) +[10:01:45] [OBSERVATION] Smoke test results: + ✓ Health endpoint: 200 OK + ✓ Database connectivity: Connected, 15 connections + ✓ Redis connectivity: Connected + ✓ Kafka connectivity: Connected + ✓ Sample recommendation request: Success (117ms) + ✓ User segment query performance: 33ms (excellent!) + ✓ Cache hit rate: 78% + ✓ All functional tests: PASSED (20/20) + +[10:01:45] [THOUGHT] Green environment fully validated. Beginning gradual traffic shift. + +[10:01:45] [ACTION] Step 5: Gradual traffic shift - Stage 1 (10%) +[10:01:45] [TOOL] Executing: traffic_shift(percentage=10, monitoring_window="5min") +[10:01:50] [OBSERVATION] Traffic distribution: + - Blue: 90%, Green: 10% + - Monitoring for 5 minutes... + +[10:02:50] [OBSERVATION] Metrics at 10% (1 minute): + - Request rate: 150 req/s to green + - Average latency: 115ms + - P95 latency: 145ms + - P99 latency: 185ms + - Error rate: 0.1% + - DB connections: Blue 40, Green 15, Total 55/250 (22%) + - Query performance: user_segment queries 30-40ms ✓ + - ✓ All metrics within target ranges + +[10:04:50] [OBSERVATION] Metrics at 10% (3 minutes): + - Average latency: 118ms (stable) + - Error rate: 0.1% (normal) + - DB connections: 57/250 (23%) + - ✓ Performance stable + +[10:06:50] [OBSERVATION] Metrics at 10% (5 minutes): + - Average latency: 116ms (stable) + - Error rate: 0.1% (normal) + - DB connections: 56/250 (22%) + - ✓ Stage 1 successful - Ready for next increment + +[10:06:50] [ACTION] Step 6: Gradual traffic shift - Stage 2 (25%) +[10:06:51] [TOOL] Executing: traffic_shift(percentage=25, monitoring_window="5min") +[10:06:55] [OBSERVATION] Traffic distribution: + - Blue: 75%, Green: 25% + +[10:07:55] [OBSERVATION] Metrics at 25% (1 minute): + - Request rate: 375 req/s to green + - Average latency: 120ms + - P99 latency: 192ms + - Error rate: 0.1% + - DB connections: Blue 35, Green 28, Total 63/250 (25%) + - ✓ Healthy + +[10:11:55] [OBSERVATION] Metrics at 25% (5 minutes): + - Average latency: 122ms (stable) + - Error rate: 0.1% (normal) + - DB connections: 65/250 (26%) + - ✓ Stage 2 successful + +[10:11:55] [ACTION] Step 7: Gradual traffic shift - Stage 3 (50%) +[10:11:56] [TOOL] Executing: traffic_shift(percentage=50, monitoring_window="8min") +[10:12:00] [OBSERVATION] Traffic distribution: + - Blue: 50%, Green: 50% + +[10:13:00] [OBSERVATION] Metrics at 50% (1 minute): + - Request rate: 750 req/s to green + - Average latency: 125ms + - P99 latency: 198ms + - Error rate: 0.2% + - DB connections: Blue 58, Green 62, Total 120/250 (48%) + - ✓ No connection pool issues (fixed from Attempt 1) + +[10:15:00] [OBSERVATION] Metrics at 50% (3 minutes): + - Average latency: 127ms (stable) + - P99 latency: 195ms + - Error rate: 0.2% + - DB connections: 122/250 (49%) + - Query performance: 32-38ms (excellent - fixed from Attempt 2) + - ✓ Both previous issues resolved! + +[10:20:00] [OBSERVATION] Metrics at 50% (8 minutes): + - Average latency: 126ms (stable) + - P99 latency: 196ms (under 500ms SLO ✓) + - Error rate: 0.2% + - DB connections: 125/250 (50%) + - ✓ Stage 3 successful - Performance excellent + +[10:20:00] [ACTION] Step 8: Gradual traffic shift - Stage 4 (75%) +[10:20:01] [TOOL] Executing: traffic_shift(percentage=75, monitoring_window="8min") +[10:20:05] [OBSERVATION] Traffic distribution: + - Blue: 25%, Green: 75% + +[10:21:05] [OBSERVATION] Metrics at 75% (1 minute): + - Request rate: 1125 req/s to green + - Average latency: 132ms + - P99 latency: 205ms + - Error rate: 0.2% + - DB connections: Blue 20, Green 88, Total 108/250 (43%) + - ✓ Healthy + +[10:24:05] [OBSERVATION] Metrics at 75% (4 minutes): + - Average latency: 134ms (stable, no degradation!) + - P99 latency: 208ms (well under SLO) + - Error rate: 0.2% + - DB connections: 110/250 (44%) + - Query performance: 35-42ms (consistent, index working perfectly) + - ✓ No latency spike (Attempt 2 issue resolved!) + +[10:28:05] [OBSERVATION] Metrics at 75% (8 minutes): + - Average latency: 135ms (stable) + - P99 latency: 210ms + - Error rate: 0.2% + - DB connections: 112/250 (45%) + - ✓ Stage 4 successful - Ready for full cutover + +[10:28:05] [THOUGHT] All stages successful. Proceeding to 100% with confidence. + +[10:28:05] [ACTION] Step 9: Final traffic shift - 100% to Green +[10:28:06] [TOOL] Executing: traffic_shift(percentage=100, monitoring_window="10min") +[10:28:10] [OBSERVATION] Traffic distribution: + - Blue: 0%, Green: 100% + - All production traffic now on v2.5.0 + +[10:29:10] [OBSERVATION] Metrics at 100% (1 minute): + - Request rate: 1500 req/s (full production load) + - Average latency: 138ms + - P95 latency: 172ms + - P99 latency: 215ms (well under 500ms SLO ✓) + - Error rate: 0.2% (baseline) + - DB connections: 115/250 (46%) + - ✓ Full cutover successful! + +[10:33:10] [OBSERVATION] Metrics at 100% (5 minutes): + - Average latency: 137ms (stable) + - P99 latency: 218ms + - Error rate: 0.2% + - DB connections: 118/250 (47%) + - Cache hit rate: 79% + - Throughput: 1500 req/s + - ✓ Service performing excellently + +[10:38:10] [OBSERVATION] Metrics at 100% (10 minutes): + - Average latency: 136ms (stable and excellent) + - P99 latency: 216ms (SLO: <500ms ✓) + - Error rate: 0.2% (normal baseline) + - DB connections: 120/250 (48% - plenty of headroom) + - Query performance: 33-40ms (index performing perfectly) + - ✓ Service stable at full production load + +[10:38:10] [SUCCESS] Deployment completed successfully! + +[10:38:10] [ACTION] Step 10: Cleanup old blue environment +[10:38:10] [TOOL] Executing: cleanup_old_environment(environment="blue") +[10:38:15] [OBSERVATION] Blue environment shutdown: + - Scaled down to 0 replicas + - Resources released + - DNS cutover complete + - Only green (v2.5.0) running + +[10:38:15] [ACTION] Final validation and monitoring setup +[10:38:15] [TOOL] Executing: setup_continuous_monitoring() +[10:38:20] [OBSERVATION] Continuous monitoring configured: + - Latency alerts: P99 > 500ms + - Error rate alerts: > 1% + - Connection pool alerts: > 70% + - Query performance alerts: > 100ms + - Resource utilization alerts: > 80% + - All dashboards updated + +[10:38:20] [SUCCESS] Deployment fully complete and stable! + +================================================================================= +DEPLOYMENT SUCCESS SUMMARY +================================================================================= + +FINAL METRICS: +- Average latency: 136ms (target: <200ms) ✓ +- P99 latency: 216ms (SLO: <500ms) ✓ +- Error rate: 0.2% (baseline, target: <1%) ✓ +- Throughput: 1500 req/s (full production) ✓ +- DB connections: 120/250 (48% utilization) ✓ +- Zero downtime achieved ✓ + +DEPLOYMENT TIMELINE: +- Total deployment time: 38 minutes +- Traffic shift duration: 36 minutes (gradual) +- Monitoring windows: Conservative (5-10 min per stage) +- Zero customer impact ✓ + +ISSUES RESOLVED: +✓ Attempt 1 issue: Database connection pool sizing + - Fixed: Increased max_connections 100→250, pool per pod 20→30 + - Result: 48% utilization at full load (plenty of headroom) + +✓ Attempt 2 issue: Missing database index + - Fixed: Created idx_user_segment before deployment + - Result: Query time 33-40ms (was 200-400ms without index) + +================================================================================= +COMPREHENSIVE LESSONS LEARNED (Across All 3 Attempts) +================================================================================= + +1. DATABASE CAPACITY PLANNING (From Attempt 1): + ⚠️ Calculate connections for BOTH environments during blue-green deployment + ⚠️ Formula: (pods_blue + pods_green) × connections_per_pod + safety_margin + ⚠️ Monitor connection utilization in real-time during deployment + ⚠️ Set alerts at 70% threshold, rollback at 85% + +2. DATABASE SCHEMA CHANGES (From Attempt 2): + ⚠️ Analyze ALL new query patterns for index requirements + ⚠️ Run EXPLAIN ANALYZE on new queries before production + ⚠️ Create required indexes BEFORE deploying code + ⚠️ Test with production-scale data volumes (not just staging) + ⚠️ Monitor query execution times during deployment + +3. DEPLOYMENT BEST PRACTICES (From Attempt 3): + ⚠️ Comprehensive pre-deployment checklist is MANDATORY + ⚠️ Gradual traffic shift with adequate monitoring windows + ⚠️ Conservative approach: 10% → 25% → 50% → 75% → 100% + ⚠️ Longer monitoring at critical stages (50%, 75%, 100%) + ⚠️ Have automated rollback ready at every stage + +4. MONITORING AND OBSERVABILITY: + ⚠️ Connection pool utilization monitoring + ⚠️ Query performance monitoring (slow query detection) + ⚠️ Latency tracking (average, P95, P99) + ⚠️ Error rate monitoring with automatic alerts + ⚠️ Resource utilization (CPU, memory, network) + +5. TESTING REQUIREMENTS: + ⚠️ Staging must have production-like data volumes + ⚠️ Load testing with both blue and green at capacity + ⚠️ Query performance testing with EXPLAIN ANALYZE + ⚠️ End-to-end smoke tests on green before traffic shift + ⚠️ Rollback procedure testing in staging + +================================================================================= +DEPLOYMENT CHECKLIST (For Future Use) +================================================================================= + +PRE-DEPLOYMENT: +□ Database capacity validated (connection pools) +□ All required indexes created and verified +□ Query performance tested with EXPLAIN ANALYZE +□ Staging tests passed with production-scale data +□ Monitoring alerts configured +□ Rollback procedure tested +□ Team notified of deployment window + +DURING DEPLOYMENT: +□ Green environment health checks passing +□ Smoke tests passed +□ Gradual traffic shift with monitoring +□ Connection pool utilization < 70% +□ Query performance < 100ms +□ Latency within SLOs +□ Error rate < 1% + +POST-DEPLOYMENT: +□ Blue environment cleaned up +□ Continuous monitoring active +□ Documentation updated +□ Lessons learned documented +□ Team notified of success + +[10:38:20] [EXPORT] Saving success report: reports/deployment-attempt3-success.json +[10:38:21] [EXPORT] Updating deployment playbook with lessons learned +[10:38:22] [NOTIFICATION] Alerting team: Deployment successful! 🎉 +[10:38:22] [SHUTDOWN] Agent session terminated + +================================================================================= +KEY SUCCESS FACTORS: +================================================================================= +1. ✓ Applied lessons from BOTH previous failures +2. ✓ Comprehensive pre-deployment validation +3. ✓ Conservative, gradual traffic shift approach +4. ✓ Adequate monitoring windows at each stage +5. ✓ Fixed root causes, not just symptoms +6. ✓ Documented everything for future reference + +ATTEMPTS SUMMARY: +- Attempt 1: Failed (connection pools) → Learned capacity planning +- Attempt 2: Failed (missing index) → Learned schema validation +- Attempt 3: Success! → Applied all lessons learned + +This demonstrates the power of learning from failures! 🚀 diff --git a/examples/resources/video/video_test.mp4 b/examples/resources/video/video_test.mp4 new file mode 100644 index 0000000..0119bc3 Binary files /dev/null and b/examples/resources/video/video_test.mp4 differ diff --git a/examples/sealos-assistant/README.md b/examples/sealos-assistant/README.md new file mode 100644 index 0000000..66f46b1 --- /dev/null +++ b/examples/sealos-assistant/README.md @@ -0,0 +1,66 @@ +# MemU Assistant - Sealos DevBox Example + +A personal AI assistant with long-term memory, designed for deployment on [Sealos DevBox](https://sealos.io/products/devbox). + +## Features + +- **Persistent Memory**: Remembers user preferences and past conversations +- **REST API**: Simple endpoints for chat, memorize, and recall +- **OpenAI Compatible**: Works with OpenAI, Nebius, Groq, and other providers +- **1-Click Deploy**: Ready for Sealos DevBox deployment + +## Quick Start + +### Local Development + +```bash +# Install dependencies +pip install -r requirements.txt + +# Configure environment +cp .env.example .env +# Edit .env with your API key + +# Run the server +python main.py +``` + +### Deploy on Sealos DevBox + +1. Create a Python DevBox on [Sealos](https://cloud.sealos.io) +2. Clone this project +3. Set environment variables +4. Click **Deploy** + +See the full guide: [docs/sealos-devbox-guide.md](../../docs/sealos-devbox-guide.md) + +## API Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/` | GET | Service info | +| `/health` | GET | Health check | +| `/chat` | POST | Chat with memory | +| `/memorize` | POST | Store information | +| `/recall` | GET | Query memories | + +## Example Usage + +```bash +# Store a memory +curl -X POST http://localhost:8000/memorize \ + -H "Content-Type: application/json" \ + -d '{"content": "I prefer Python and dark mode"}' + +# Chat with memory +curl -X POST http://localhost:8000/chat \ + -H "Content-Type: application/json" \ + -d '{"message": "What are my preferences?"}' + +# Recall memories +curl "http://localhost:8000/recall?query=preferences" +``` + +## License + +MIT - Part of the MemU project diff --git a/examples/sealos-assistant/entrypoint.sh b/examples/sealos-assistant/entrypoint.sh new file mode 100644 index 0000000..5961ea1 --- /dev/null +++ b/examples/sealos-assistant/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Sealos DevBox entrypoint script +# This script is executed when deploying to production + +set -e + +# Activate virtual environment if it exists +if [ -d "venv" ]; then + source venv/bin/activate +fi + +# Start the application +exec uvicorn main:app --host 0.0.0.0 --port ${PORT:-8000} diff --git a/examples/sealos-assistant/main.py b/examples/sealos-assistant/main.py new file mode 100644 index 0000000..64fde9b --- /dev/null +++ b/examples/sealos-assistant/main.py @@ -0,0 +1,286 @@ +""" +Personal AI Assistant with Long-Term Memory +Powered by MemU + FastAPI on Sealos DevBox + +This example demonstrates how to build a memory-enabled AI assistant +that can be deployed on Sealos DevBox with 1-click deployment. + +Usage: + # Local development + pip install -r requirements.txt + python main.py + + # Or with uvicorn + uvicorn main:app --reload --host 0.0.0.0 --port 8000 +""" + +import os +import sys +from contextlib import asynccontextmanager + +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +load_dotenv() + +# Add src to path for local development +src_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "src")) +if os.path.exists(src_path): + sys.path.insert(0, src_path) + +from memu.app import MemoryService + +# Global memory service instance +memory_service: MemoryService | None = None + + +def get_llm_profiles() -> dict: + """Build LLM profiles from environment variables.""" + base_url = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1") + api_key = os.getenv("OPENAI_API_KEY") + + if not api_key: + raise ValueError("OPENAI_API_KEY environment variable is required") + + return { + "default": { + "provider": "openai", + "base_url": base_url, + "api_key": api_key, + "chat_model": os.getenv("CHAT_MODEL", "gpt-4o-mini"), + "client_backend": "sdk", + }, + "embedding": { + "provider": "openai", + "base_url": base_url, + "api_key": api_key, + "embed_model": os.getenv("EMBED_MODEL", "text-embedding-3-small"), + "client_backend": "sdk", + }, + } + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Initialize MemU memory service on startup.""" + global memory_service + + try: + llm_profiles = get_llm_profiles() + memory_service = MemoryService(llm_profiles=llm_profiles) + print("✓ MemU Memory Service initialized successfully") + except Exception as e: + print(f"✗ Failed to initialize MemU: {e}") + raise + + yield + + print("Shutting down MemU Assistant...") + + +app = FastAPI( + title="MemU Assistant", + description="AI Assistant with Long-Term Memory powered by MemU", + version="1.0.0", + lifespan=lifespan, +) + +# Enable CORS for web clients +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +# Request/Response Models +class ChatRequest(BaseModel): + message: str + user_id: str = "default" + + +class ChatResponse(BaseModel): + response: str + memories_used: int + memories_stored: int + + +class MemorizeRequest(BaseModel): + content: str + user_id: str = "default" + + +class MemorizeResponse(BaseModel): + status: str + items_created: int + categories: int + + +class RecallResponse(BaseModel): + query: str + memories_found: int + memories: list[dict] + + +# API Endpoints +@app.get("/") +async def root(): + """Service information and available endpoints.""" + return { + "service": "MemU Assistant", + "description": "AI Assistant with Long-Term Memory", + "version": "1.0.0", + "status": "running", + "endpoints": { + "GET /": "This info", + "GET /health": "Health check", + "POST /chat": "Chat with memory-aware AI", + "POST /memorize": "Store information in memory", + "GET /recall": "Query stored memories", + }, + } + + +@app.get("/health") +async def health(): + """Health check endpoint.""" + return { + "status": "healthy", + "memory_service_initialized": memory_service is not None, + } + + +@app.post("/chat", response_model=ChatResponse) +async def chat(request: ChatRequest): + """ + Chat with the AI assistant. + + The assistant will: + 1. Retrieve relevant memories from past conversations + 2. Use those memories as context for the response + 3. Store new information from the conversation + """ + if not memory_service: + raise HTTPException(status_code=503, detail="Memory service not initialized") + + try: + # Retrieve relevant memories + retrieve_result = await memory_service.retrieve(queries=[{"role": "user", "content": request.message}]) + + memories = retrieve_result.get("items", []) + + # Build context from memories + memory_context = [] + for mem in memories[:5]: + if isinstance(mem, dict): + summary = mem.get("summary", str(mem)) + memory_context.append(summary) + + # Generate response (in production, use full LLM with memory context) + if memory_context: + response_text = ( + f"Based on what I remember about you, here's my response to: '{request.message}'\n\n" + f"Relevant context from our past conversations:\n" + + "\n".join(f"- {ctx[:100]}..." if len(ctx) > 100 else f"- {ctx}" for ctx in memory_context) + ) + else: + response_text = f"I received your message: '{request.message}'. I don't have any relevant memories yet." + + # Store the conversation + import tempfile + + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False, encoding="utf-8") as f: + f.write(f"User ({request.user_id}) said: {request.message}") + temp_file = f.name + + try: + memorize_result = await memory_service.memorize( + resource_url=temp_file, + modality="text", + ) + memories_stored = len(memorize_result.get("items", [])) + finally: + os.unlink(temp_file) + + return ChatResponse( + response=response_text, + memories_used=len(memories), + memories_stored=memories_stored, + ) + + except Exception as e: + raise HTTPException(status_code=500, detail=f"Chat failed: {e!s}") + + +@app.post("/memorize", response_model=MemorizeResponse) +async def memorize(request: MemorizeRequest): + """Store information in long-term memory.""" + if not memory_service: + raise HTTPException(status_code=503, detail="Memory service not initialized") + + try: + import tempfile + + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False, encoding="utf-8") as f: + f.write(f"[User: {request.user_id}] {request.content}") + temp_file = f.name + + try: + result = await memory_service.memorize( + resource_url=temp_file, + modality="text", + ) + return MemorizeResponse( + status="stored", + items_created=len(result.get("items", [])), + categories=len(result.get("categories", [])), + ) + finally: + os.unlink(temp_file) + + except Exception as e: + raise HTTPException(status_code=500, detail=f"Memorize failed: {e!s}") + + +@app.get("/recall", response_model=RecallResponse) +async def recall(query: str, limit: int = 5): + """Recall memories related to a query.""" + if not memory_service: + raise HTTPException(status_code=503, detail="Memory service not initialized") + + try: + result = await memory_service.retrieve(queries=[{"role": "user", "content": query}]) + + items = result.get("items", [])[:limit] + memories = [] + for item in items: + if isinstance(item, dict): + memories.append({ + "summary": item.get("summary", str(item)), + "category": item.get("category", "unknown"), + }) + else: + memories.append({"summary": str(item), "category": "unknown"}) + + return RecallResponse( + query=query, + memories_found=len(memories), + memories=memories, + ) + + except Exception as e: + raise HTTPException(status_code=500, detail=f"Recall failed: {e!s}") + + +if __name__ == "__main__": + import uvicorn + + host = os.getenv("HOST", "0.0.0.0") + port = int(os.getenv("PORT", 8000)) + + print(f"Starting MemU Assistant on {host}:{port}") + uvicorn.run("main:app", host=host, port=port, reload=True) diff --git a/examples/sealos-assistant/requirements.txt b/examples/sealos-assistant/requirements.txt new file mode 100644 index 0000000..0aaff19 --- /dev/null +++ b/examples/sealos-assistant/requirements.txt @@ -0,0 +1,4 @@ +memu>=0.1.0 +fastapi>=0.100.0 +uvicorn[standard]>=0.23.0 +python-dotenv>=1.0.0 diff --git a/examples/sealos_support_agent.py b/examples/sealos_support_agent.py new file mode 100644 index 0000000..710ef00 --- /dev/null +++ b/examples/sealos_support_agent.py @@ -0,0 +1,70 @@ +import sys +import time + +# Intentamos importar la librería instalada por uv +try: + from memu import Memory # noqa: F401 + + MEMU_INSTALLED = True +except ImportError as e: + # Si falla, guardamos el error para debug + MEMU_INSTALLED = False + IMPORT_ERROR = str(e) + + +def print_slow(text, delay=0.02): + """Typing effect for realism""" + for char in text: + sys.stdout.write(char) + sys.stdout.flush() + time.sleep(delay) + print() + + +def run_rigorous_demo(): + print("\n🚀 Starting Sealos Support Agent Demo (Offline Mode)") + print("===================================================\n") + + # 1. ENVIRONMENT CHECK + if MEMU_INSTALLED: + print("✅ Environment Check: MemU Library detected (Installed via uv).") + print("✅ Runtime: Sealos Devbox (Python 3.13+)") + else: + # En caso de error, mostramos advertencia pero permitimos la captura + print("⚠️ Warning: MemU library not detected. Running in Simulation Mode.") + if "IMPORT_ERROR" in globals(): + print(f" Debug Error: {IMPORT_ERROR}") + + time.sleep(0.5) + + # 2. MEMORY INGESTION (PHASE 1) + print("\n📝 --- Phase 1: Ingesting Conversation History ---") + print('👤 Captain: "I\'m getting a 502 Bad Gateway error on port 3000."') + print_slow("🤖 Agent: (Processing input through Memory Pipeline...)", delay=0.01) + + time.sleep(1.0) + print("✅ Memory stored! extracted 2 items:") + print(" - [issue] 502 Bad Gateway error") + print(" - [context] port 3000 configuration") + + # 3. CONTEXT RETRIEVAL (PHASE 2) + print("\n🔍 --- Phase 2: Retrieval on New Interaction (New Session) ---") + print('👤 Captain: "Hello, any updates?"') + print_slow("🤖 Agent: (Searching vector store for user 'Captain'...)", delay=0.01) + + time.sleep(1.0) + print("\n💡 Retrieved Context:") + print(" Found Memory (Score: 0.98): User reported 502 error on port 3000") + print(" Found Memory (Score: 0.95): User was frustrated with timeout") + + # 4. AGENT RESPONSE (PHASE 3) + print("\n💬 --- Phase 3: Agent Response ---") + response = '🤖 Agent: "Welcome back, Captain. Regarding the 502 Bad Gateway error on port 3000 you reported earlier - have you tried checking the firewall logs?"' + print_slow(response) + + print("\n✨ Demo Completed Successfully") + print("===================================================") + + +if __name__ == "__main__": + run_rigorous_demo() diff --git a/examples/test_nebius_provider.py b/examples/test_nebius_provider.py new file mode 100644 index 0000000..5df5e59 --- /dev/null +++ b/examples/test_nebius_provider.py @@ -0,0 +1,228 @@ +""" +Test script to verify Nebius AI works with MemU. + +Before running: +1. Get your Nebius API key from https://tokenfactory.nebius.com +2. Set environment variable: NEBIUS_API_KEY=your_key_here + +Usage: + cd memU + set NEBIUS_API_KEY=your_key_here + python examples/test_nebius_provider.py + +Nebius provides: +- Chat models: Qwen, DeepSeek, Llama, etc. +- Embedding models: BGE, E5, Qwen3-Embedding +- All via OpenAI-compatible API +""" + +import asyncio +import os +import sys + +# Add src to path for local development +src_path = os.path.abspath("src") +sys.path.insert(0, src_path) + +# Nebius configuration +NEBIUS_BASE_URL = "https://api.tokenfactory.nebius.com/v1/" +# Available chat models (pick one): +# - "Qwen/Qwen3-30B-A3B-Instruct-2507" (fast, cheap) +# - "Qwen/Qwen3-32B" (good balance) +# - "deepseek-ai/DeepSeek-V3-0324" (powerful) +# - "meta-llama/Llama-3.3-70B-Instruct" (reliable) +NEBIUS_CHAT_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507" +# Available embedding models: +# - "BAAI/bge-multilingual-gemma2" (3584 dims, multilingual) +# - "BAAI/BGE-ICL" (4096 dims) +# - "intfloat/e5-mistral-7b-instruct" (4096 dims) +# - "Qwen/Qwen3-Embedding-8B" (4096 dims) +NEBIUS_EMBED_MODEL = "BAAI/bge-multilingual-gemma2" + + +async def test_nebius_chat(): + """Test Nebius chat completion.""" + from openai import AsyncOpenAI + + api_key = os.environ.get("NEBIUS_API_KEY") + if not api_key: + print("ERROR: Set NEBIUS_API_KEY environment variable") + return False + + client = AsyncOpenAI( + base_url=NEBIUS_BASE_URL, + api_key=api_key, + ) + + print(f"Testing Nebius Chat API ({NEBIUS_CHAT_MODEL})...") + try: + response = await client.chat.completions.create( + model=NEBIUS_CHAT_MODEL, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Say hello in one sentence."}, + ], + max_tokens=50, + ) + content = response.choices[0].message.content + # Truncate long responses for display + display = content[:100] + "..." if len(content) > 100 else content + print(f" Response: {display}") + print(" ✓ Chat API works!") + return True + except Exception as e: + print(f" ✗ Chat API failed: {e}") + return False + + +async def test_nebius_embeddings(): + """Test Nebius embeddings API.""" + from openai import AsyncOpenAI + + api_key = os.environ.get("NEBIUS_API_KEY") + if not api_key: + print("ERROR: Set NEBIUS_API_KEY environment variable") + return False + + client = AsyncOpenAI( + base_url=NEBIUS_BASE_URL, + api_key=api_key, + ) + + print(f"\nTesting Nebius Embeddings API ({NEBIUS_EMBED_MODEL})...") + try: + response = await client.embeddings.create( + model=NEBIUS_EMBED_MODEL, + input=["Hello world", "This is a test"], + ) + print(f" Embedding dimensions: {len(response.data[0].embedding)}") + print(f" Number of embeddings: {len(response.data)}") + print(" ✓ Embeddings API works!") + return True + except Exception as e: + print(f" ✗ Embeddings API failed: {e}") + return False + + +async def test_memu_with_nebius(): + """Test MemU with Nebius as the LLM provider.""" + from memu.app import MemoryService + + api_key = os.environ.get("NEBIUS_API_KEY") + if not api_key: + print("ERROR: Set NEBIUS_API_KEY environment variable") + return False + + print("\nTesting MemU with Nebius provider...") + + # Configure MemU to use Nebius (using dict config like example_1) + llm_profiles = { + "default": { + "provider": "openai", + "base_url": NEBIUS_BASE_URL, + "api_key": api_key, + "chat_model": NEBIUS_CHAT_MODEL, + "client_backend": "sdk", + }, + "embedding": { + "provider": "openai", + "base_url": NEBIUS_BASE_URL, + "api_key": api_key, + "embed_model": NEBIUS_EMBED_MODEL, + "client_backend": "sdk", + }, + } + + try: + # Create MemU service with Nebius + service = MemoryService(llm_profiles=llm_profiles) + print(" ✓ MemoryService initialized with Nebius!") + + # Test memorize with a file (create temp file) + print("\n Testing memorize...") + import tempfile + + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False, encoding="utf-8") as f: + f.write("User likes Python programming and AI development. They prefer dark mode in their IDE.") + temp_file = f.name + + try: + result = await service.memorize( + resource_url=temp_file, + modality="text", + ) + items_count = len(result.get("items", [])) + categories_count = len(result.get("categories", [])) + print(f" ✓ Memorized! Items: {items_count}, Categories: {categories_count}") + + # Show what was extracted + for item in result.get("items", [])[:3]: + summary = item.get("summary", "")[:80] + print(f" - {summary}...") + finally: + os.unlink(temp_file) + + # Test retrieve + print("\n Testing retrieve...") + retrieve_result = await service.retrieve( + queries=[{"role": "user", "content": "What programming language does the user like?"}] + ) + print(f" ✓ Retrieved! Needs retrieval: {retrieve_result.get('needs_retrieval')}") + + items = retrieve_result.get("items", []) + if items: + print(f" Found {len(items)} relevant items:") + for item in items[:3]: + if isinstance(item, dict): + summary = item.get("summary", str(item))[:60] + else: + summary = str(item)[:60] + print(f" - {summary}...") + + print("\n" + "=" * 60) + print("✓ SUCCESS: MemU works with Nebius!") + print("=" * 60) + return True + + except Exception as e: + print(f" ✗ MemU with Nebius failed: {e}") + import traceback + + traceback.print_exc() + return False + + +async def main(): + print("=" * 60) + print("Nebius AI + MemU Integration Test") + print("=" * 60) + print(f"Base URL: {NEBIUS_BASE_URL}") + print(f"Chat Model: {NEBIUS_CHAT_MODEL}") + print(f"Embed Model: {NEBIUS_EMBED_MODEL}") + print("=" * 60 + "\n") + + # Check for API key first + if not os.environ.get("NEBIUS_API_KEY"): + print("ERROR: NEBIUS_API_KEY environment variable not set!") + print("\nTo get your API key:") + print("1. Go to https://tokenfactory.nebius.com") + print("2. Create an account / Log in") + print("3. Get your API key") + print("4. Run: set NEBIUS_API_KEY=your_key_here") + return + + # Test individual APIs first + chat_ok = await test_nebius_chat() + embed_ok = await test_nebius_embeddings() + + if chat_ok and embed_ok: + # Test full MemU integration + await test_memu_with_nebius() + else: + print("\n" + "=" * 60) + print("✗ FAILED: Basic API tests failed, skipping MemU test") + print("=" * 60) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/npm/README.md b/npm/README.md new file mode 100644 index 0000000..ec8b255 --- /dev/null +++ b/npm/README.md @@ -0,0 +1,32 @@ +# memu-cli + +> 🚧 **Under heavy construction** — memU is undergoing a major rework; commands and flags may change. Expected to stabilize around **July 15, 2026**. + +CLI for [memU](https://github.com/NevaMind-AI/MemU) — personal memory as files: fast retrieval, higher accuracy, lower cost. + +This package is a thin launcher. The engine is the Python package [`memu-py`](https://pypi.org/project/memu-py/) (Python ≥ 3.13); the shim delegates to `uvx`, `pipx run`, or an installed `python3 -m memu`, in that order. Install [uv](https://docs.astral.sh/uv/) for the smoothest zero-setup experience. + +## Usage + +```bash +export OPENAI_API_KEY=sk-... + +# diff-sync a workspace folder (chat/ -> memory, agent/ -> skills, other -> index) +npx memu-cli memorize-workspace ./workspace + +# single-shot embedding retrieval (LLM-free, fast) +npx memu-cli retrieve-workspace "deploy checklist" + +# rebuild the INDEX.md / MEMORY.md / SKILL.md markdown tree +npx memu-cli export + +# legacy pair: memorize a single file / LLM-routed retrieval +npx memu-cli memorize notes/meeting.md +npx memu-cli retrieve "What are this user's launch preferences?" +``` + +State persists in a local SQLite database (`./data/memu.sqlite3` by default), so memorize in one invocation and retrieve in the next. Run `npx memu-cli --help` or see the [main README](https://github.com/NevaMind-AI/MemU#readme) for all flags and `MEMU_*` environment variables. + +## License + +Apache-2.0 diff --git a/npm/bin/memu.js b/npm/bin/memu.js new file mode 100755 index 0000000..23ab440 --- /dev/null +++ b/npm/bin/memu.js @@ -0,0 +1,56 @@ +#!/usr/bin/env node +// memu-cli: thin launcher for the Python package `memu-py`. +// +// The memU engine is Python (>= 3.13). This shim finds a runner and delegates, +// in order of preference: +// 1. $MEMU_PYTHON -m memu (explicit interpreter override) +// 2. uvx --from memu-py memu (no install needed, cached by uv) +// 3. pipx run --spec memu-py memu (no install needed, cached by pipx) +// 4. python3 -m memu (requires `pip install memu-py`) +"use strict"; + +const { spawnSync } = require("node:child_process"); + +const args = process.argv.slice(2); + +function has(cmd) { + const probe = spawnSync(cmd, ["--version"], { stdio: "ignore", shell: false }); + return probe.error === undefined && probe.status === 0; +} + +function run(cmd, prefix) { + const child = spawnSync(cmd, [...prefix, ...args], { stdio: "inherit", shell: false }); + if (child.error) { + console.error(`memu-cli: failed to launch ${cmd}: ${child.error.message}`); + process.exit(1); + } + process.exit(child.status === null ? 1 : child.status); +} + +if (process.env.MEMU_PYTHON) { + run(process.env.MEMU_PYTHON, ["-m", "memu"]); +} +if (has("uvx")) { + run("uvx", ["--from", "memu-py", "memu"]); +} +if (has("pipx")) { + run("pipx", ["run", "--spec", "memu-py", "memu"]); +} +if (has("python3")) { + const probe = spawnSync("python3", ["-c", "import memu"], { stdio: "ignore" }); + if (probe.status === 0) { + run("python3", ["-m", "memu"]); + } +} + +console.error( + [ + "memu-cli: no Python runner found. The memU engine is the Python package `memu-py` (Python >= 3.13).", + "Install one of:", + " - uv (https://docs.astral.sh/uv/) -> memu-cli will use `uvx` automatically", + " - pipx (https://pipx.pypa.io/) -> memu-cli will use `pipx run` automatically", + " - pip -> `pip install memu-py`, then re-run", + "Or point MEMU_PYTHON at an interpreter that has memu-py installed.", + ].join("\n") +); +process.exit(1); diff --git a/npm/package.json b/npm/package.json new file mode 100644 index 0000000..1c050bb --- /dev/null +++ b/npm/package.json @@ -0,0 +1,33 @@ +{ + "name": "memu-cli", + "version": "1.5.1", + "description": "CLI for memU \u2014 personal memory as files. Thin launcher for the Python package memu-py.", + "bin": { + "memu": "bin/memu.js" + }, + "files": [ + "bin" + ], + "engines": { + "node": ">=18" + }, + "keywords": [ + "memu", + "memory", + "agent", + "context-engineering", + "retrieval", + "llm", + "cli" + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/NevaMind-AI/MemU.git", + "directory": "npm" + }, + "homepage": "https://github.com/NevaMind-AI/MemU#readme", + "bugs": { + "url": "https://github.com/NevaMind-AI/MemU/issues" + } +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3c2e700 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,182 @@ +[project] +name = "memu-py" +version = "1.5.1" +authors = [ + {name = "MemU Team", email = "contact@nevamind.ai"}, +] +description = "Personal memory as files — fast retrieval, higher accuracy, lower cost." +readme = "README.md" +# license = {file = "LICENSE"} +requires-python = ">=3.13" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.13", +] +keywords = ["agent", "agentic", "agent-harness", "harness", "loop-engineering", "context-engineering", "context-window", "memory", "personal-information", "workspace", "retrieval", "llm"] +dependencies = [ + "anthropic>=0.40.0", + "defusedxml>=0.7.1", + "httpx>=0.28.1", + "numpy>=2.3.4", + "openai>=2.8.0", + "pydantic>=2.12.4", + "sqlmodel>=0.0.27", + "alembic>=1.14.0", + "pendulum>=3.1.0", + "langchain-core>=1.2.7", +] + +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[tool.maturin] +module-name = "memu._core" +python-packages = ["memu"] +python-source = "src" + +[dependency-groups] +dev = [ + {include-group = "docs"}, + {include-group = "lint"}, + {include-group = "test"}, + "langchain-openai>=1.1.7", + "langgraph>=1.0.6", +] +docs = [ + "mkdocs>=1.6.1", + "mkdocs-material>=9.6.23", + "mkdocstrings>=0.30.1", + "mkdocstrings-python>=1.18.2", +] +lint = [ + "deptry>=0.23.1", + "mypy>=1.18.2", + "pre-commit>=4.3.0", + "ruff>=0.14.3", + "types-defusedxml>=0.7.0", +] +test = [ + "pytest>=8.4.2", + "pytest-asyncio>=0.24.0", + "pytest-cov>=7.0.0", +] + +[project.optional-dependencies] +postgres = ["pgvector>=0.3.4", "sqlalchemy[postgresql-psycopgbinary]>=2.0.36"] +langgraph = ["langgraph>=0.0.10", "langchain-core>=0.1.0"] +claude = ["claude-agent-sdk>=0.1.24"] +lazyllm = ["lazyllm>=0.7.3"] +# Rich document ingestion (PDF, Word, PowerPoint, Excel, ...) via MarkItDown. +document = ["markitdown[docx,pptx,xlsx,xls,pdf]>=0.1.0"] + +[project.scripts] +memu = "memu.cli:main" + +[project.urls] +"Homepage" = "https://github.com/NevaMind-AI/MemU" +"Bug Tracker" = "https://github.com/NevaMind-AI/MemU/issues" +"Documentation" = "https://github.com/NevaMind-AI/MemU#readme" + +[tool.deptry.per_rule_ignores] +# Optional dependencies used in examples/ or imported lazily behind extras. +DEP002 = ["claude-agent-sdk", "markitdown"] + +[tool.mypy] +files = ["src", "tests"] +python_version = "3.13" +disallow_untyped_defs = true +disallow_any_unimported = true +no_implicit_optional = true +check_untyped_defs = true +warn_return_any = true +warn_unused_ignores = true +show_error_codes = true + +[[tool.mypy.overrides]] +module = ["tests.*"] +disallow_untyped_defs = false +disallow_incomplete_defs = false +warn_unused_ignores = false +disable_error_code = ["attr-defined", "call-arg"] + +[[tool.mypy.overrides]] +module = ["lazyllm.*"] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = ["pgvector.*"] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = ["markitdown", "markitdown.*"] +ignore_missing_imports = true + +[tool.ruff] +target-version = "py313" +line-length = 120 +fix = true + +[tool.ruff.lint] +select = [ + # flake8-2020 + "YTT", + # flake8-bandit + "S", + # flake8-bugbear + "B", + # flake8-builtins + "A", + # flake8-comprehensions + "C4", + # flake8-debugger + "T10", + # flake8-simplify + "SIM", + # isort + "I", + # mccabe + "C90", + # pycodestyle + "E", "W", + # pyflakes + "F", + # pygrep-hooks + "PGH", + # pyupgrade + "UP", + # ruff + "RUF", + # tryceratops + "TRY", +] +ignore = [ + # LineTooLong + "E501", + # DoNotAssignLambda + "E731", +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["S101"] +"examples/*" = ["E402", "TRY003", "B904", "S104", "TRY300", "SIM108"] + +[tool.ruff.format] +preview = true + +[tool.coverage.report] +skip_empty = true + +[tool.coverage.run] +branch = true +source = ["memu"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +log_cli = true +log_cli_level = "INFO" +asyncio_mode = "auto" diff --git a/readme/README_en.md b/readme/README_en.md new file mode 100644 index 0000000..99cbe52 --- /dev/null +++ b/readme/README_en.md @@ -0,0 +1,542 @@ +![MemU Banner](../assets/banner.png) + +
+ +# memU + +### Personal memory, stored as files + +**Fast retrieval. Higher accuracy. Lower cost.** + +[![PyPI version](https://badge.fury.io/py/memu-py.svg)](https://badge.fury.io/py/memu-py) +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) +[![Discord](https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white)](https://discord.com/invite/hQZntfGsbJ) +[![Twitter](https://img.shields.io/badge/Twitter-Follow-1DA1F2?logo=x&logoColor=white)](https://x.com/memU_ai) + +NevaMind-AI%2FmemU | Trendshift + +**[English](README_en.md) | [中文](README_zh.md) | [日本語](README_ja.md) | [한국어](README_ko.md) | [Español](README_es.md) | [Français](README_fr.md)** + +
+ +--- + +> [!WARNING] +> 🚧 **Under heavy construction** — memU is undergoing a major rework. APIs, CLI commands, and docs may change without notice. Things are expected to stabilize around **July 15, 2026**. + +memU compiles conversations, documents, code, images, audio, video, URLs, and tool traces into human-readable Markdown files (`INDEX.md`, `MEMORY.md`, `SKILL.md`). Agents traverse the tree and load only what the moment needs — instead of rescanning everything or stuffing long histories into every prompt: + +- **`INDEX.md`** — a map of everything: categories, files, and summaries, so the agent knows where to look first +- **`MEMORY.md`** — profile, preferences, goals, facts, and key events extracted from source data +- **`SKILL.md`** — learned tool patterns and workflows, auto-extracted from tool traces and refined on every `memorize()` + +```txt +workspace/ +├── INDEX.md ← map of everything: categories, files, and summaries +├── MEMORY.md ← profile, preferences, goals, and key events +└── skill/ + ├── {skill_name}/ + │ └── SKILL.md ← a learned skill or tool pattern + └── {another_skill}/ + └── SKILL.md +``` + +Three things make it different from stuffing everything into the prompt: + +- **Fast retrieval** — walk to the right folder and rank the right files instead of scanning everything every time. +- **Higher accuracy** — scope by user, task, or session, and trace every item back to the exact source it came from. +- **Lower cost** — retrieve compact, scoped context instead of reinjecting long histories into every prompt. +- **Yours to inspect** — a human-readable file tree you can audit, edit, and route through your own storage and LLM providers. + +--- + +## 🔄 How It Works + +Think of it as two file-system operations: **writing** raw sources into organized memory, and **reading** the right files back into the agent. + +``` +WRITE — memorize() READ — retrieve() +────────────────────────────────────────────── ────────────────────────────────────────────── +raw files → extract → files + folders query → walk folders → ranked files +───────────── ───────── ────────────── ───── ──────────── ───────────── +chat logs → parse → profile / event items user / task query +documents / URLs → facts → knowledge / skill items │ +images / video → caption → resources + summaries ├─ route + scope → relevant folders (categories) +audio → transcribe→ event / knowledge items ├─ rank by relevance → matching files (items) +tool logs → mine → tool / skill items └─ trace to source → original resources +``` + +**Writing to the file system (`memorize`)** + +1. **Ingest** — store each source as a `Resource` (the raw file) with its modality and source location +2. **Preprocess** — parse text, caption images/video, transcribe audio, and normalize inputs +3. **Extract** — turn raw content into typed `RecallEntry` records (the files): profile, event, knowledge, behavior, skill, or tool memories +4. **Organize** — sort entries into `RecallFile` folders (a memory category, or a skill), cross-link, embed, and summarize into a browsable tree +5. **Persist** — write records, relations, embeddings, and folder summaries through the configured backend + +**Reading from the file system (`retrieve`)** + +6. **Retrieve** — navigate the folders and return only the files relevant to the current user, agent, session, or task + +--- + +## 🗂️ The Memory File System + +memU's primary output is a navigable memory tree — folders, files, and the source artifacts behind them — persisted through repository contracts and returned as dictionaries from `memorize()` and `retrieve()`. + +```txt +RecallFile ← folder: a topic with an evolving summary +├── name, track (memory | skill), description, content +├── embedding +└── RecallEntry[] ← files: typed, atomic memories + ├── memory_type: profile | event | knowledge | behavior | skill | tool + ├── summary, extra, happened_at, embedding + └── Resource ← source: the raw file this memory came from + └── url, modality, local_path, caption, embedding +``` + +| Record | File-System Role | Used By | +|--------|------------------|---------| +| `RecallFile` | **Folder** — groups related memories (or holds a skill) and keeps a topic-level summary; a `track` field marks it `memory` or `skill` | Load compact context for broad queries | +| `RecallEntry` | **File** — a typed atomic memory with a summary and optional metadata | Inject precise facts, preferences, events, skills, and tool patterns | +| `Resource` | **Source artifact** — the original file behind a memory, with caption/text | Trace context back to where it came from | +| `RecallFileEntry` | **Link** — the edge that files an entry under a folder | Navigate related memories without reprocessing the source | + +This gives agents a stable file system for memory: ingest raw sources once, then request scoped and ranked files instead of rereading every source artifact. + +--- + +## 🧩 What memU Builds + +Every layer of the file system is stored as a structured record: + +| Layer | What It Represents | Why Agents Use It | +|-------|--------------------|-------------------| +| **RecallFile** | Auto-generated folder: a topic (or a skill) with an evolving summary | Load high-level context before drilling into details | +| **RecallEntry** | A file: atomic structured memory with a type and summary | Inject precise facts, preferences, events, skills, and tool patterns | +| **Resource** | Source artifact behind a file: conversation, document, image, video, audio, URL, or file | Trace memory back to its source | +| **RecallFileEntry** | The link that files an entry under a folder | Navigate related memories without reprocessing the source | +| **Embedding** | Vector index over folders, files, and sources | Retrieve relevant context with low latency | + +Example `memorize()` output: + +```json +{ + "resource": { + "id": "res_01", + "url": "files/launch-meeting.mp4", + "modality": "video", + "caption": "A product planning discussion about onboarding and launch risks." + }, + "items": [ + { + "id": "mem_01", + "memory_type": "event", + "summary": "The team decided to simplify onboarding before the next launch review." + }, + { + "id": "mem_02", + "memory_type": "profile", + "summary": "The user prefers concise implementation plans with explicit verification steps." + }, + { + "id": "mem_03", + "memory_type": "tool", + "summary": "Use repository-wide search before editing configuration files to avoid missing duplicated settings." + } + ], + "categories": [ + { + "id": "cat_01", + "name": "product_goals", + "summary": "Current launch priorities, onboarding decisions, and unresolved risks." + } + ], + "relations": [ + { "item_id": "mem_01", "category_id": "cat_01" } + ] +} +``` + +Then an agent can call `retrieve()` to get a scoped, ranked context payload: + +```python +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What context matters for this launch task?"}}], + where={"user_id": "123"}, +) +``` + +--- + +## ⭐️ Star the repository + + + +If you find memU useful or interesting, a GitHub Star ⭐️ would be greatly appreciated. + +--- + +## ✨ Core Features + +| Capability | Description | +|------------|-------------| +| 🗂️ **Multimodal Ingestion** | Write conversations, documents, images, video, audio, URLs, logs, and local files into memory | +| 📁 **Memory File System** | Persist folders (categories), files (items), source artifacts, links, summaries, and embeddings | +| 🧠 **Typed Memory Extraction** | Extract profile, event, knowledge, behavior, skill, and tool memories from raw sources | +| 🧭 **Self-Organizing Folders** | Auto-build categories, links, summaries, and embeddings without manual tagging | +| 🤖 **Agent-Ready Retrieval** | Read scoped, ranked context that can be injected into any agent workflow | +| 🧱 **Pluggable Storage** | Use in-memory, SQLite, or Postgres backends with the same repository contracts | +| 🔀 **Profile-Based LLM Routing** | Route chat, embedding, vision, and transcription work through configurable LLM profiles | + +--- + +## 🎯 Use Cases + +### 1. **Conversation Memory** +*Turn chat logs into user preferences, goals, events, and relationship context.* + +```python +await service.memorize( + resource_url="examples/resources/conversations/conv1.json", + modality="conversation", + user={"user_id": "123"}, +) + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What should I remember about this user?"}}], + where={"user_id": "123"}, +) +``` + +### 2. **Workspace Context for Coding Agents** +*Convert docs, PR notes, logs, and design decisions into reusable project memory.* + +```python +await service.memorize(resource_url="docs/architecture.md", modality="document") +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "How should I structure this module?"}}], +) +``` + +### 3. **Multimodal Knowledge Layer** +*Extract searchable facts from documents, screenshots, images, videos, and audio notes.* + +```python +await service.memorize(resource_url="examples/resources/docs/doc1.txt", modality="document") +await service.memorize(resource_url="examples/resources/images/image1.png", modality="image") +# Audio is supported for your own .mp3/.wav/.m4a files. +await service.memorize(resource_url="meeting-audio.mp3", modality="audio") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What matters for the next research plan?"}}], +) +``` + +### 4. **Tool and Agent Learning** +*Turn execution traces into tool memories that tell future agents when to use a tool and what mistakes to avoid.* + +```python +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "Which tools worked for config editing?"}}], +) +``` + +--- + +## 🗂️ Architecture + +The memory file system is hierarchical enough for browsing and structured enough for direct retrieval: + +structure + +| Layer | Primary Role | Retrieval Role | +|-------|--------------|----------------| +| **Category (folder)** | Maintain topic-level summaries | Assemble compact context for broad queries | +| **Item (file)** | Store typed atomic memories | Load precise facts, events, preferences, skills, and tool patterns | +| **Resource (source)** | Preserve source artifacts and captions | Recall original context when item/category summaries are not enough | + +See [docs/architecture.md](../docs/architecture.md) for the runtime view of `MemoryService`, workflow pipelines, storage backends, and LLM routing. + +--- + +## 🚀 Quick Start + +### Option 1: Cloud Version + +👉 **[memu.so](https://memu.so)** — Hosted API for managed ingestion, structured memory, and retrieval + +For enterprise deployment: **info@nevamind.ai** + +#### Cloud API (v3) + +| Base URL | `https://api.memu.so` | +|----------|----------------------| +| Auth | `Authorization: Bearer ` | + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/api/v3/memory/memorize` | Ingest raw data and build structured memory | +| `GET` | `/api/v3/memory/memorize/status/{task_id}` | Check processing status | +| `POST` | `/api/v3/memory/categories` | List auto-generated categories | +| `POST` | `/api/v3/memory/retrieve` | Query memory for agent context | + +📚 **[Full API Documentation](https://memu.pro/docs#cloud-version)** + +--- + +### Option 2: Self-Hosted + +#### Installation + +From a clone of this repository: + +```bash +uv sync +# or, for the full development setup: +make install +``` + +To install the published package instead: + +```bash +pip install memu-py +``` + +> **Requirements**: Python 3.13+. The default examples use OpenAI, so set `OPENAI_API_KEY` or pass another provider through `llm_profiles`. + +**Run an in-memory smoke script:** +```bash +export OPENAI_API_KEY=your_key +cd tests +uv run python test_inmemory.py +``` + +**Run with PostgreSQL + pgvector:** +```bash +uv sync --extra postgres +docker run -d --name memu-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=memu \ + -p 5432:5432 \ + pgvector/pgvector:pg16 + +export OPENAI_API_KEY=your_key +export POSTGRES_DSN=postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu +cd tests +uv run python test_postgres.py +``` + +--- + +### Custom LLM and Embedding Providers + +```python +from memu import MemUService + +service = MemUService( + llm_profiles={ + "default": { + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "api_key": "your_key", + "chat_model": "qwen3-max", + "client_backend": "sdk" + }, + "embedding": { + "base_url": "https://api.voyageai.com/v1", + "api_key": "your_key", + "embed_model": "voyage-3.5-lite" + } + }, +) +``` + +--- + +### OpenRouter Integration + +```python +from memu import MemoryService + +service = MemoryService( + llm_profiles={ + "default": { + "provider": "openrouter", + "client_backend": "httpx", + "base_url": "https://openrouter.ai", + "api_key": "your_key", + "chat_model": "anthropic/claude-3.5-sonnet", + "embed_model": "openai/text-embedding-3-small", + }, + }, + database_config={"metadata_store": {"provider": "inmemory"}}, +) +``` + +--- + +## 📖 Core APIs + +### `memorize()` — Structure Raw Data + +memorize + +```python +result = await service.memorize( + resource_url="path/to/file.json", # local file path or HTTP URL + modality="conversation", # conversation | document | image | video | audio + user={"user_id": "123"}, # optional: scope to a user or agent +) +# Returns after processing completes: +# { "resource": {...}, "items": [...], "categories": [...], "relations": [...] } +``` + +- Converts raw input into typed memory items +- Categorizes and embeds items without manual tagging +- Preserves source resources and item-category relations + +--- + +### `retrieve()` — Load Agent Context + +retrieve + +```python +# The retrieval strategy is set once on the service via retrieve_config: +# MemoryService(retrieve_config={"method": "rag"}) # vector-first recall +# MemoryService(retrieve_config={"method": "llm"}) # LLM-ranked recall +result = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What are their preferences?"}}], + where={"user_id": "123"}, # scope filter +) +# Returns: +# { +# "needs_retrieval": true, +# "original_query": "...", +# "rewritten_query": "...", +# "next_step_query": "...", +# "categories": [...], +# "items": [...], +# "resources": [...] +# } +``` + +| `retrieve_config.method` | Behavior | Cost | Best For | +|--------------------------|----------|------|----------| +| `rag` | Vector-first category/item/resource recall, with optional LLM routing and sufficiency checks enabled by default | Embeddings plus LLM calls unless `route_intention` and `sufficiency_check` are disabled | Fast scoped recall with controllable reasoning | +| `llm` | LLM-ranked category/item/resource recall | LLM ranking at each tier | Deeper semantic ranking | + +--- + +## 💡 Example Workflows + +### Always-Learning Assistant +```bash +export OPENAI_API_KEY=your_key +uv run python examples/example_1_conversation_memory.py +``` +Automatically extracts preferences, builds relationship models, and surfaces relevant context in future conversations. + +### Self-Improving Agent +```bash +uv run python examples/example_2_skill_extraction.py +``` +Monitors agent actions, identifies patterns in successes and failures, auto-generates skill guides from experience. + +### Multimodal Context Builder +```bash +uv run python examples/example_3_multimodal_memory.py +``` +Cross-references text, images, and documents automatically into a unified memory layer. + +--- + +## 📊 Performance + +memU achieves **92.09% average accuracy** on the Locomo benchmark across all reasoning tasks. + +benchmark + +View detailed results: [memU-experiment](https://github.com/NevaMind-AI/memU-experiment) + +--- + +## 🧩 Ecosystem + +| Repository | Description | +|------------|-------------| +| **[memU](https://github.com/NevaMind-AI/memU)** | Core memory file system — ingestion, extraction, retrieval | +| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | Backend with real-time sync and webhook triggers | +| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | Visual dashboard for browsing and monitoring memory | + +**Quick Links:** +- 🚀 [Try MemU Cloud](https://app.memu.so/quick-start) +- 📚 [API Documentation](https://memu.pro/docs) +- 💬 [Discord Community](https://discord.com/invite/hQZntfGsbJ) + +--- + +## 🤝 Partners + +
+ +Ten +OpenAgents +Milvus +xRoute +Jazz +Buddie +Bytebase +LazyLLM +Clawdchat + +
+ +--- + +## 🤝 Contributing + +```bash +# Fork and clone +git clone https://github.com/YOUR_USERNAME/memU.git +cd memU + +# Install dev dependencies +make install + +# Run quality checks before submitting +make check +``` + +See [CONTRIBUTING.md](../CONTRIBUTING.md) for full guidelines. + +**Prerequisites:** Python 3.13+, [uv](https://github.com/astral-sh/uv), Git + +--- + +## 📄 License + +[Apache License 2.0](../LICENSE.txt) + +--- + +## 🌍 Community + +- **GitHub Issues**: [Report bugs & request features](https://github.com/NevaMind-AI/memU/issues) +- **Discord**: [Join the community](https://discord.com/invite/hQZntfGsbJ) +- **X (Twitter)**: [Follow @memU_ai](https://x.com/memU_ai) +- **Contact**: info@nevamind.ai + +--- + +
+ +⭐ **Star us on GitHub** to get notified about new releases! + +
diff --git a/readme/README_es.md b/readme/README_es.md new file mode 100644 index 0000000..947db04 --- /dev/null +++ b/readme/README_es.md @@ -0,0 +1,542 @@ +![MemU Banner](../assets/banner.png) + +
+ +# memU + +### Memoria personal en archivos + +**Recuperación rápida · Mayor precisión · Menor costo** + +[![PyPI version](https://badge.fury.io/py/memu-py.svg)](https://badge.fury.io/py/memu-py) +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) +[![Discord](https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white)](https://discord.com/invite/hQZntfGsbJ) +[![Twitter](https://img.shields.io/badge/Twitter-Follow-1DA1F2?logo=x&logoColor=white)](https://x.com/memU_ai) + +NevaMind-AI%2FmemU | Trendshift + +**[English](README_en.md) | [中文](README_zh.md) | [日本語](README_ja.md) | [한국어](README_ko.md) | [Español](README_es.md) | [Français](README_fr.md)** + +
+ +--- + +> [!WARNING] +> 🚧 **En plena reconstrucción** — memU está en pleno rediseño; las APIs, los comandos de la CLI y la documentación pueden cambiar sin previo aviso. Se espera que se estabilice alrededor del **15 de julio de 2026**. + +memU compila conversaciones, documentos, código, imágenes, audio, video, URLs y trazas de herramientas en archivos Markdown legibles por humanos (`INDEX.md`, `MEMORY.md`, `SKILL.md`). El agente recorre el árbol y carga solo la memoria que necesita en el momento — sin escanear todo ni meter historiales largos en cada prompt: + +- **`INDEX.md`** — mapa de todo: categorías, archivos y resúmenes, para que el agente sepa dónde mirar primero +- **`MEMORY.md`** — perfil, preferencias, objetivos, hechos y eventos clave extraídos de los datos fuente +- **`SKILL.md`** — patrones de herramientas y flujos de trabajo reutilizables, extraídos automáticamente de trazas y refinados en cada `memorize()` + +```txt +workspace/ +├── INDEX.md ← mapa de todo: categorías, archivos y resúmenes +├── MEMORY.md ← perfil, preferencias, objetivos y eventos clave +└── skill/ + ├── {skill_name}/ + │ └── SKILL.md ← una habilidad o patrón de herramienta aprendido + └── {another_skill}/ + └── SKILL.md +``` + +Tres cosas lo diferencian de meter todo en el prompt: + +- **Recuperación rápida** — ir a la carpeta relevante y ordenar archivos en lugar de escanear todo cada vez. +- **Mayor precisión** — acotar por usuario, tarea o sesión, y rastrear cada elemento hasta su fuente original. +- **Menor costo** — recuperar contexto compacto y acotado en lugar de reinyectar historiales largos en cada prompt. +- **Auditable** — un árbol de archivos legible por humanos que puedes revisar, editar y enrutar con tu propio almacenamiento y proveedores de LLM. + +--- + +## 🔄 Cómo funciona + +Piénsalo como dos operaciones de sistema de archivos: **escribir** fuentes en bruto en una memoria organizada y **leer** los archivos correctos de vuelta hacia el agente. + +``` +WRITE — memorize() READ — retrieve() +────────────────────────────────────────────── ────────────────────────────────────────────── +raw files → extract → files + folders query → walk folders → ranked files +───────────── ───────── ────────────── ───── ──────────── ───────────── +chat logs → parse → profile / event items user / task query +documents / URLs → facts → knowledge / skill items │ +images / video → caption → resources + summaries ├─ route + scope → relevant folders (categories) +audio → transcribe→ event / knowledge items ├─ rank by relevance → matching files (items) +tool logs → mine → tool / skill items └─ trace to source → original resources +``` + +**Escribir en el sistema de archivos (`memorize`)** + +1. **Ingerir (Ingest)**: almacena cada fuente como un `Resource` (el archivo en bruto) con su modalidad y ubicación de origen +2. **Preprocesar (Preprocess)**: analiza texto, genera descripciones de imágenes/video, transcribe audio y normaliza las entradas +3. **Extraer (Extract)**: convierte el contenido en bruto en registros `RecallEntry` tipados (los archivos): memorias de tipo profile, event, knowledge, behavior, skill o tool +4. **Organizar (Organize)**: clasifica los elementos en carpetas `RecallFile` (una categoría de memoria, o una habilidad), los enlaza entre sí, los vectoriza y los resume en un árbol navegable +5. **Persistir (Persist)**: escribe registros, relaciones, embeddings y resúmenes de carpeta a través del backend configurado + +**Leer del sistema de archivos (`retrieve`)** + +6. **Recuperar (Retrieve)**: navega por las carpetas y devuelve solo los archivos relevantes para el usuario, agente, sesión o tarea actuales + +--- + +## 🗂️ El sistema de archivos de memoria + +La salida principal de memU es un árbol de memoria navegable —carpetas, archivos y los artefactos de origen detrás de ellos— persistido mediante contratos de repositorio y devuelto como diccionarios desde `memorize()` y `retrieve()`. + +```txt +RecallFile ← carpeta: un tema con un resumen en evolución +├── name, track (memory | skill), description, content +├── embedding +└── RecallEntry[] ← archivos: memorias atómicas y tipadas + ├── memory_type: profile | event | knowledge | behavior | skill | tool + ├── summary, extra, happened_at, embedding + └── Resource ← fuente: el archivo en bruto del que proviene esta memoria + └── url, modality, local_path, caption, embedding +``` + +| Registro | Rol en el sistema de archivos | Usado para | +|--------|------------------|---------| +| `RecallFile` | **Carpeta**: agrupa memorias relacionadas (o contiene una habilidad) y mantiene un resumen a nivel de tema; un campo `track` lo marca como `memory` o `skill` | Cargar contexto compacto para consultas amplias | +| `RecallEntry` | **Archivo**: memoria atómica tipada con un resumen y metadatos opcionales | Inyectar hechos, preferencias, eventos, habilidades y patrones de herramientas precisos | +| `Resource` | **Artefacto de origen**: el archivo original detrás de una memoria, con descripción/texto | Rastrear el contexto hasta su origen | +| `RecallFileEntry` | **Enlace**: la arista que archiva un elemento bajo una carpeta | Navegar memorias relacionadas sin reprocesar la fuente | + +Esto da a los agentes un sistema de archivos de memoria estable: ingieren las fuentes en bruto una sola vez y luego solicitan archivos delimitados y ordenados, en lugar de releer cada artefacto de origen. + +--- + +## 🧩 Qué construye memU + +Cada capa del sistema de archivos se almacena como un registro estructurado: + +| Capa | Qué representa | Por qué la usan los agentes | +|-------|--------------------|-------------------| +| **RecallFile** | Carpeta autogenerada: un tema (o una habilidad) con un resumen en evolución | Cargar contexto de alto nivel antes de profundizar en detalles | +| **RecallEntry** | Un archivo: memoria estructurada atómica con un tipo y un resumen | Inyectar hechos, preferencias, eventos, habilidades y patrones de herramientas precisos | +| **Resource** | Artefacto de origen detrás de un archivo: conversación, documento, imagen, video, audio, URL o archivo | Rastrear la memoria hasta su origen | +| **RecallFileEntry** | El enlace que archiva un elemento bajo una carpeta | Navegar memorias relacionadas sin reprocesar la fuente | +| **Embedding** | Índice vectorial sobre carpetas, archivos y fuentes | Recuperar contexto relevante con baja latencia | + +Ejemplo de salida de `memorize()`: + +```json +{ + "resource": { + "id": "res_01", + "url": "files/launch-meeting.mp4", + "modality": "video", + "caption": "A product planning discussion about onboarding and launch risks." + }, + "items": [ + { + "id": "mem_01", + "memory_type": "event", + "summary": "The team decided to simplify onboarding before the next launch review." + }, + { + "id": "mem_02", + "memory_type": "profile", + "summary": "The user prefers concise implementation plans with explicit verification steps." + }, + { + "id": "mem_03", + "memory_type": "tool", + "summary": "Use repository-wide search before editing configuration files to avoid missing duplicated settings." + } + ], + "categories": [ + { + "id": "cat_01", + "name": "product_goals", + "summary": "Current launch priorities, onboarding decisions, and unresolved risks." + } + ], + "relations": [ + { "item_id": "mem_01", "category_id": "cat_01" } + ] +} +``` + +Luego un agente puede llamar a `retrieve()` para obtener una carga de contexto delimitada y ordenada por relevancia: + +```python +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What context matters for this launch task?"}}], + where={"user_id": "123"}, +) +``` + +--- + +## ⭐️ Dale una estrella al repositorio + + + +Si encuentras memU útil o interesante, una estrella ⭐️ en GitHub sería muy apreciada. + +--- + +## ✨ Funciones principales + +| Capacidad | Descripción | +|------------|-------------| +| 🗂️ **Ingesta multimodal** | Escribe conversaciones, documentos, imágenes, video, audio, URLs, registros y archivos locales en la memoria | +| 📁 **Sistema de archivos de memoria** | Persiste carpetas (categorías), archivos (elementos), artefactos de origen, enlaces, resúmenes y embeddings | +| 🧠 **Extracción de memoria tipada** | Extrae memorias profile, event, knowledge, behavior, skill y tool a partir de fuentes en bruto | +| 🧭 **Carpetas autoorganizadas** | Construye automáticamente categorías, enlaces, resúmenes y embeddings sin etiquetado manual | +| 🤖 **Recuperación lista para agentes** | Lee contexto delimitado y ordenado que puede inyectarse en cualquier flujo de trabajo de agente | +| 🧱 **Almacenamiento conectable** | Usa backends in-memory, SQLite o Postgres con los mismos contratos de repositorio | +| 🔀 **Enrutamiento de LLM basado en perfiles** | Enruta tareas de chat, embeddings, visión y transcripción a través de perfiles de LLM configurables | + +--- + +## 🎯 Casos de uso + +### 1. **Memoria de conversación** +*Convierte registros de chat en preferencias, objetivos, eventos y contexto de relación del usuario.* + +```python +await service.memorize( + resource_url="examples/resources/conversations/conv1.json", + modality="conversation", + user={"user_id": "123"}, +) + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What should I remember about this user?"}}], + where={"user_id": "123"}, +) +``` + +### 2. **Contexto de espacio de trabajo para agentes de programación** +*Convierte documentos, notas de PR, registros y decisiones de diseño en memoria de proyecto reutilizable.* + +```python +await service.memorize(resource_url="docs/architecture.md", modality="document") +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "How should I structure this module?"}}], +) +``` + +### 3. **Capa de conocimiento multimodal** +*Extrae hechos buscables de documentos, capturas de pantalla, imágenes, videos y notas de audio.* + +```python +await service.memorize(resource_url="examples/resources/docs/doc1.txt", modality="document") +await service.memorize(resource_url="examples/resources/images/image1.png", modality="image") +# Audio is supported for your own .mp3/.wav/.m4a files. +await service.memorize(resource_url="meeting-audio.mp3", modality="audio") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What matters for the next research plan?"}}], +) +``` + +### 4. **Aprendizaje de herramientas y agentes** +*Convierte las trazas de ejecución en memorias de herramientas que indican a los agentes futuros cuándo usar una herramienta y qué errores evitar.* + +```python +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "Which tools worked for config editing?"}}], +) +``` + +--- + +## 🗂️ Arquitectura + +El sistema de archivos de memoria es lo bastante jerárquico para navegarlo y lo bastante estructurado para una recuperación directa: + +structure + +| Capa | Rol principal | Rol en la recuperación | +|-------|--------------|----------------| +| **Category (carpeta)** | Mantener resúmenes a nivel de tema | Ensamblar contexto compacto para consultas amplias | +| **Item (archivo)** | Almacenar memorias atómicas tipadas | Cargar hechos, eventos, preferencias, habilidades y patrones de herramientas precisos | +| **Resource (fuente)** | Preservar artefactos de origen y descripciones | Recuperar el contexto original cuando los resúmenes de elemento/categoría no bastan | + +Consulta [docs/architecture.md](../docs/architecture.md) para la vista en tiempo de ejecución de `MemoryService`, los pipelines de flujo de trabajo, los backends de almacenamiento y el enrutamiento de LLM. + +--- + +## 🚀 Inicio rápido + +### Opción 1: Versión en la nube + +👉 **[memu.so](https://memu.so)**: API alojada para ingesta gestionada, memoria estructurada y recuperación + +Para despliegue empresarial: **info@nevamind.ai** + +#### Cloud API (v3) + +| Base URL | `https://api.memu.so` | +|----------|----------------------| +| Auth | `Authorization: Bearer ` | + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/api/v3/memory/memorize` | Ingiere datos en bruto y construye memoria estructurada | +| `GET` | `/api/v3/memory/memorize/status/{task_id}` | Consulta el estado del procesamiento | +| `POST` | `/api/v3/memory/categories` | Lista las categorías autogeneradas | +| `POST` | `/api/v3/memory/retrieve` | Consulta la memoria para obtener contexto del agente | + +📚 **[Documentación completa de la API](https://memu.pro/docs#cloud-version)** + +--- + +### Opción 2: Autoalojado + +#### Instalación + +Desde un clon de este repositorio: + +```bash +uv sync +# o, para la configuración completa de desarrollo: +make install +``` + +Para instalar el paquete publicado en su lugar: + +```bash +pip install memu-py +``` + +> **Requisitos**: Python 3.13+. Los ejemplos por defecto usan OpenAI, así que define `OPENAI_API_KEY` o pasa otro proveedor mediante `llm_profiles`. + +**Ejecuta un script de prueba en memoria:** +```bash +export OPENAI_API_KEY=your_key +cd tests +uv run python test_inmemory.py +``` + +**Ejecuta con PostgreSQL + pgvector:** +```bash +uv sync --extra postgres +docker run -d --name memu-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=memu \ + -p 5432:5432 \ + pgvector/pgvector:pg16 + +export OPENAI_API_KEY=your_key +export POSTGRES_DSN=postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu +cd tests +uv run python test_postgres.py +``` + +--- + +### Proveedores personalizados de LLM y embeddings + +```python +from memu import MemUService + +service = MemUService( + llm_profiles={ + "default": { + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "api_key": "your_key", + "chat_model": "qwen3-max", + "client_backend": "sdk" + }, + "embedding": { + "base_url": "https://api.voyageai.com/v1", + "api_key": "your_key", + "embed_model": "voyage-3.5-lite" + } + }, +) +``` + +--- + +### Integración con OpenRouter + +```python +from memu import MemoryService + +service = MemoryService( + llm_profiles={ + "default": { + "provider": "openrouter", + "client_backend": "httpx", + "base_url": "https://openrouter.ai", + "api_key": "your_key", + "chat_model": "anthropic/claude-3.5-sonnet", + "embed_model": "openai/text-embedding-3-small", + }, + }, + database_config={"metadata_store": {"provider": "inmemory"}}, +) +``` + +--- + +## 📖 APIs principales + +### `memorize()`: estructura datos en bruto + +memorize + +```python +result = await service.memorize( + resource_url="path/to/file.json", # ruta de archivo local o URL HTTP + modality="conversation", # conversation | document | image | video | audio + user={"user_id": "123"}, # opcional: delimitar a un usuario o agente +) +# Devuelve tras completar el procesamiento: +# { "resource": {...}, "items": [...], "categories": [...], "relations": [...] } +``` + +- Convierte la entrada en bruto en elementos de memoria tipados +- Categoriza y vectoriza los elementos sin etiquetado manual +- Preserva los recursos de origen y las relaciones elemento-categoría + +--- + +### `retrieve()`: carga contexto del agente + +retrieve + +```python +# La estrategia de recuperación se define una vez en el servicio mediante retrieve_config: +# MemoryService(retrieve_config={"method": "rag"}) # recuperación con vectores primero +# MemoryService(retrieve_config={"method": "llm"}) # recuperación ordenada por LLM +result = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What are their preferences?"}}], + where={"user_id": "123"}, # filtro de alcance +) +# Devuelve: +# { +# "needs_retrieval": true, +# "original_query": "...", +# "rewritten_query": "...", +# "next_step_query": "...", +# "categories": [...], +# "items": [...], +# "resources": [...] +# } +``` + +| `retrieve_config.method` | Comportamiento | Coste | Mejor para | +|--------------------------|----------|------|----------| +| `rag` | Recuperación de categorías/elementos/recursos con vectores primero, con enrutamiento LLM y comprobaciones de suficiencia opcionales activadas por defecto | Embeddings más llamadas a LLM, salvo que se desactiven `route_intention` y `sufficiency_check` | Recuperación delimitada y rápida con razonamiento controlable | +| `llm` | Recuperación de categorías/elementos/recursos ordenada por LLM | Ordenación por LLM en cada nivel | Ordenación semántica más profunda | + +--- + +## 💡 Flujos de trabajo de ejemplo + +### Asistente que aprende siempre +```bash +export OPENAI_API_KEY=your_key +uv run python examples/example_1_conversation_memory.py +``` +Extrae preferencias automáticamente, construye modelos de relación y hace aflorar contexto relevante en conversaciones futuras. + +### Agente que se mejora a sí mismo +```bash +uv run python examples/example_2_skill_extraction.py +``` +Supervisa las acciones del agente, identifica patrones en éxitos y fracasos, y autogenera guías de habilidades a partir de la experiencia. + +### Constructor de contexto multimodal +```bash +uv run python examples/example_3_multimodal_memory.py +``` +Cruza automáticamente texto, imágenes y documentos en una capa de memoria unificada. + +--- + +## 📊 Rendimiento + +memU alcanza un **92,09 % de precisión promedio** en el benchmark Locomo en todas las tareas de razonamiento. + +benchmark + +Ver resultados detallados: [memU-experiment](https://github.com/NevaMind-AI/memU-experiment) + +--- + +## 🧩 Ecosistema + +| Repositorio | Descripción | +|------------|-------------| +| **[memU](https://github.com/NevaMind-AI/memU)** | Sistema de archivos de memoria central: ingesta, extracción, recuperación | +| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | Backend con sincronización en tiempo real y disparadores webhook | +| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | Panel visual para explorar y monitorizar la memoria | + +**Enlaces rápidos:** +- 🚀 [Prueba MemU Cloud](https://app.memu.so/quick-start) +- 📚 [Documentación de la API](https://memu.pro/docs) +- 💬 [Comunidad de Discord](https://discord.com/invite/hQZntfGsbJ) + +--- + +## 🤝 Socios + +
+ +Ten +OpenAgents +Milvus +xRoute +Jazz +Buddie +Bytebase +LazyLLM +Clawdchat + +
+ +--- + +## 🤝 Contribuir + +```bash +# Haz fork y clona +git clone https://github.com/YOUR_USERNAME/memU.git +cd memU + +# Instala las dependencias de desarrollo +make install + +# Ejecuta las comprobaciones de calidad antes de enviar +make check +``` + +Consulta [CONTRIBUTING.md](../CONTRIBUTING.md) para las pautas completas. + +**Requisitos previos:** Python 3.13+, [uv](https://github.com/astral-sh/uv), Git + +--- + +## 📄 Licencia + +[Apache License 2.0](../LICENSE.txt) + +--- + +## 🌍 Comunidad + +- **GitHub Issues**: [Reporta errores y solicita funciones](https://github.com/NevaMind-AI/memU/issues) +- **Discord**: [Únete a la comunidad](https://discord.com/invite/hQZntfGsbJ) +- **X (Twitter)**: [Sigue a @memU_ai](https://x.com/memU_ai) +- **Contacto**: info@nevamind.ai + +--- + +
+ +⭐ **Danos una estrella en GitHub** para recibir notificaciones sobre nuevas versiones. + +
diff --git a/readme/README_fr.md b/readme/README_fr.md new file mode 100644 index 0000000..58efc18 --- /dev/null +++ b/readme/README_fr.md @@ -0,0 +1,542 @@ +![MemU Banner](../assets/banner.png) + +
+ +# memU + +### Mémoire personnelle en fichiers + +**Récupération rapide · Plus grande précision · Coût réduit** + +[![PyPI version](https://badge.fury.io/py/memu-py.svg)](https://badge.fury.io/py/memu-py) +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) +[![Discord](https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white)](https://discord.com/invite/hQZntfGsbJ) +[![Twitter](https://img.shields.io/badge/Twitter-Follow-1DA1F2?logo=x&logoColor=white)](https://x.com/memU_ai) + +NevaMind-AI%2FmemU | Trendshift + +**[English](README_en.md) | [中文](README_zh.md) | [日本語](README_ja.md) | [한국어](README_ko.md) | [Español](README_es.md) | [Français](README_fr.md)** + +
+ +--- + +> [!WARNING] +> 🚧 **En pleine refonte** — memU fait l'objet d'une refonte majeure ; les APIs, les commandes CLI et la documentation peuvent changer sans préavis. Stabilisation prévue autour du **15 juillet 2026**. + +memU compile conversations, documents, code, images, audio, vidéo, URLs et traces d'outils en fichiers Markdown lisibles par l'humain (`INDEX.md`, `MEMORY.md`, `SKILL.md`). L'agent parcourt l'arborescence et ne charge que la mémoire utile au moment présent — sans tout scanner ni remplir chaque prompt d'historiques longs : + +- **`INDEX.md`** — carte de tout : catégories, fichiers et résumés, pour que l'agent sache où chercher en premier +- **`MEMORY.md`** — profil, préférences, objectifs, faits et événements clés extraits des données sources +- **`SKILL.md`** — schémas d'outils et workflows réutilisables, extraits automatiquement des traces et affinés à chaque `memorize()` + +```txt +workspace/ +├── INDEX.md ← carte de tout : catégories, fichiers et résumés +├── MEMORY.md ← profil, préférences, objectifs et événements clés +└── skill/ + ├── {skill_name}/ + │ └── SKILL.md ← une compétence ou un schéma d'outil appris + └── {another_skill}/ + └── SKILL.md +``` + +Trois éléments le distinguent de tout mettre dans le prompt : + +- **Récupération rapide** — aller au dossier pertinent et classer les fichiers au lieu de tout scanner à chaque fois. +- **Plus grande précision** — cadrer par utilisateur, tâche ou session, et remonter chaque élément jusqu'à sa source exacte. +- **Coût réduit** — récupérer un contexte compact et cadré au lieu de réinjecter de longs historiques dans chaque prompt. +- **Auditable** — une arborescence lisible par l'humain que vous pouvez inspecter, modifier et router via votre propre stockage et vos fournisseurs LLM. + +--- + +## 🔄 Comment ça marche + +Voyez cela comme deux opérations de système de fichiers : **écrire** des sources brutes dans une mémoire organisée, et **relire** les bons fichiers vers l'agent. + +``` +WRITE — memorize() READ — retrieve() +────────────────────────────────────────────── ────────────────────────────────────────────── +raw files → extract → files + folders query → walk folders → ranked files +───────────── ───────── ────────────── ───── ──────────── ───────────── +chat logs → parse → profile / event items user / task query +documents / URLs → facts → knowledge / skill items │ +images / video → caption → resources + summaries ├─ route + scope → relevant folders (categories) +audio → transcribe→ event / knowledge items ├─ rank by relevance → matching files (items) +tool logs → mine → tool / skill items └─ trace to source → original resources +``` + +**Écrire dans le système de fichiers (`memorize`)** + +1. **Ingérer (Ingest)** : stocke chaque source sous forme de `Resource` (le fichier brut) avec sa modalité et son emplacement d'origine +2. **Prétraiter (Preprocess)** : analyse le texte, légende les images/vidéos, transcrit l'audio et normalise les entrées +3. **Extraire (Extract)** : transforme le contenu brut en enregistrements `RecallEntry` typés (les fichiers) : mémoires de type profile, event, knowledge, behavior, skill ou tool +4. **Organiser (Organize)** : range les éléments dans des dossiers `RecallFile` (une catégorie de mémoire, ou une compétence), les relie entre eux, les vectorise et les résume en une arborescence navigable +5. **Persister (Persist)** : écrit les enregistrements, relations, embeddings et résumés de dossier via le backend configuré + +**Lire depuis le système de fichiers (`retrieve`)** + +6. **Récupérer (Retrieve)** : parcourt les dossiers et ne renvoie que les fichiers pertinents pour l'utilisateur, l'agent, la session ou la tâche en cours + +--- + +## 🗂️ Le système de fichiers de mémoire + +La sortie principale de memU est une arborescence de mémoire navigable — dossiers, fichiers et les artefacts source qui se trouvent derrière — persistée via des contrats de dépôt et renvoyée sous forme de dictionnaires par `memorize()` et `retrieve()`. + +```txt +RecallFile ← dossier : un thème avec un résumé en évolution +├── name, track (memory | skill), description, content +├── embedding +└── RecallEntry[] ← fichiers : des mémoires atomiques et typées + ├── memory_type: profile | event | knowledge | behavior | skill | tool + ├── summary, extra, happened_at, embedding + └── Resource ← source : le fichier brut dont provient cette mémoire + └── url, modality, local_path, caption, embedding +``` + +| Enregistrement | Rôle dans le système de fichiers | Utilisé pour | +|--------|------------------|---------| +| `RecallFile` | **Dossier** : regroupe les mémoires liées (ou contient une compétence) et tient un résumé au niveau du thème ; un champ `track` le marque `memory` ou `skill` | Charger un contexte compact pour les requêtes larges | +| `RecallEntry` | **Fichier** : mémoire atomique typée avec un résumé et des métadonnées optionnelles | Injecter des faits, préférences, événements, compétences et schémas d'outils précis | +| `Resource` | **Artefact source** : le fichier d'origine derrière une mémoire, avec légende/texte | Remonter le contexte jusqu'à son origine | +| `RecallFileEntry` | **Lien** : l'arête qui classe un élément sous un dossier | Naviguer entre les mémoires liées sans retraiter la source | + +Cela donne aux agents un système de fichiers de mémoire stable : ils ingèrent les sources brutes une seule fois, puis demandent des fichiers délimités et classés plutôt que de relire chaque artefact source. + +--- + +## 🧩 Ce que memU construit + +Chaque couche du système de fichiers est stockée sous forme d'enregistrement structuré : + +| Couche | Ce qu'elle représente | Pourquoi les agents l'utilisent | +|-------|--------------------|-------------------| +| **RecallFile** | Dossier auto-généré : un thème (ou une compétence) avec un résumé en évolution | Charger un contexte de haut niveau avant de plonger dans les détails | +| **RecallEntry** | Un fichier : mémoire structurée atomique avec un type et un résumé | Injecter des faits, préférences, événements, compétences et schémas d'outils précis | +| **Resource** | Artefact source derrière un fichier : conversation, document, image, vidéo, audio, URL ou fichier | Remonter la mémoire jusqu'à son origine | +| **RecallFileEntry** | Le lien qui classe un élément sous un dossier | Naviguer entre les mémoires liées sans retraiter la source | +| **Embedding** | Index vectoriel couvrant dossiers, fichiers et sources | Récupérer le contexte pertinent avec une faible latence | + +Exemple de sortie de `memorize()` : + +```json +{ + "resource": { + "id": "res_01", + "url": "files/launch-meeting.mp4", + "modality": "video", + "caption": "A product planning discussion about onboarding and launch risks." + }, + "items": [ + { + "id": "mem_01", + "memory_type": "event", + "summary": "The team decided to simplify onboarding before the next launch review." + }, + { + "id": "mem_02", + "memory_type": "profile", + "summary": "The user prefers concise implementation plans with explicit verification steps." + }, + { + "id": "mem_03", + "memory_type": "tool", + "summary": "Use repository-wide search before editing configuration files to avoid missing duplicated settings." + } + ], + "categories": [ + { + "id": "cat_01", + "name": "product_goals", + "summary": "Current launch priorities, onboarding decisions, and unresolved risks." + } + ], + "relations": [ + { "item_id": "mem_01", "category_id": "cat_01" } + ] +} +``` + +Un agent peut ensuite appeler `retrieve()` pour obtenir une charge de contexte délimitée et classée par pertinence : + +```python +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What context matters for this launch task?"}}], + where={"user_id": "123"}, +) +``` + +--- + +## ⭐️ Mettez une étoile au dépôt + + + +Si vous trouvez memU utile ou intéressant, une étoile ⭐️ sur GitHub serait grandement appréciée. + +--- + +## ✨ Fonctionnalités principales + +| Capacité | Description | +|------------|-------------| +| 🗂️ **Ingestion multimodale** | Écrit conversations, documents, images, vidéos, audio, URL, journaux et fichiers locaux dans la mémoire | +| 📁 **Système de fichiers de mémoire** | Persiste dossiers (catégories), fichiers (éléments), artefacts source, liens, résumés et embeddings | +| 🧠 **Extraction de mémoire typée** | Extrait des mémoires profile, event, knowledge, behavior, skill et tool à partir de sources brutes | +| 🧭 **Dossiers auto-organisés** | Construit automatiquement catégories, liens, résumés et embeddings sans étiquetage manuel | +| 🤖 **Récupération prête pour les agents** | Lit un contexte délimité et classé pouvant être injecté dans n'importe quel workflow d'agent | +| 🧱 **Stockage enfichable** | Utilise des backends in-memory, SQLite ou Postgres avec les mêmes contrats de dépôt | +| 🔀 **Routage de LLM basé sur les profils** | Achemine les tâches de chat, embeddings, vision et transcription via des profils de LLM configurables | + +--- + +## 🎯 Cas d'usage + +### 1. **Mémoire de conversation** +*Transforme les journaux de chat en préférences, objectifs, événements et contexte relationnel de l'utilisateur.* + +```python +await service.memorize( + resource_url="examples/resources/conversations/conv1.json", + modality="conversation", + user={"user_id": "123"}, +) + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What should I remember about this user?"}}], + where={"user_id": "123"}, +) +``` + +### 2. **Contexte d'espace de travail pour les agents de codage** +*Convertit documents, notes de PR, journaux et décisions de conception en mémoire de projet réutilisable.* + +```python +await service.memorize(resource_url="docs/architecture.md", modality="document") +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "How should I structure this module?"}}], +) +``` + +### 3. **Couche de connaissances multimodale** +*Extrait des faits interrogeables à partir de documents, captures d'écran, images, vidéos et notes audio.* + +```python +await service.memorize(resource_url="examples/resources/docs/doc1.txt", modality="document") +await service.memorize(resource_url="examples/resources/images/image1.png", modality="image") +# Audio is supported for your own .mp3/.wav/.m4a files. +await service.memorize(resource_url="meeting-audio.mp3", modality="audio") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What matters for the next research plan?"}}], +) +``` + +### 4. **Apprentissage des outils et des agents** +*Transforme les traces d'exécution en mémoires d'outils qui indiquent aux agents futurs quand utiliser un outil et quelles erreurs éviter.* + +```python +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "Which tools worked for config editing?"}}], +) +``` + +--- + +## 🗂️ Architecture + +Le système de fichiers de mémoire est assez hiérarchique pour être parcouru et assez structuré pour une récupération directe : + +structure + +| Couche | Rôle principal | Rôle dans la récupération | +|-------|--------------|----------------| +| **Category (dossier)** | Maintenir des résumés au niveau du thème | Assembler un contexte compact pour les requêtes larges | +| **Item (fichier)** | Stocker des mémoires atomiques typées | Charger des faits, événements, préférences, compétences et schémas d'outils précis | +| **Resource (source)** | Préserver les artefacts source et les légendes | Rappeler le contexte d'origine lorsque les résumés d'élément/catégorie ne suffisent pas | + +Consultez [docs/architecture.md](../docs/architecture.md) pour la vue à l'exécution de `MemoryService`, des pipelines de workflow, des backends de stockage et du routage de LLM. + +--- + +## 🚀 Démarrage rapide + +### Option 1 : Version cloud + +👉 **[memu.so](https://memu.so)** : API hébergée pour l'ingestion gérée, la mémoire structurée et la récupération + +Pour un déploiement en entreprise : **info@nevamind.ai** + +#### Cloud API (v3) + +| Base URL | `https://api.memu.so` | +|----------|----------------------| +| Auth | `Authorization: Bearer ` | + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/api/v3/memory/memorize` | Ingère des données brutes et construit une mémoire structurée | +| `GET` | `/api/v3/memory/memorize/status/{task_id}` | Vérifie l'état du traitement | +| `POST` | `/api/v3/memory/categories` | Liste les catégories auto-générées | +| `POST` | `/api/v3/memory/retrieve` | Interroge la mémoire pour obtenir le contexte de l'agent | + +📚 **[Documentation complète de l'API](https://memu.pro/docs#cloud-version)** + +--- + +### Option 2 : Auto-hébergé + +#### Installation + +À partir d'un clone de ce dépôt : + +```bash +uv sync +# ou, pour la configuration de développement complète : +make install +``` + +Pour installer le paquet publié à la place : + +```bash +pip install memu-py +``` + +> **Prérequis** : Python 3.13+. Les exemples par défaut utilisent OpenAI ; définissez donc `OPENAI_API_KEY` ou passez un autre fournisseur via `llm_profiles`. + +**Exécuter un script de test en mémoire :** +```bash +export OPENAI_API_KEY=your_key +cd tests +uv run python test_inmemory.py +``` + +**Exécuter avec PostgreSQL + pgvector :** +```bash +uv sync --extra postgres +docker run -d --name memu-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=memu \ + -p 5432:5432 \ + pgvector/pgvector:pg16 + +export OPENAI_API_KEY=your_key +export POSTGRES_DSN=postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu +cd tests +uv run python test_postgres.py +``` + +--- + +### Fournisseurs de LLM et d'embeddings personnalisés + +```python +from memu import MemUService + +service = MemUService( + llm_profiles={ + "default": { + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "api_key": "your_key", + "chat_model": "qwen3-max", + "client_backend": "sdk" + }, + "embedding": { + "base_url": "https://api.voyageai.com/v1", + "api_key": "your_key", + "embed_model": "voyage-3.5-lite" + } + }, +) +``` + +--- + +### Intégration OpenRouter + +```python +from memu import MemoryService + +service = MemoryService( + llm_profiles={ + "default": { + "provider": "openrouter", + "client_backend": "httpx", + "base_url": "https://openrouter.ai", + "api_key": "your_key", + "chat_model": "anthropic/claude-3.5-sonnet", + "embed_model": "openai/text-embedding-3-small", + }, + }, + database_config={"metadata_store": {"provider": "inmemory"}}, +) +``` + +--- + +## 📖 APIs principales + +### `memorize()` : structurer les données brutes + +memorize + +```python +result = await service.memorize( + resource_url="path/to/file.json", # chemin de fichier local ou URL HTTP + modality="conversation", # conversation | document | image | video | audio + user={"user_id": "123"}, # optionnel : délimiter à un utilisateur ou un agent +) +# Renvoie une fois le traitement terminé : +# { "resource": {...}, "items": [...], "categories": [...], "relations": [...] } +``` + +- Convertit l'entrée brute en éléments de mémoire typés +- Catégorise et vectorise les éléments sans étiquetage manuel +- Préserve les ressources source et les relations élément-catégorie + +--- + +### `retrieve()` : charger le contexte de l'agent + +retrieve + +```python +# La stratégie de récupération est définie une fois sur le service via retrieve_config : +# MemoryService(retrieve_config={"method": "rag"}) # rappel vecteurs d'abord +# MemoryService(retrieve_config={"method": "llm"}) # rappel classé par LLM +result = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What are their preferences?"}}], + where={"user_id": "123"}, # filtre de portée +) +# Renvoie : +# { +# "needs_retrieval": true, +# "original_query": "...", +# "rewritten_query": "...", +# "next_step_query": "...", +# "categories": [...], +# "items": [...], +# "resources": [...] +# } +``` + +| `retrieve_config.method` | Comportement | Coût | Idéal pour | +|--------------------------|----------|------|----------| +| `rag` | Rappel des catégories/éléments/ressources avec vecteurs d'abord, avec routage LLM et vérifications de suffisance optionnels activés par défaut | Embeddings plus appels LLM, sauf si `route_intention` et `sufficiency_check` sont désactivés | Rappel délimité et rapide avec un raisonnement contrôlable | +| `llm` | Rappel des catégories/éléments/ressources classé par LLM | Classement par LLM à chaque niveau | Classement sémantique plus profond | + +--- + +## 💡 Exemples de workflows + +### Assistant en apprentissage permanent +```bash +export OPENAI_API_KEY=your_key +uv run python examples/example_1_conversation_memory.py +``` +Extrait automatiquement les préférences, construit des modèles de relation et fait remonter le contexte pertinent dans les conversations futures. + +### Agent qui s'améliore lui-même +```bash +uv run python examples/example_2_skill_extraction.py +``` +Surveille les actions de l'agent, identifie les schémas de réussites et d'échecs, et génère automatiquement des guides de compétences à partir de l'expérience. + +### Constructeur de contexte multimodal +```bash +uv run python examples/example_3_multimodal_memory.py +``` +Recoupe automatiquement texte, images et documents dans une couche de mémoire unifiée. + +--- + +## 📊 Performances + +memU atteint une **précision moyenne de 92,09 %** sur le benchmark Locomo, sur l'ensemble des tâches de raisonnement. + +benchmark + +Voir les résultats détaillés : [memU-experiment](https://github.com/NevaMind-AI/memU-experiment) + +--- + +## 🧩 Écosystème + +| Dépôt | Description | +|------------|-------------| +| **[memU](https://github.com/NevaMind-AI/memU)** | Système de fichiers de mémoire central : ingestion, extraction, récupération | +| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | Backend avec synchronisation en temps réel et déclencheurs webhook | +| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | Tableau de bord visuel pour explorer et surveiller la mémoire | + +**Liens rapides :** +- 🚀 [Essayer MemU Cloud](https://app.memu.so/quick-start) +- 📚 [Documentation de l'API](https://memu.pro/docs) +- 💬 [Communauté Discord](https://discord.com/invite/hQZntfGsbJ) + +--- + +## 🤝 Partenaires + +
+ +Ten +OpenAgents +Milvus +xRoute +Jazz +Buddie +Bytebase +LazyLLM +Clawdchat + +
+ +--- + +## 🤝 Contribuer + +```bash +# Forkez et clonez +git clone https://github.com/YOUR_USERNAME/memU.git +cd memU + +# Installez les dépendances de développement +make install + +# Lancez les contrôles qualité avant de soumettre +make check +``` + +Consultez [CONTRIBUTING.md](../CONTRIBUTING.md) pour les directives complètes. + +**Prérequis :** Python 3.13+, [uv](https://github.com/astral-sh/uv), Git + +--- + +## 📄 Licence + +[Apache License 2.0](../LICENSE.txt) + +--- + +## 🌍 Communauté + +- **GitHub Issues** : [Signaler des bugs et demander des fonctionnalités](https://github.com/NevaMind-AI/memU/issues) +- **Discord** : [Rejoindre la communauté](https://discord.com/invite/hQZntfGsbJ) +- **X (Twitter)** : [Suivre @memU_ai](https://x.com/memU_ai) +- **Contact** : info@nevamind.ai + +--- + +
+ +⭐ **Mettez-nous une étoile sur GitHub** pour être averti des nouvelles versions ! + +
diff --git a/readme/README_ja.md b/readme/README_ja.md new file mode 100644 index 0000000..1847f0b --- /dev/null +++ b/readme/README_ja.md @@ -0,0 +1,542 @@ +![MemU Banner](../assets/banner.png) + +
+ +# memU + +### 個人の記憶を、ファイルとして + +**高速な検索 · 高い精度 · 低コスト** + +[![PyPI version](https://badge.fury.io/py/memu-py.svg)](https://badge.fury.io/py/memu-py) +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) +[![Discord](https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white)](https://discord.com/invite/hQZntfGsbJ) +[![Twitter](https://img.shields.io/badge/Twitter-Follow-1DA1F2?logo=x&logoColor=white)](https://x.com/memU_ai) + +NevaMind-AI%2FmemU | Trendshift + +**[English](README_en.md) | [中文](README_zh.md) | [日本語](README_ja.md) | [한국어](README_ko.md) | [Español](README_es.md) | [Français](README_fr.md)** + +
+ +--- + +> [!WARNING] +> 🚧 **大規模改修中** — memU は大規模なリワークを進めており、API・CLI コマンド・ドキュメントは予告なく変更される可能性があります。**2026年7月15日**頃に安定する見込みです。 + +memU は、会話、ドキュメント、コード、画像、音声、動画、URL、ツールトレースを、人間が読める Markdown ファイル(`INDEX.md`、`MEMORY.md`、`SKILL.md`)にコンパイルします。エージェントはファイルツリーをたどり、その場に必要な記憶だけを読み込みます——すべてをスキャンしたり、長い履歴を毎回のプロンプトに詰め込んだりする必要はありません: + +- **`INDEX.md`** —— 全体のマップ:カテゴリ、ファイル、サマリー。まずどこを見るべきかをエージェントに示す +- **`MEMORY.md`** —— プロフィール、嗜好、目標、事実、およびソースデータから抽出した主要な出来事 +- **`SKILL.md`** —— ツールトレースから自動抽出され、毎回の `memorize()` で洗練される再利用可能なツールパターンとワークフロー + +```txt +workspace/ +├── INDEX.md ← 全体のマップ:カテゴリ、ファイル、サマリー +├── MEMORY.md ← プロフィール、嗜好、目標、主要な出来事 +└── skill/ + ├── {skill_name}/ + │ └── SKILL.md ← 学習したスキルまたはツールのパターン + └── {another_skill}/ + └── SKILL.md +``` + +すべてをプロンプトに詰め込む方式との違いは 3 点です: + +- **高速な検索** —— 毎回すべてをスキャンするのではなく、関連フォルダへたどり着き、ファイルをランク付けする。 +- **高い精度** —— ユーザー、タスク、セッションでスコープを絞り、各項目を元のソースまで遡れる。 +- **低コスト** —— 長い履歴を毎回注入するのではなく、コンパクトでスコープされたコンテキストだけを取得する。 +- **監査可能** —— 人間が読めるファイルツリーを監査・編集し、独自のストレージと LLM プロバイダーでルーティングできる。 + +--- + +## 🔄 仕組み + +2 つのファイルシステム操作として考えてください:生のソースを整理された記憶に**書き込み**、適切なファイルをエージェントに**読み戻す**。 + +``` +WRITE — memorize() READ — retrieve() +────────────────────────────────────────────── ────────────────────────────────────────────── +raw files → extract → files + folders query → walk folders → ranked files +───────────── ───────── ────────────── ───── ──────────── ───────────── +chat logs → parse → profile / event items user / task query +documents / URLs → facts → knowledge / skill items │ +images / video → caption → resources + summaries ├─ route + scope → relevant folders (categories) +audio → transcribe→ event / knowledge items ├─ rank by relevance → matching files (items) +tool logs → mine → tool / skill items └─ trace to source → original resources +``` + +**ファイルシステムへの書き込み(`memorize`)** + +1. **取り込み(Ingest)** — 各ソースを `Resource`(生のファイル)として、そのモダリティとソースの場所とともに保存する +2. **前処理(Preprocess)** — テキストを解析し、画像/動画にキャプションを付け、音声を文字起こしし、入力を正規化する +3. **抽出(Extract)** — 生のコンテンツを型付きの `RecallEntry` レコード(ファイル)に変換する:profile、event、knowledge、behavior、skill、tool の記憶 +4. **整理(Organize)** — アイテムを `RecallFile` フォルダ(記憶カテゴリ、またはスキル)に振り分け、相互リンクし、埋め込みを作り、閲覧可能なツリーへ要約する +5. **永続化(Persist)** — 設定されたバックエンドを通じてレコード、関係、埋め込み、フォルダのサマリーを書き込む + +**ファイルシステムからの読み込み(`retrieve`)** + +6. **検索(Retrieve)** — フォルダをたどり、現在のユーザー、エージェント、セッション、タスクに関連するファイルだけを返す + +--- + +## 🗂️ 記憶ファイルシステム + +memU の主要な出力は、ナビゲート可能な記憶のツリーです——フォルダ、ファイル、そしてそれらの背後にあるソース素材——リポジトリ契約を通じて永続化され、`memorize()` と `retrieve()` から辞書として返されます。 + +```txt +RecallFile ← フォルダ:進化するサマリーを持つトピック +├── name, track (memory | skill), description, content +├── embedding +└── RecallEntry[] ← ファイル:型付きのアトミックな記憶 + ├── memory_type: profile | event | knowledge | behavior | skill | tool + ├── summary, extra, happened_at, embedding + └── Resource ← ソース:この記憶の由来となった生のファイル + └── url, modality, local_path, caption, embedding +``` + +| レコード | ファイルシステム上の役割 | 用途 | +|--------|------------------|---------| +| `RecallFile` | **フォルダ** — 関連する記憶をまとめ(またはスキルを保持し)、トピックレベルのサマリーを保持する。`track` フィールドが `memory` か `skill` かを示す | 広いクエリ向けにコンパクトなコンテキストを読み込む | +| `RecallEntry` | **ファイル** — サマリーと任意のメタデータを持つ型付きアトミック記憶 | 正確な事実、嗜好、出来事、スキル、ツールパターンを注入する | +| `Resource` | **ソース素材** — 記憶の背後にある元ファイル(キャプション/テキスト付き) | コンテキストをその由来まで遡る | +| `RecallFileEntry` | **リンク** — アイテムをフォルダ配下に振り分けるエッジ | ソースを再処理せずに関連する記憶をナビゲートする | + +これによりエージェントは安定した記憶のファイルシステムを得ます:生のソースは一度取り込めばよく、以降はすべてのソース素材を読み直す代わりに、スコープ付き・ランク付けされたファイルを要求できます。 + +--- + +## 🧩 memU が構築するもの + +ファイルシステムの各層は構造化レコードとして保存されます: + +| 層 | 何を表すか | エージェントが使う理由 | +|-------|--------------------|-------------------| +| **RecallFile** | 自動生成されるフォルダ:進化するサマリーを持つトピック(またはスキル) | 詳細に踏み込む前に高レベルのコンテキストを読み込む | +| **RecallEntry** | ファイル:型とサマリーを持つアトミックな構造化記憶 | 正確な事実、嗜好、出来事、スキル、ツールパターンを注入する | +| **Resource** | ファイルの背後にあるソース素材:会話、ドキュメント、画像、動画、音声、URL、ファイル | 記憶をその由来まで遡る | +| **RecallFileEntry** | アイテムをフォルダ配下に振り分けるリンク | ソースを再処理せずに関連する記憶をナビゲートする | +| **Embedding** | フォルダ・ファイル・ソースにまたがるベクトルインデックス | 低レイテンシで関連コンテキストを検索する | + +`memorize()` の出力例: + +```json +{ + "resource": { + "id": "res_01", + "url": "files/launch-meeting.mp4", + "modality": "video", + "caption": "A product planning discussion about onboarding and launch risks." + }, + "items": [ + { + "id": "mem_01", + "memory_type": "event", + "summary": "The team decided to simplify onboarding before the next launch review." + }, + { + "id": "mem_02", + "memory_type": "profile", + "summary": "The user prefers concise implementation plans with explicit verification steps." + }, + { + "id": "mem_03", + "memory_type": "tool", + "summary": "Use repository-wide search before editing configuration files to avoid missing duplicated settings." + } + ], + "categories": [ + { + "id": "cat_01", + "name": "product_goals", + "summary": "Current launch priorities, onboarding decisions, and unresolved risks." + } + ], + "relations": [ + { "item_id": "mem_01", "category_id": "cat_01" } + ] +} +``` + +その後、エージェントは `retrieve()` を呼び出してスコープ付き・ランク付けされたコンテキストのペイロードを取得できます: + +```python +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What context matters for this launch task?"}}], + where={"user_id": "123"}, +) +``` + +--- + +## ⭐️ リポジトリにスターを + + + +memU が役に立つ・面白いと感じたら、GitHub のスター ⭐️ をいただけると大変ありがたいです。 + +--- + +## ✨ 主な機能 + +| 機能 | 説明 | +|------------|-------------| +| 🗂️ **マルチモーダル取り込み** | 会話、ドキュメント、画像、動画、音声、URL、ログ、ローカルファイルを記憶に書き込む | +| 📁 **記憶ファイルシステム** | フォルダ(カテゴリ)、ファイル(アイテム)、ソース素材、リンク、サマリー、埋め込みを永続化する | +| 🧠 **型付き記憶の抽出** | 生のソースから profile、event、knowledge、behavior、skill、tool の記憶を抽出する | +| 🧭 **自己組織化フォルダ** | 手動タグ付けなしでカテゴリ、リンク、サマリー、埋め込みを自動構築する | +| 🤖 **エージェント対応の検索** | 任意のエージェントワークフローに注入できる、スコープ付き・ランク付けされたコンテキストを読み出す | +| 🧱 **プラグイン可能なストレージ** | 同一のリポジトリ契約で in-memory、SQLite、Postgres バックエンドを使い分ける | +| 🔀 **プロファイルベースの LLM ルーティング** | 設定可能な LLM プロファイルでチャット、埋め込み、ビジョン、文字起こしの処理をルーティングする | + +--- + +## 🎯 ユースケース + +### 1. **会話の記憶** +*チャットログをユーザーの嗜好、目標、出来事、関係性のコンテキストに変える。* + +```python +await service.memorize( + resource_url="examples/resources/conversations/conv1.json", + modality="conversation", + user={"user_id": "123"}, +) + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What should I remember about this user?"}}], + where={"user_id": "123"}, +) +``` + +### 2. **コーディングエージェント向けのワークスペースコンテキスト** +*ドキュメント、PR メモ、ログ、設計判断を再利用可能なプロジェクト記憶に変える。* + +```python +await service.memorize(resource_url="docs/architecture.md", modality="document") +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "How should I structure this module?"}}], +) +``` + +### 3. **マルチモーダルなナレッジ層** +*ドキュメント、スクリーンショット、画像、動画、音声メモから検索可能な事実を抽出する。* + +```python +await service.memorize(resource_url="examples/resources/docs/doc1.txt", modality="document") +await service.memorize(resource_url="examples/resources/images/image1.png", modality="image") +# Audio is supported for your own .mp3/.wav/.m4a files. +await service.memorize(resource_url="meeting-audio.mp3", modality="audio") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What matters for the next research plan?"}}], +) +``` + +### 4. **ツールとエージェントの学習** +*実行トレースをツール記憶に変え、将来のエージェントにいつツールを使うべきか・どんな失敗を避けるべきかを伝える。* + +```python +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "Which tools worked for config editing?"}}], +) +``` + +--- + +## 🗂️ アーキテクチャ + +記憶ファイルシステムは、閲覧できるほど階層的であり、直接検索できるほど構造化されています: + +structure + +| 層 | 主な役割 | 検索での役割 | +|-------|--------------|----------------| +| **Category(フォルダ)** | トピックレベルのサマリーを維持する | 広いクエリ向けにコンパクトなコンテキストを組み立てる | +| **Item(ファイル)** | 型付きのアトミック記憶を保存する | 正確な事実、出来事、嗜好、スキル、ツールパターンを読み込む | +| **Resource(ソース)** | ソース素材とキャプションを保持する | アイテム/カテゴリのサマリーで不十分なとき元のコンテキストを呼び戻す | + +`MemoryService`、ワークフローパイプライン、ストレージバックエンド、LLM ルーティングのランタイムビューについては [docs/architecture.md](../docs/architecture.md) を参照してください。 + +--- + +## 🚀 クイックスタート + +### オプション 1:クラウド版 + +👉 **[memu.so](https://memu.so)** — 取り込み、構造化記憶、検索をマネージドで提供するホスト型 API + +エンタープライズ導入:**info@nevamind.ai** + +#### Cloud API (v3) + +| Base URL | `https://api.memu.so` | +|----------|----------------------| +| Auth | `Authorization: Bearer ` | + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/api/v3/memory/memorize` | 生データを取り込み構造化記憶を構築する | +| `GET` | `/api/v3/memory/memorize/status/{task_id}` | 処理ステータスを確認する | +| `POST` | `/api/v3/memory/categories` | 自動生成されたカテゴリを一覧する | +| `POST` | `/api/v3/memory/retrieve` | エージェントのコンテキストを得るために記憶を検索する | + +📚 **[完全な API ドキュメント](https://memu.pro/docs#cloud-version)** + +--- + +### オプション 2:セルフホスト + +#### インストール + +本リポジトリのクローンから: + +```bash +uv sync +# または、フルの開発セットアップ: +make install +``` + +公開パッケージをインストールする場合: + +```bash +pip install memu-py +``` + +> **要件**:Python 3.13+。デフォルトの例は OpenAI を使うため、`OPENAI_API_KEY` を設定するか、`llm_profiles` で別のプロバイダーを渡してください。 + +**インメモリのスモークスクリプトを実行:** +```bash +export OPENAI_API_KEY=your_key +cd tests +uv run python test_inmemory.py +``` + +**PostgreSQL + pgvector で実行:** +```bash +uv sync --extra postgres +docker run -d --name memu-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=memu \ + -p 5432:5432 \ + pgvector/pgvector:pg16 + +export OPENAI_API_KEY=your_key +export POSTGRES_DSN=postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu +cd tests +uv run python test_postgres.py +``` + +--- + +### カスタム LLM と埋め込みプロバイダー + +```python +from memu import MemUService + +service = MemUService( + llm_profiles={ + "default": { + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "api_key": "your_key", + "chat_model": "qwen3-max", + "client_backend": "sdk" + }, + "embedding": { + "base_url": "https://api.voyageai.com/v1", + "api_key": "your_key", + "embed_model": "voyage-3.5-lite" + } + }, +) +``` + +--- + +### OpenRouter 連携 + +```python +from memu import MemoryService + +service = MemoryService( + llm_profiles={ + "default": { + "provider": "openrouter", + "client_backend": "httpx", + "base_url": "https://openrouter.ai", + "api_key": "your_key", + "chat_model": "anthropic/claude-3.5-sonnet", + "embed_model": "openai/text-embedding-3-small", + }, + }, + database_config={"metadata_store": {"provider": "inmemory"}}, +) +``` + +--- + +## 📖 コア API + +### `memorize()` — 生データを構造化する + +memorize + +```python +result = await service.memorize( + resource_url="path/to/file.json", # ローカルファイルパスまたは HTTP URL + modality="conversation", # conversation | document | image | video | audio + user={"user_id": "123"}, # 任意:ユーザーやエージェントにスコープを限定 +) +# 処理完了後に返る: +# { "resource": {...}, "items": [...], "categories": [...], "relations": [...] } +``` + +- 生の入力を型付きのメモリアイテムに変換する +- 手動タグ付けなしでアイテムを分類し埋め込みを作る +- ソースリソースとアイテム–カテゴリ関係を保持する + +--- + +### `retrieve()` — エージェントのコンテキストを読み込む + +retrieve + +```python +# 検索戦略は retrieve_config でサービスに一度だけ設定します: +# MemoryService(retrieve_config={"method": "rag"}) # ベクトル優先の想起 +# MemoryService(retrieve_config={"method": "llm"}) # LLM ランク付けの想起 +result = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What are their preferences?"}}], + where={"user_id": "123"}, # スコープフィルタ +) +# 返り値: +# { +# "needs_retrieval": true, +# "original_query": "...", +# "rewritten_query": "...", +# "next_step_query": "...", +# "categories": [...], +# "items": [...], +# "resources": [...] +# } +``` + +| `retrieve_config.method` | 動作 | コスト | 適した用途 | +|--------------------------|----------|------|----------| +| `rag` | ベクトル優先のカテゴリ/アイテム/リソース想起。任意の LLM ルーティングと充足性チェックがデフォルトで有効 | `route_intention` と `sufficiency_check` を無効にしない限り、埋め込みに加えて LLM 呼び出し | 推論を制御可能にした高速なスコープ想起 | +| `llm` | LLM がランク付けするカテゴリ/アイテム/リソース想起 | 各階層での LLM ランク付け | より深い意味的ランク付け | + +--- + +## 💡 サンプルワークフロー + +### 学び続けるアシスタント +```bash +export OPENAI_API_KEY=your_key +uv run python examples/example_1_conversation_memory.py +``` +嗜好を自動的に抽出し、関係モデルを構築し、将来の会話で関連コンテキストを浮かび上がらせます。 + +### 自己改善するエージェント +```bash +uv run python examples/example_2_skill_extraction.py +``` +エージェントの行動を監視し、成功と失敗のパターンを特定し、経験からスキルガイドを自動生成します。 + +### マルチモーダル・コンテキストビルダー +```bash +uv run python examples/example_3_multimodal_memory.py +``` +テキスト、画像、ドキュメントを自動的に相互参照し、統一された記憶層にまとめます。 + +--- + +## 📊 パフォーマンス + +memU は Locomo ベンチマークのすべての推論タスクで **平均 92.09% の精度** を達成しています。 + +benchmark + +詳細な結果を見る:[memU-experiment](https://github.com/NevaMind-AI/memU-experiment) + +--- + +## 🧩 エコシステム + +| リポジトリ | 説明 | +|------------|-------------| +| **[memU](https://github.com/NevaMind-AI/memU)** | コアの記憶ファイルシステム —— 取り込み、抽出、検索 | +| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | リアルタイム同期と webhook トリガーを備えたバックエンド | +| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | 記憶を閲覧・監視するためのビジュアルダッシュボード | + +**クイックリンク:** +- 🚀 [MemU Cloud を試す](https://app.memu.so/quick-start) +- 📚 [API ドキュメント](https://memu.pro/docs) +- 💬 [Discord コミュニティ](https://discord.com/invite/hQZntfGsbJ) + +--- + +## 🤝 パートナー + +
+ +Ten +OpenAgents +Milvus +xRoute +Jazz +Buddie +Bytebase +LazyLLM +Clawdchat + +
+ +--- + +## 🤝 コントリビュート + +```bash +# フォークしてクローン +git clone https://github.com/YOUR_USERNAME/memU.git +cd memU + +# 開発依存をインストール +make install + +# 送信前に品質チェックを実行 +make check +``` + +詳しいガイドラインは [CONTRIBUTING.md](../CONTRIBUTING.md) を参照してください。 + +**前提条件:** Python 3.13+、[uv](https://github.com/astral-sh/uv)、Git + +--- + +## 📄 ライセンス + +[Apache License 2.0](../LICENSE.txt) + +--- + +## 🌍 コミュニティ + +- **GitHub Issues**:[バグ報告と機能リクエスト](https://github.com/NevaMind-AI/memU/issues) +- **Discord**:[コミュニティに参加](https://discord.com/invite/hQZntfGsbJ) +- **X (Twitter)**:[@memU_ai をフォロー](https://x.com/memU_ai) +- **連絡先**:info@nevamind.ai + +--- + +
+ +⭐ **GitHub でスターを** つけて、新リリースの通知を受け取りましょう! + +
diff --git a/readme/README_ko.md b/readme/README_ko.md new file mode 100644 index 0000000..d246001 --- /dev/null +++ b/readme/README_ko.md @@ -0,0 +1,542 @@ +![MemU Banner](../assets/banner.png) + +
+ +# memU + +### 개인 기억, 파일로 저장 + +**빠른 검색 · 높은 정확도 · 낮은 비용** + +[![PyPI version](https://badge.fury.io/py/memu-py.svg)](https://badge.fury.io/py/memu-py) +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) +[![Discord](https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white)](https://discord.com/invite/hQZntfGsbJ) +[![Twitter](https://img.shields.io/badge/Twitter-Follow-1DA1F2?logo=x&logoColor=white)](https://x.com/memU_ai) + +NevaMind-AI%2FmemU | Trendshift + +**[English](README_en.md) | [中文](README_zh.md) | [日本語](README_ja.md) | [한국어](README_ko.md) | [Español](README_es.md) | [Français](README_fr.md)** + +
+ +--- + +> [!WARNING] +> 🚧 **대규모 개편 진행 중** — memU는 대규모 리워크 중이며 API, CLI 명령어, 문서가 예고 없이 변경될 수 있습니다. **2026년 7월 15일**경 안정화될 예정입니다. + +memU는 대화, 문서, 코드, 이미지, 오디오, 비디오, URL, 도구 트레이스를 사람이 읽을 수 있는 Markdown 파일(`INDEX.md`, `MEMORY.md`, `SKILL.md`)로 컴파일합니다. 에이전트는 파일 트리를 탐색하며 그 순간 필요한 기억만 불러옵니다 — 매번 전체를 스캔하거나 긴 기록을 프롬프트에 넣을 필요가 없습니다: + +- **`INDEX.md`** — 전체 지도: 카테고리, 파일, 요약. 에이전트가 먼저 어디를 봐야 하는지 알려줍니다 +- **`MEMORY.md`** — 프로필, 선호, 목표, 사실, 그리고 소스 데이터에서 추출한 주요 사건 +- **`SKILL.md`** — 도구 트레이스에서 자동 추출되고 매 `memorize()`마다 정제되는 재사용 가능한 도구 패턴과 워크플로 + +```txt +workspace/ +├── INDEX.md ← 전체 지도: 카테고리, 파일, 요약 +├── MEMORY.md ← 프로필, 선호, 목표, 주요 사건 +└── skill/ + ├── {skill_name}/ + │ └── SKILL.md ← 학습한 스킬 또는 도구 패턴 + └── {another_skill}/ + └── SKILL.md +``` + +모든 것을 프롬프트에 넣는 방식과 달리 memU는 세 가지 핵심 이점이 있습니다: + +- **빠른 검색** — 매번 전체를 스캔하는 대신 관련 폴더로 이동하고 파일을 순위화합니다. +- **높은 정확도** — 사용자, 작업, 세션별로 범위를 좁히고 각 항목을 원본 소스까지 추적합니다. +- **낮은 비용** — 긴 기록을 매번 주입하는 대신 압축된 범위 지정 컨텍스트만 검색합니다. +- **감사 가능** — 사람이 읽을 수 있는 파일 트리를 검토·편집하고, 자체 스토리지와 LLM 제공자로 라우팅할 수 있습니다. + +--- + +## 🔄 작동 방식 + +두 가지 파일 시스템 작업으로 생각하세요: 원본 소스를 정리된 메모리에 **쓰기**, 그리고 알맞은 파일을 에이전트로 다시 **읽기**. + +``` +WRITE — memorize() READ — retrieve() +────────────────────────────────────────────── ────────────────────────────────────────────── +raw files → extract → files + folders query → walk folders → ranked files +───────────── ───────── ────────────── ───── ──────────── ───────────── +chat logs → parse → profile / event items user / task query +documents / URLs → facts → knowledge / skill items │ +images / video → caption → resources + summaries ├─ route + scope → relevant folders (categories) +audio → transcribe→ event / knowledge items ├─ rank by relevance → matching files (items) +tool logs → mine → tool / skill items └─ trace to source → original resources +``` + +**파일 시스템에 쓰기(`memorize`)** + +1. **수집(Ingest)** — 각 소스를 모달리티와 소스 위치와 함께 `Resource`(원본 파일)로 저장합니다 +2. **전처리(Preprocess)** — 텍스트를 파싱하고, 이미지/비디오에 캡션을 달고, 오디오를 전사하고, 입력을 정규화합니다 +3. **추출(Extract)** — 원본 콘텐츠를 타입이 지정된 `RecallEntry` 레코드(파일)로 변환합니다: profile, event, knowledge, behavior, skill, tool 메모리 +4. **조직화(Organize)** — 아이템을 `RecallFile` 폴더(메모리 카테고리 또는 스킬)로 분류하고, 상호 링크하고, 임베딩하고, 탐색 가능한 트리로 요약합니다 +5. **영속화(Persist)** — 설정된 백엔드를 통해 레코드, 관계, 임베딩, 폴더 요약을 기록합니다 + +**파일 시스템에서 읽기(`retrieve`)** + +6. **검색(Retrieve)** — 폴더를 탐색하여 현재 사용자, 에이전트, 세션 또는 태스크에 관련된 파일만 반환합니다 + +--- + +## 🗂️ 메모리 파일 시스템 + +memU의 주요 산출물은 탐색 가능한 메모리 트리입니다 — 폴더, 파일, 그리고 그 뒤에 있는 소스 산출물 — 리포지토리 계약을 통해 영속화되며 `memorize()`와 `retrieve()`에서 딕셔너리로 반환됩니다. + +```txt +RecallFile ← 폴더: 진화하는 요약을 가진 주제 +├── name, track (memory | skill), description, content +├── embedding +└── RecallEntry[] ← 파일: 타입이 지정된 원자적 메모리 + ├── memory_type: profile | event | knowledge | behavior | skill | tool + ├── summary, extra, happened_at, embedding + └── Resource ← 소스: 이 메모리가 비롯된 원본 파일 + └── url, modality, local_path, caption, embedding +``` + +| 레코드 | 파일 시스템 역할 | 사용 목적 | +|--------|------------------|---------| +| `RecallFile` | **폴더** — 관련 메모리를 묶고(또는 스킬을 담고) 주제 수준 요약을 유지; `track` 필드가 `memory` 또는 `skill`을 표시 | 폭넓은 질의를 위해 간결한 컨텍스트를 로드 | +| `RecallEntry` | **파일** — 요약과 선택적 메타데이터를 가진 타입 지정 원자 메모리 | 정확한 사실, 선호, 사건, 스킬, 도구 패턴을 주입 | +| `Resource` | **소스 산출물** — 메모리 뒤의 원본 파일(캡션/텍스트 포함) | 컨텍스트를 그 출처까지 추적 | +| `RecallFileEntry` | **링크** — 아이템을 폴더 아래에 분류하는 엣지 | 소스를 재처리하지 않고 관련 메모리를 탐색 | + +이를 통해 에이전트는 안정적인 메모리 파일 시스템을 얻습니다: 원본 소스는 한 번만 수집하면 되고, 이후에는 모든 소스 산출물을 다시 읽는 대신 범위가 지정되고 순위가 매겨진 파일을 요청할 수 있습니다. + +--- + +## 🧩 memU가 구축하는 것 + +파일 시스템의 각 계층은 구조화 레코드로 저장됩니다: + +| 계층 | 무엇을 나타내는가 | 에이전트가 사용하는 이유 | +|-------|--------------------|-------------------| +| **RecallFile** | 자동 생성 폴더: 진화하는 요약을 가진 주제(또는 스킬) | 세부로 파고들기 전에 상위 수준 컨텍스트를 로드 | +| **RecallEntry** | 파일: 타입과 요약을 가진 원자적 구조화 메모리 | 정확한 사실, 선호, 사건, 스킬, 도구 패턴을 주입 | +| **Resource** | 파일 뒤의 소스 산출물: 대화, 문서, 이미지, 비디오, 오디오, URL, 파일 | 메모리를 그 출처까지 추적 | +| **RecallFileEntry** | 아이템을 폴더 아래에 분류하는 링크 | 소스를 재처리하지 않고 관련 메모리를 탐색 | +| **Embedding** | 폴더, 파일, 소스를 아우르는 벡터 인덱스 | 낮은 지연으로 관련 컨텍스트를 검색 | + +`memorize()` 출력 예시: + +```json +{ + "resource": { + "id": "res_01", + "url": "files/launch-meeting.mp4", + "modality": "video", + "caption": "A product planning discussion about onboarding and launch risks." + }, + "items": [ + { + "id": "mem_01", + "memory_type": "event", + "summary": "The team decided to simplify onboarding before the next launch review." + }, + { + "id": "mem_02", + "memory_type": "profile", + "summary": "The user prefers concise implementation plans with explicit verification steps." + }, + { + "id": "mem_03", + "memory_type": "tool", + "summary": "Use repository-wide search before editing configuration files to avoid missing duplicated settings." + } + ], + "categories": [ + { + "id": "cat_01", + "name": "product_goals", + "summary": "Current launch priorities, onboarding decisions, and unresolved risks." + } + ], + "relations": [ + { "item_id": "mem_01", "category_id": "cat_01" } + ] +} +``` + +그런 다음 에이전트는 `retrieve()`를 호출하여 범위가 지정되고 순위가 매겨진 컨텍스트 페이로드를 얻을 수 있습니다: + +```python +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What context matters for this launch task?"}}], + where={"user_id": "123"}, +) +``` + +--- + +## ⭐️ 저장소에 스타 누르기 + + + +memU가 유용하거나 흥미롭다고 느끼셨다면, GitHub 스타 ⭐️ 를 눌러 주시면 정말 감사하겠습니다. + +--- + +## ✨ 핵심 기능 + +| 기능 | 설명 | +|------------|-------------| +| 🗂️ **멀티모달 수집** | 대화, 문서, 이미지, 비디오, 오디오, URL, 로그, 로컬 파일을 메모리에 기록 | +| 📁 **메모리 파일 시스템** | 폴더(카테고리), 파일(아이템), 소스 산출물, 링크, 요약, 임베딩을 영속화 | +| 🧠 **타입 지정 메모리 추출** | 원본 소스에서 profile, event, knowledge, behavior, skill, tool 메모리를 추출 | +| 🧭 **자기 조직화 폴더** | 수동 태깅 없이 카테고리, 링크, 요약, 임베딩을 자동 구축 | +| 🤖 **에이전트 친화적 검색** | 어떤 에이전트 워크플로에도 주입할 수 있는, 범위 지정되고 순위가 매겨진 컨텍스트를 읽기 | +| 🧱 **플러그형 스토리지** | 동일한 리포지토리 계약으로 in-memory, SQLite, Postgres 백엔드 사용 | +| 🔀 **프로파일 기반 LLM 라우팅** | 설정 가능한 LLM 프로파일로 채팅, 임베딩, 비전, 전사 작업을 라우팅 | + +--- + +## 🎯 활용 사례 + +### 1. **대화 메모리** +*채팅 로그를 사용자 선호, 목표, 사건, 관계 컨텍스트로 전환.* + +```python +await service.memorize( + resource_url="examples/resources/conversations/conv1.json", + modality="conversation", + user={"user_id": "123"}, +) + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What should I remember about this user?"}}], + where={"user_id": "123"}, +) +``` + +### 2. **코딩 에이전트를 위한 워크스페이스 컨텍스트** +*문서, PR 메모, 로그, 설계 결정을 재사용 가능한 프로젝트 메모리로 전환.* + +```python +await service.memorize(resource_url="docs/architecture.md", modality="document") +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "How should I structure this module?"}}], +) +``` + +### 3. **멀티모달 지식 계층** +*문서, 스크린샷, 이미지, 비디오, 음성 메모에서 검색 가능한 사실을 추출.* + +```python +await service.memorize(resource_url="examples/resources/docs/doc1.txt", modality="document") +await service.memorize(resource_url="examples/resources/images/image1.png", modality="image") +# Audio is supported for your own .mp3/.wav/.m4a files. +await service.memorize(resource_url="meeting-audio.mp3", modality="audio") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What matters for the next research plan?"}}], +) +``` + +### 4. **도구와 에이전트 학습** +*실행 트레이스를 도구 메모리로 전환하여, 미래의 에이전트에게 언제 도구를 써야 하고 어떤 실수를 피해야 하는지 알려줍니다.* + +```python +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "Which tools worked for config editing?"}}], +) +``` + +--- + +## 🗂️ 아키텍처 + +메모리 파일 시스템은 탐색할 만큼 계층적이면서 직접 검색할 만큼 구조화되어 있습니다: + +structure + +| 계층 | 주요 역할 | 검색 역할 | +|-------|--------------|----------------| +| **Category(폴더)** | 주제 수준 요약을 유지 | 폭넓은 질의를 위해 간결한 컨텍스트를 조립 | +| **Item(파일)** | 타입 지정 원자 메모리를 저장 | 정확한 사실, 사건, 선호, 스킬, 도구 패턴을 로드 | +| **Resource(소스)** | 소스 산출물과 캡션을 보존 | 아이템/카테고리 요약으로 부족할 때 원본 컨텍스트를 회수 | + +`MemoryService`, 워크플로 파이프라인, 스토리지 백엔드, LLM 라우팅의 런타임 관점은 [docs/architecture.md](../docs/architecture.md)를 참고하세요. + +--- + +## 🚀 빠른 시작 + +### 옵션 1: 클라우드 버전 + +👉 **[memu.so](https://memu.so)** — 관리형 수집, 구조화 메모리, 검색을 제공하는 호스팅 API + +엔터프라이즈 배포: **info@nevamind.ai** + +#### Cloud API (v3) + +| Base URL | `https://api.memu.so` | +|----------|----------------------| +| Auth | `Authorization: Bearer ` | + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/api/v3/memory/memorize` | 원본 데이터를 수집하고 구조화 메모리를 구축 | +| `GET` | `/api/v3/memory/memorize/status/{task_id}` | 처리 상태 확인 | +| `POST` | `/api/v3/memory/categories` | 자동 생성된 카테고리 나열 | +| `POST` | `/api/v3/memory/retrieve` | 에이전트 컨텍스트를 위해 메모리 질의 | + +📚 **[전체 API 문서](https://memu.pro/docs#cloud-version)** + +--- + +### 옵션 2: 셀프 호스팅 + +#### 설치 + +이 저장소의 클론에서: + +```bash +uv sync +# 또는, 전체 개발 환경 설정: +make install +``` + +게시된 패키지를 설치하려면: + +```bash +pip install memu-py +``` + +> **요구 사항**: Python 3.13+. 기본 예제는 OpenAI를 사용하므로 `OPENAI_API_KEY`를 설정하거나 `llm_profiles`로 다른 제공자를 전달하세요. + +**인메모리 스모크 스크립트 실행:** +```bash +export OPENAI_API_KEY=your_key +cd tests +uv run python test_inmemory.py +``` + +**PostgreSQL + pgvector로 실행:** +```bash +uv sync --extra postgres +docker run -d --name memu-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=memu \ + -p 5432:5432 \ + pgvector/pgvector:pg16 + +export OPENAI_API_KEY=your_key +export POSTGRES_DSN=postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu +cd tests +uv run python test_postgres.py +``` + +--- + +### 사용자 지정 LLM 및 임베딩 제공자 + +```python +from memu import MemUService + +service = MemUService( + llm_profiles={ + "default": { + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "api_key": "your_key", + "chat_model": "qwen3-max", + "client_backend": "sdk" + }, + "embedding": { + "base_url": "https://api.voyageai.com/v1", + "api_key": "your_key", + "embed_model": "voyage-3.5-lite" + } + }, +) +``` + +--- + +### OpenRouter 연동 + +```python +from memu import MemoryService + +service = MemoryService( + llm_profiles={ + "default": { + "provider": "openrouter", + "client_backend": "httpx", + "base_url": "https://openrouter.ai", + "api_key": "your_key", + "chat_model": "anthropic/claude-3.5-sonnet", + "embed_model": "openai/text-embedding-3-small", + }, + }, + database_config={"metadata_store": {"provider": "inmemory"}}, +) +``` + +--- + +## 📖 핵심 API + +### `memorize()` — 원본 데이터 구조화 + +memorize + +```python +result = await service.memorize( + resource_url="path/to/file.json", # 로컬 파일 경로 또는 HTTP URL + modality="conversation", # conversation | document | image | video | audio + user={"user_id": "123"}, # 선택: 사용자 또는 에이전트로 범위 한정 +) +# 처리 완료 후 반환: +# { "resource": {...}, "items": [...], "categories": [...], "relations": [...] } +``` + +- 원본 입력을 타입 지정 메모리 아이템으로 변환 +- 수동 태깅 없이 아이템을 분류하고 임베딩 +- 소스 리소스와 아이템–카테고리 관계를 보존 + +--- + +### `retrieve()` — 에이전트 컨텍스트 로드 + +retrieve + +```python +# 검색 전략은 retrieve_config로 서비스에 한 번만 설정합니다: +# MemoryService(retrieve_config={"method": "rag"}) # 벡터 우선 회수 +# MemoryService(retrieve_config={"method": "llm"}) # LLM 순위 회수 +result = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What are their preferences?"}}], + where={"user_id": "123"}, # 범위 필터 +) +# 반환: +# { +# "needs_retrieval": true, +# "original_query": "...", +# "rewritten_query": "...", +# "next_step_query": "...", +# "categories": [...], +# "items": [...], +# "resources": [...] +# } +``` + +| `retrieve_config.method` | 동작 | 비용 | 적합한 용도 | +|--------------------------|----------|------|----------| +| `rag` | 벡터 우선의 카테고리/아이템/리소스 회수. 선택적 LLM 라우팅과 충분성 검사가 기본 활성화 | `route_intention`과 `sufficiency_check`를 비활성화하지 않는 한, 임베딩에 더해 LLM 호출 | 추론을 제어 가능한 빠른 범위 회수 | +| `llm` | LLM이 순위를 매기는 카테고리/아이템/리소스 회수 | 각 계층에서 LLM 순위 매김 | 더 깊은 의미적 순위 매김 | + +--- + +## 💡 예시 워크플로 + +### 항상 학습하는 어시스턴트 +```bash +export OPENAI_API_KEY=your_key +uv run python examples/example_1_conversation_memory.py +``` +선호를 자동으로 추출하고, 관계 모델을 구축하며, 이후 대화에서 관련 컨텍스트를 떠올립니다. + +### 자기 개선하는 에이전트 +```bash +uv run python examples/example_2_skill_extraction.py +``` +에이전트의 행동을 모니터링하고, 성공과 실패의 패턴을 식별하며, 경험으로부터 스킬 가이드를 자동 생성합니다. + +### 멀티모달 컨텍스트 빌더 +```bash +uv run python examples/example_3_multimodal_memory.py +``` +텍스트, 이미지, 문서를 자동으로 상호 참조하여 통합된 메모리 계층으로 묶습니다. + +--- + +## 📊 성능 + +memU는 Locomo 벤치마크의 모든 추론 작업에서 **평균 92.09% 정확도**를 달성합니다. + +benchmark + +자세한 결과 보기: [memU-experiment](https://github.com/NevaMind-AI/memU-experiment) + +--- + +## 🧩 에코시스템 + +| 저장소 | 설명 | +|------------|-------------| +| **[memU](https://github.com/NevaMind-AI/memU)** | 핵심 메모리 파일 시스템 — 수집, 추출, 검색 | +| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | 실시간 동기화와 webhook 트리거를 갖춘 백엔드 | +| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | 메모리를 탐색하고 모니터링하는 비주얼 대시보드 | + +**빠른 링크:** +- 🚀 [MemU Cloud 사용해 보기](https://app.memu.so/quick-start) +- 📚 [API 문서](https://memu.pro/docs) +- 💬 [Discord 커뮤니티](https://discord.com/invite/hQZntfGsbJ) + +--- + +## 🤝 파트너 + +
+ +Ten +OpenAgents +Milvus +xRoute +Jazz +Buddie +Bytebase +LazyLLM +Clawdchat + +
+ +--- + +## 🤝 기여하기 + +```bash +# 포크하고 클론 +git clone https://github.com/YOUR_USERNAME/memU.git +cd memU + +# 개발 의존성 설치 +make install + +# 제출 전 품질 검사 실행 +make check +``` + +자세한 가이드라인은 [CONTRIBUTING.md](../CONTRIBUTING.md)를 참고하세요. + +**필수 조건:** Python 3.13+, [uv](https://github.com/astral-sh/uv), Git + +--- + +## 📄 라이선스 + +[Apache License 2.0](../LICENSE.txt) + +--- + +## 🌍 커뮤니티 + +- **GitHub Issues**: [버그 신고 및 기능 요청](https://github.com/NevaMind-AI/memU/issues) +- **Discord**: [커뮤니티 참여](https://discord.com/invite/hQZntfGsbJ) +- **X (Twitter)**: [@memU_ai 팔로우](https://x.com/memU_ai) +- **연락처**: info@nevamind.ai + +--- + +
+ +⭐ **GitHub에서 스타를** 눌러 새 릴리스 알림을 받으세요! + +
diff --git a/readme/README_zh.md b/readme/README_zh.md new file mode 100644 index 0000000..ab6b74d --- /dev/null +++ b/readme/README_zh.md @@ -0,0 +1,542 @@ +![MemU Banner](../assets/banner.png) + +
+ +# memU + +### 个人记忆,存为文件 + +**快速检索 · 更高准确性 · 更低成本** + +[![PyPI version](https://badge.fury.io/py/memu-py.svg)](https://badge.fury.io/py/memu-py) +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) +[![Discord](https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white)](https://discord.com/invite/hQZntfGsbJ) +[![Twitter](https://img.shields.io/badge/Twitter-Follow-1DA1F2?logo=x&logoColor=white)](https://x.com/memU_ai) + +NevaMind-AI%2FmemU | Trendshift + +**[English](README_en.md) | [中文](README_zh.md) | [日本語](README_ja.md) | [한국어](README_ko.md) | [Español](README_es.md) | [Français](README_fr.md)** + +
+ +--- + +> [!WARNING] +> 🚧 **正在大规模重构** — memU 正在进行重大重构,API、CLI 命令和文档可能随时变动,预计 **2026 年 7 月 15 日**前后趋于稳定。 + +memU 将对话、文档、代码、图片、音频、视频、URL 和工具轨迹编译成人类可读的 Markdown 文件(`INDEX.md`、`MEMORY.md`、`SKILL.md`)。智能体遍历文件树,只加载当下需要的记忆——而不是每次扫描全部内容,或把冗长历史塞进 prompt: + +- **`INDEX.md`** —— 全局地图:类别、文件与摘要,让智能体先知道该去哪里找 +- **`MEMORY.md`** —— 画像、偏好、目标、事实,以及从来源数据中提取的关键事件 +- **`SKILL.md`** —— 从工具轨迹中自动提取、并在每次 `memorize()` 时持续精炼的可复用工具模式与工作流 + +```txt +workspace/ +├── INDEX.md ← 全局地图:类别、文件与摘要 +├── MEMORY.md ← 画像、偏好、目标与关键事件 +└── skill/ + ├── {skill_name}/ + │ └── SKILL.md ← 一项学到的技能或工具模式 + └── {another_skill}/ + └── SKILL.md +``` + +与把所有内容塞进 prompt 相比,memU 有三个核心优势: + +- **快速检索** —— 直接定位相关文件夹并排序,而不是每次都扫描全部记忆。 +- **更高准确性** —— 按用户、任务或会话限定范围,并将每条记忆追溯到其原始来源。 +- **更低成本** —— 检索紧凑、有范围的上下文,而不是把冗长历史反复注入每次 prompt。 +- **可审计** —— 人类可读的文件树,可检查、编辑,并通过自有存储与 LLM 提供商路由。 + +--- + +## 🔄 工作原理 + +可以把它看成两个文件系统操作:把原始来源**写入**有组织的记忆,再把正确的文件**读回**给智能体。 + +``` +WRITE — memorize() READ — retrieve() +────────────────────────────────────────────── ────────────────────────────────────────────── +raw files → extract → files + folders query → walk folders → ranked files +───────────── ───────── ────────────── ───── ──────────── ───────────── +chat logs → parse → profile / event items user / task query +documents / URLs → facts → knowledge / skill items │ +images / video → caption → resources + summaries ├─ route + scope → relevant folders (categories) +audio → transcribe→ event / knowledge items ├─ rank by relevance → matching files (items) +tool logs → mine → tool / skill items └─ trace to source → original resources +``` + +**写入文件系统(`memorize`)** + +1. **摄入(Ingest)** — 把每个来源作为一个 `Resource`(原始文件)存下,记录其模态与来源位置 +2. **预处理(Preprocess)** — 解析文本、为图片/视频生成描述、转写音频,并规范化输入 +3. **提取(Extract)** — 把原始内容转成带类型的 `RecallEntry` 记录(文件):profile、event、knowledge、behavior、skill 或 tool 记忆 +4. **组织(Organize)** — 把记忆项归入 `RecallFile` 文件夹(一个记忆分类,或一个技能),交叉关联、向量化,并汇总成可浏览的树 +5. **持久化(Persist)** — 通过所配置的后端写入记录、关系、向量与文件夹摘要 + +**从文件系统读取(`retrieve`)** + +6. **检索(Retrieve)** — 在文件夹中导航,仅返回与当前用户、智能体、会话或任务相关的文件 + +--- + +## 🗂️ 记忆文件系统 + +memU 的主要产出是一棵可浏览的记忆树——文件夹、文件,以及它们背后的来源素材——通过仓储契约持久化,并以字典形式从 `memorize()` 和 `retrieve()` 返回。 + +```txt +RecallFile ← 文件夹:带演进式摘要的主题 +├── name, track (memory | skill), description, content +├── embedding +└── RecallEntry[] ← 文件:带类型的原子记忆 + ├── memory_type: profile | event | knowledge | behavior | skill | tool + ├── summary, extra, happened_at, embedding + └── Resource ← 来源:这条记忆所来自的原始文件 + └── url, modality, local_path, caption, embedding +``` + +| 记录 | 文件系统角色 | 用途 | +|--------|------------------|---------| +| `RecallFile` | **文件夹** — 归集相关记忆(或承载一个技能)并维护主题级摘要;`track` 字段标记为 `memory` 或 `skill` | 为宽泛查询加载紧凑上下文 | +| `RecallEntry` | **文件** — 带类型的原子记忆,含摘要与可选元数据 | 注入精确的事实、偏好、事件、技能与工具模式 | +| `Resource` | **来源素材** — 记忆背后的原始文件,附带描述/文本 | 把上下文追溯回它的来源 | +| `RecallFileEntry` | **链接** — 把记忆项归档到文件夹下的边 | 在不重新处理来源的情况下导航相关记忆 | + +这为智能体提供了一个稳定的记忆文件系统:原始来源只需摄入一次,之后便可请求限定范围、经过排序的文件,而无需重读每一份来源素材。 + +--- + +## 🧩 memU 构建了什么 + +文件系统的每一层都以结构化记录的形式存储: + +| 层 | 代表什么 | 智能体为何使用 | +|-------|--------------------|-------------------| +| **RecallFile** | 自动生成的文件夹:带演进式摘要的主题(或一个技能) | 先加载高层上下文,再深入细节 | +| **RecallEntry** | 文件:带类型与摘要的原子结构化记忆 | 注入精确的事实、偏好、事件、技能与工具模式 | +| **Resource** | 文件背后的来源素材:对话、文档、图片、视频、音频、URL 或文件 | 把记忆追溯回其来源 | +| **RecallFileEntry** | 把记忆项归档到文件夹下的链接 | 在不重新处理来源的情况下导航相关记忆 | +| **Embedding** | 覆盖文件夹、文件与来源的向量索引 | 以低延迟检索相关上下文 | + +`memorize()` 输出示例: + +```json +{ + "resource": { + "id": "res_01", + "url": "files/launch-meeting.mp4", + "modality": "video", + "caption": "A product planning discussion about onboarding and launch risks." + }, + "items": [ + { + "id": "mem_01", + "memory_type": "event", + "summary": "The team decided to simplify onboarding before the next launch review." + }, + { + "id": "mem_02", + "memory_type": "profile", + "summary": "The user prefers concise implementation plans with explicit verification steps." + }, + { + "id": "mem_03", + "memory_type": "tool", + "summary": "Use repository-wide search before editing configuration files to avoid missing duplicated settings." + } + ], + "categories": [ + { + "id": "cat_01", + "name": "product_goals", + "summary": "Current launch priorities, onboarding decisions, and unresolved risks." + } + ], + "relations": [ + { "item_id": "mem_01", "category_id": "cat_01" } + ] +} +``` + +随后,智能体可以调用 `retrieve()` 获取一份限定范围、经过排序的上下文负载: + +```python +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What context matters for this launch task?"}}], + where={"user_id": "123"}, +) +``` + +--- + +## ⭐️ 给仓库点个 Star + + + +如果你觉得 memU 有用或有意思,欢迎在 GitHub 上点一个 Star ⭐️,我们将不胜感激。 + +--- + +## ✨ 核心特性 + +| 能力 | 说明 | +|------------|-------------| +| 🗂️ **多模态摄入** | 把对话、文档、图片、视频、音频、URL、日志与本地文件写入记忆 | +| 📁 **记忆文件系统** | 持久化文件夹(类别)、文件(记忆项)、来源素材、链接、摘要与向量 | +| 🧠 **带类型的记忆提取** | 从原始来源提取 profile、event、knowledge、behavior、skill 与 tool 记忆 | +| 🧭 **自组织文件夹** | 自动构建类别、链接、摘要与向量,无需手工打标签 | +| 🤖 **面向智能体的检索** | 读取限定范围、经过排序的上下文,可注入任意智能体工作流 | +| 🧱 **可插拔存储** | 使用 in-memory、SQLite 或 Postgres 后端,共享同一套仓储契约 | +| 🔀 **基于 Profile 的 LLM 路由** | 通过可配置的 LLM profile 路由对话、向量、视觉与转写任务 | + +--- + +## 🎯 应用场景 + +### 1. **对话记忆** +*把聊天记录转化为用户偏好、目标、事件与关系上下文。* + +```python +await service.memorize( + resource_url="examples/resources/conversations/conv1.json", + modality="conversation", + user={"user_id": "123"}, +) + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What should I remember about this user?"}}], + where={"user_id": "123"}, +) +``` + +### 2. **面向编码智能体的工作区上下文** +*把文档、PR 说明、日志与设计决策转化为可复用的项目记忆。* + +```python +await service.memorize(resource_url="docs/architecture.md", modality="document") +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "How should I structure this module?"}}], +) +``` + +### 3. **多模态知识层** +*从文档、截图、图片、视频与语音笔记中提取可检索的事实。* + +```python +await service.memorize(resource_url="examples/resources/docs/doc1.txt", modality="document") +await service.memorize(resource_url="examples/resources/images/image1.png", modality="image") +# Audio is supported for your own .mp3/.wav/.m4a files. +await service.memorize(resource_url="meeting-audio.mp3", modality="audio") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What matters for the next research plan?"}}], +) +``` + +### 4. **工具与智能体学习** +*把执行轨迹转化为工具记忆,告诉未来的智能体何时使用某个工具、应避免哪些错误。* + +```python +await service.memorize(resource_url="examples/resources/logs/log1.txt", modality="document") + +context = await service.retrieve( + queries=[{"role": "user", "content": {"text": "Which tools worked for config editing?"}}], +) +``` + +--- + +## 🗂️ 架构 + +记忆文件系统既层级化到足以浏览,又结构化到足以直接检索: + +structure + +| 层 | 主要职责 | 检索职责 | +|-------|--------------|----------------| +| **Category(文件夹)** | 维护主题级摘要 | 为宽泛查询组装紧凑上下文 | +| **Item(文件)** | 存储带类型的原子记忆 | 加载精确的事实、事件、偏好、技能与工具模式 | +| **Resource(来源)** | 保留来源素材与描述 | 当记忆项/类别摘要不够时召回原始上下文 | + +关于 `MemoryService`、工作流流水线、存储后端与 LLM 路由的运行时视图,参见 [docs/architecture.md](../docs/architecture.md)。 + +--- + +## 🚀 快速开始 + +### 方式一:云端版本 + +👉 **[memu.so](https://memu.so)** — 托管 API,提供托管式摄入、结构化记忆与检索 + +企业部署请联系:**info@nevamind.ai** + +#### Cloud API (v3) + +| Base URL | `https://api.memu.so` | +|----------|----------------------| +| Auth | `Authorization: Bearer ` | + +| Method | Endpoint | Description | +|--------|----------|-------------| +| `POST` | `/api/v3/memory/memorize` | 摄入原始数据并构建结构化记忆 | +| `GET` | `/api/v3/memory/memorize/status/{task_id}` | 查询处理状态 | +| `POST` | `/api/v3/memory/categories` | 列出自动生成的类别 | +| `POST` | `/api/v3/memory/retrieve` | 查询记忆以获取智能体上下文 | + +📚 **[完整 API 文档](https://memu.pro/docs#cloud-version)** + +--- + +### 方式二:自托管 + +#### 安装 + +从本仓库的克隆中安装: + +```bash +uv sync +# 或者,进行完整的开发环境安装: +make install +``` + +或者安装已发布的包: + +```bash +pip install memu-py +``` + +> **环境要求**:Python 3.13+。默认示例使用 OpenAI,请设置 `OPENAI_API_KEY`,或通过 `llm_profiles` 传入其它提供方。 + +**运行内存模式冒烟脚本:** +```bash +export OPENAI_API_KEY=your_key +cd tests +uv run python test_inmemory.py +``` + +**使用 PostgreSQL + pgvector 运行:** +```bash +uv sync --extra postgres +docker run -d --name memu-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=memu \ + -p 5432:5432 \ + pgvector/pgvector:pg16 + +export OPENAI_API_KEY=your_key +export POSTGRES_DSN=postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu +cd tests +uv run python test_postgres.py +``` + +--- + +### 自定义 LLM 与 Embedding 提供方 + +```python +from memu import MemUService + +service = MemUService( + llm_profiles={ + "default": { + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "api_key": "your_key", + "chat_model": "qwen3-max", + "client_backend": "sdk" + }, + "embedding": { + "base_url": "https://api.voyageai.com/v1", + "api_key": "your_key", + "embed_model": "voyage-3.5-lite" + } + }, +) +``` + +--- + +### OpenRouter 集成 + +```python +from memu import MemoryService + +service = MemoryService( + llm_profiles={ + "default": { + "provider": "openrouter", + "client_backend": "httpx", + "base_url": "https://openrouter.ai", + "api_key": "your_key", + "chat_model": "anthropic/claude-3.5-sonnet", + "embed_model": "openai/text-embedding-3-small", + }, + }, + database_config={"metadata_store": {"provider": "inmemory"}}, +) +``` + +--- + +## 📖 核心 API + +### `memorize()` — 结构化原始数据 + +memorize + +```python +result = await service.memorize( + resource_url="path/to/file.json", # 本地文件路径或 HTTP URL + modality="conversation", # conversation | document | image | video | audio + user={"user_id": "123"}, # 可选:限定到某个用户或智能体 +) +# 处理完成后返回: +# { "resource": {...}, "items": [...], "categories": [...], "relations": [...] } +``` + +- 把原始输入转换为带类型的记忆项 +- 自动对记忆项分类并向量化,无需手工打标签 +- 保留来源资源以及记忆项—类别关系 + +--- + +### `retrieve()` — 加载智能体上下文 + +retrieve + +```python +# 检索策略在服务上通过 retrieve_config 一次性设定: +# MemoryService(retrieve_config={"method": "rag"}) # 向量优先召回 +# MemoryService(retrieve_config={"method": "llm"}) # LLM 排序召回 +result = await service.retrieve( + queries=[{"role": "user", "content": {"text": "What are their preferences?"}}], + where={"user_id": "123"}, # 范围过滤 +) +# 返回: +# { +# "needs_retrieval": true, +# "original_query": "...", +# "rewritten_query": "...", +# "next_step_query": "...", +# "categories": [...], +# "items": [...], +# "resources": [...] +# } +``` + +| `retrieve_config.method` | 行为 | 成本 | 适用场景 | +|--------------------------|----------|------|----------| +| `rag` | 向量优先的类别/记忆项/资源召回,默认启用可选的 LLM 路由与充分性检查 | 向量调用加 LLM 调用,除非关闭 `route_intention` 与 `sufficiency_check` | 可控推理下的快速限定召回 | +| `llm` | 由 LLM 排序的类别/记忆项/资源召回 | 每一层都进行 LLM 排序 | 更深的语义排序 | + +--- + +## 💡 示例工作流 + +### 持续学习的助手 +```bash +export OPENAI_API_KEY=your_key +uv run python examples/example_1_conversation_memory.py +``` +自动提取偏好、构建关系模型,并在未来对话中浮现相关上下文。 + +### 自我改进的智能体 +```bash +uv run python examples/example_2_skill_extraction.py +``` +监控智能体的行为,识别成功与失败中的模式,从经验中自动生成技能指南。 + +### 多模态上下文构建器 +```bash +uv run python examples/example_3_multimodal_memory.py +``` +自动交叉关联文本、图片与文档,汇入统一的记忆层。 + +--- + +## 📊 性能 + +memU 在 Locomo 基准的所有推理任务上取得了 **92.09% 的平均准确率**。 + +benchmark + +查看详细结果:[memU-experiment](https://github.com/NevaMind-AI/memU-experiment) + +--- + +## 🧩 生态 + +| 仓库 | 说明 | +|------------|-------------| +| **[memU](https://github.com/NevaMind-AI/memU)** | 核心记忆文件系统 —— 摄入、提取、检索 | +| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | 带实时同步与 webhook 触发的后端 | +| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | 用于浏览与监控记忆的可视化面板 | + +**快速链接:** +- 🚀 [试用 MemU Cloud](https://app.memu.so/quick-start) +- 📚 [API 文档](https://memu.pro/docs) +- 💬 [Discord 社区](https://discord.com/invite/hQZntfGsbJ) + +--- + +## 🤝 合作伙伴 + +
+ +Ten +OpenAgents +Milvus +xRoute +Jazz +Buddie +Bytebase +LazyLLM +Clawdchat + +
+ +--- + +## 🤝 贡献 + +```bash +# Fork 并克隆 +git clone https://github.com/YOUR_USERNAME/memU.git +cd memU + +# 安装开发依赖 +make install + +# 提交前运行质量检查 +make check +``` + +完整指南参见 [CONTRIBUTING.md](../CONTRIBUTING.md)。 + +**前置条件:** Python 3.13+、[uv](https://github.com/astral-sh/uv)、Git + +--- + +## 📄 许可证 + +[Apache License 2.0](../LICENSE.txt) + +--- + +## 🌍 社区 + +- **GitHub Issues**:[报告 bug 与提交功能请求](https://github.com/NevaMind-AI/memU/issues) +- **Discord**:[加入社区](https://discord.com/invite/hQZntfGsbJ) +- **X (Twitter)**:[关注 @memU_ai](https://x.com/memU_ai) +- **联系**:info@nevamind.ai + +--- + +
+ +⭐ **在 GitHub 上给我们点 Star**,第一时间获取新版本通知! + +
diff --git a/scripts/db.py b/scripts/db.py new file mode 100644 index 0000000..689e286 --- /dev/null +++ b/scripts/db.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +"""Programmatic Alembic entry point for the Postgres backend. + +The migration environment is parameterized by a user *scope model* (see +``memu.database.postgres.schema.get_metadata``). The bare ``alembic`` CLI cannot +pass that in, so this wrapper builds the config via +``memu.database.postgres.migration.make_alembic_config`` and drives Alembic's +command API directly. + +The default scope model is ``None`` (the base schema with no scope columns), +which is the schema committed under ``migrations/versions``. + +Usage: + python scripts/db.py revision -m "add foo" # autogenerate a revision + python scripts/db.py upgrade [head] # apply migrations + python scripts/db.py downgrade -1 # revert one revision + python scripts/db.py current # show applied revision + python scripts/db.py history # show revision history + +DSN resolution: --dsn, else $MEMU_DB_DSN, else a localhost default. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +from alembic import command + +from memu.database.postgres.migration import make_alembic_config + +DEFAULT_DSN = "postgresql+psycopg://postgres:postgres@localhost:5432/memu" + + +def _config(dsn: str): + # scope_model=None -> base schema (matches committed baseline revision). + return make_alembic_config(dsn=dsn, scope_model=None) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="MemU Postgres migrations") + parser.add_argument( + "--dsn", + default=os.environ.get("MEMU_DB_DSN", DEFAULT_DSN), + help="SQLAlchemy DSN (default: $MEMU_DB_DSN or localhost memu db)", + ) + sub = parser.add_subparsers(dest="cmd", required=True) + + p_rev = sub.add_parser("revision", help="create a new revision") + p_rev.add_argument("-m", "--message", required=True) + p_rev.add_argument( + "--no-autogenerate", + action="store_true", + help="create an empty revision instead of diffing against the DB", + ) + + p_up = sub.add_parser("upgrade", help="apply migrations") + p_up.add_argument("revision", nargs="?", default="head") + + p_down = sub.add_parser("downgrade", help="revert migrations") + p_down.add_argument("revision") + + sub.add_parser("current", help="show current revision") + + p_hist = sub.add_parser("history", help="show revision history") + p_hist.add_argument("-v", "--verbose", action="store_true") + + args = parser.parse_args(argv) + cfg = _config(args.dsn) + + if args.cmd == "revision": + command.revision(cfg, message=args.message, autogenerate=not args.no_autogenerate) + elif args.cmd == "upgrade": + command.upgrade(cfg, args.revision) + elif args.cmd == "downgrade": + command.downgrade(cfg, args.revision) + elif args.cmd == "current": + command.current(cfg, verbose=True) + elif args.cmd == "history": + command.history(cfg, verbose=args.verbose) + else: # pragma: no cover - argparse guards this + parser.error(f"unknown command: {args.cmd}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..130f3b6 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,18 @@ +[flake8] +max-line-length = 120 +extend-ignore = E203,W503,E501 +exclude = + .git, + __pycache__, + .venv, + venv, + build, + dist, + *.egg-info, + .pytest_cache, + .mypy_cache +per-file-ignores = + */test*.py:E402 + **/test_*.py:E402 + **/tests.py:E402 + **/quick_memory_test.py:E402 diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..c45f563 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,15 @@ +use pyo3::prelude::*; + +#[pyfunction] +fn hello_from_bin() -> String { + "Hello from memu!".to_string() +} + +/// A Python module implemented in Rust. The name of this function must match +/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to +/// import the module. +#[pymodule] +fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(hello_from_bin, m)?)?; + Ok(()) +} diff --git a/src/memu/__init__.py b/src/memu/__init__.py new file mode 100644 index 0000000..38772b8 --- /dev/null +++ b/src/memu/__init__.py @@ -0,0 +1,9 @@ +from memu._core import hello_from_bin +from memu.app.service import MemoryService + +# Public alias used in documentation examples +MemUService = MemoryService + + +def _rust_entry() -> str: + return hello_from_bin() diff --git a/src/memu/__main__.py b/src/memu/__main__.py new file mode 100644 index 0000000..b5ccaa6 --- /dev/null +++ b/src/memu/__main__.py @@ -0,0 +1,7 @@ +"""Allow ``python -m memu`` to invoke the CLI (used by the npm launcher).""" + +import sys + +from memu.cli import main + +sys.exit(main()) diff --git a/src/memu/_core.pyi b/src/memu/_core.pyi new file mode 100644 index 0000000..d52129e --- /dev/null +++ b/src/memu/_core.pyi @@ -0,0 +1 @@ +def hello_from_bin() -> str: ... diff --git a/src/memu/app/__init__.py b/src/memu/app/__init__.py new file mode 100644 index 0000000..039e922 --- /dev/null +++ b/src/memu/app/__init__.py @@ -0,0 +1,37 @@ +from memu.app.service import MemoryService +from memu.app.settings import ( + BlobConfig, + DatabaseConfig, + DefaultUserModel, + EmbeddingConfig, + EmbeddingProfilesConfig, + LLMConfig, + LLMProfilesConfig, + MemorizeConfig, + RetrieveConfig, + UserConfig, +) +from memu.workflow.runner import ( + LocalWorkflowRunner, + WorkflowRunner, + register_workflow_runner, + resolve_workflow_runner, +) + +__all__ = [ + "BlobConfig", + "DatabaseConfig", + "DefaultUserModel", + "EmbeddingConfig", + "EmbeddingProfilesConfig", + "LLMConfig", + "LLMProfilesConfig", + "LocalWorkflowRunner", + "MemorizeConfig", + "MemoryService", + "RetrieveConfig", + "UserConfig", + "WorkflowRunner", + "register_workflow_runner", + "resolve_workflow_runner", +] diff --git a/src/memu/app/client_pool.py b/src/memu/app/client_pool.py new file mode 100644 index 0000000..9e84518 --- /dev/null +++ b/src/memu/app/client_pool.py @@ -0,0 +1,63 @@ +"""Generic lazy client pool keyed by profile name. + +``MemoryService`` needs three structurally identical caches: one each for the +chat (LLM), vision (VLM) and embedding clients. Each maps a profile name to a +lazily-built, cached client. This pool factors out that duplicated bookkeeping +so adding a fourth capability (e.g. rerank/STT) is a single instantiation rather +than another copy of the get/cache/build dance. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping + + +class ClientPool[TConfig, TClient]: + """Lazily build and cache one client per named profile. + + Args: + profiles: Mapping of profile name -> config object. + builder: Factory turning a config into a concrete client. + label: Human-readable capability name used in error messages + (e.g. ``"llm"``, ``"vlm"``, ``"embedding"``). + default_profile: Profile name used when ``get()`` is called with ``None``. + """ + + def __init__( + self, + *, + profiles: Mapping[str, TConfig], + builder: Callable[[TConfig], TClient], + label: str, + default_profile: str = "default", + ) -> None: + self._profiles = profiles + self._builder = builder + self._label = label + self._default_profile = default_profile + self._cache: dict[str, TClient] = {} + + def config(self, profile: str | None = None) -> TConfig | None: + """Return the config for ``profile`` (or the default), if present.""" + return self._profiles.get(profile or self._default_profile) + + def get(self, profile: str | None = None) -> TClient: + """Return the cached client for ``profile``, building it on first use. + + Raises: + KeyError: if no profile with that name is configured. + """ + name = profile or self._default_profile + cached = self._cache.get(name) + if cached is not None: + return cached + cfg = self._profiles.get(name) + if cfg is None: + msg = f"Unknown {self._label} profile '{name}'" + raise KeyError(msg) + client = self._builder(cfg) + self._cache[name] = client + return client + + +__all__ = ["ClientPool"] diff --git a/src/memu/app/crud.py b/src/memu/app/crud.py new file mode 100644 index 0000000..4ca3571 --- /dev/null +++ b/src/memu/app/crud.py @@ -0,0 +1,792 @@ +from __future__ import annotations + +import asyncio +import json +import logging +from collections.abc import Awaitable, Callable, Mapping +from typing import TYPE_CHECKING, Any, cast, get_args + +from pydantic import BaseModel + +from memu.database.models import EntryType, RecallFile +from memu.prompts.category_patch import CATEGORY_PATCH_PROMPT +from memu.workflow.step import WorkflowState, WorkflowStep + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from memu.app.service import Context + from memu.app.settings import PatchConfig + from memu.database.interfaces import Database + + +class CRUDMixin: + if TYPE_CHECKING: + _run_workflow: Callable[..., Awaitable[WorkflowState]] + _get_context: Callable[[], Context] + _get_database: Callable[[], Database] + _get_step_llm_client: Callable[[Mapping[str, Any] | None], Any] + _get_step_embedding_client: Callable[[Mapping[str, Any] | None], Any] + _get_llm_client: Callable[..., Any] + _model_dump_without_embeddings: Callable[[BaseModel], dict[str, Any]] + _extract_json_blob: Callable[[str], str] + _escape_prompt_value: Callable[[str], str] + user_model: type[BaseModel] + patch_config: PatchConfig + _ensure_categories_ready: Callable[[Context, Database, Mapping[str, Any] | None], Awaitable[None]] + + async def list_recall_entries( + self, + where: dict[str, Any] | None = None, + ) -> dict[str, Any]: + ctx = self._get_context() + store = self._get_database() + where_filters = self._normalize_where(where) + + state: WorkflowState = { + "ctx": ctx, + "store": store, + "where": where_filters, + } + + result = await self._run_workflow("crud_list_recall_entries", state) + response = cast(dict[str, Any] | None, result.get("response")) + if response is None: + msg = "List memory items workflow failed to produce a response" + raise RuntimeError(msg) + return response + + async def list_recall_files( + self, + where: dict[str, Any] | None = None, + ) -> dict[str, Any]: + ctx = self._get_context() + store = self._get_database() + where_filters = self._normalize_where(where) + + state: WorkflowState = { + "ctx": ctx, + "store": store, + "where": where_filters, + } + result = await self._run_workflow("crud_list_recall_files", state) + response = cast(dict[str, Any] | None, result.get("response")) + if response is None: + msg = "List memory categories workflow failed to produce a response" + raise RuntimeError(msg) + return response + + async def clear_memory( + self, + where: dict[str, Any] | None = None, + ) -> dict[str, Any]: + ctx = self._get_context() + store = self._get_database() + where_filters = self._normalize_where(where) + + state: WorkflowState = { + "ctx": ctx, + "store": store, + "where": where_filters, + } + + result = await self._run_workflow("crud_clear_memory", state) + response = cast(dict[str, Any] | None, result.get("response")) + if response is None: + msg = "Clear memory workflow failed to produce a response" + raise RuntimeError(msg) + return response + + def _build_list_recall_entries_workflow(self) -> list[WorkflowStep]: + steps = [ + WorkflowStep( + step_id="list_recall_entries", + role="read_memories", + handler=self._crud_list_recall_entries, + requires={"ctx", "store", "where"}, + produces={"items"}, + capabilities={"db"}, + ), + WorkflowStep( + step_id="build_response", + role="emit", + handler=self._crud_build_list_items_response, + requires={"items", "ctx", "store"}, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + @staticmethod + def _list_list_memories_initial_keys() -> set[str]: + return { + "ctx", + "store", + "where", + } + + def _build_list_recall_files_workflow(self) -> list[WorkflowStep]: + steps = [ + WorkflowStep( + step_id="list_recall_files", + role="read_categories", + handler=self._crud_list_recall_files, + requires={"ctx", "store", "where"}, + produces={"categories"}, + capabilities={"db"}, + ), + WorkflowStep( + step_id="build_response", + role="emit", + handler=self._crud_build_list_categories_response, + requires={"categories", "ctx", "store"}, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + def _build_clear_memory_workflow(self) -> list[WorkflowStep]: + steps = [ + WorkflowStep( + step_id="clear_memory_relations", + role="delete_memories", + handler=self._crud_clear_memory_relations, + requires={"ctx", "store", "where"}, + produces={"deleted_relations"}, + capabilities={"db"}, + ), + WorkflowStep( + step_id="clear_recall_files", + role="delete_memories", + handler=self._crud_clear_recall_files, + requires={"ctx", "store", "where"}, + produces={"deleted_categories"}, + capabilities={"db"}, + ), + WorkflowStep( + step_id="clear_recall_entries", + role="delete_memories", + handler=self._crud_clear_recall_entries, + requires={"ctx", "store", "where"}, + produces={"deleted_items"}, + capabilities={"db"}, + ), + WorkflowStep( + step_id="clear_memory_resources", + role="delete_memories", + handler=self._crud_clear_memory_resources, + requires={"ctx", "store", "where"}, + produces={"deleted_resources"}, + capabilities={"db"}, + ), + WorkflowStep( + step_id="build_response", + role="emit", + handler=self._crud_build_clear_memory_response, + requires={ + "ctx", + "store", + "deleted_relations", + "deleted_categories", + "deleted_items", + "deleted_resources", + }, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + @staticmethod + def _list_clear_memories_initial_keys() -> set[str]: + return { + "ctx", + "store", + "where", + } + + def _normalize_where(self, where: Mapping[str, Any] | None) -> dict[str, Any]: + """Validate and clean the `where` scope filters against the configured user model.""" + if not where: + return {} + + valid_fields = set(getattr(self.user_model, "model_fields", {}).keys()) + cleaned: dict[str, Any] = {} + + for raw_key, value in where.items(): + if value is None: + continue + field = raw_key.split("__", 1)[0] + if field not in valid_fields: + msg = f"Unknown filter field '{field}' for current user scope" + raise ValueError(msg) + cleaned[raw_key] = value + + return cleaned + + def _crud_list_recall_entries(self, state: WorkflowState, step_context: Any) -> WorkflowState: + where_filters = state.get("where") or {} + store = state["store"] + items = store.recall_entry_repo.list_items(where_filters) + state["items"] = items + return state + + def _crud_list_recall_files(self, state: WorkflowState, step_context: Any) -> WorkflowState: + where_filters = state.get("where") or {} + store = state["store"] + # Lists memory-track files only; skill-track RecallFiles (ADR 0006) are excluded. + categories = store.recall_file_repo.list_categories({**where_filters, "track": "memory"}) + state["categories"] = categories + return state + + def _crud_build_list_items_response(self, state: WorkflowState, step_context: Any) -> WorkflowState: + items = state["items"] + items_list = [self._model_dump_without_embeddings(item) for item in items.values()] + response = { + "items": items_list, + } + state["response"] = response + return state + + def _crud_build_list_categories_response(self, state: WorkflowState, step_context: Any) -> WorkflowState: + categories = state["categories"] + categories_list = [self._model_dump_without_embeddings(category) for category in categories.values()] + response = { + "categories": categories_list, + } + state["response"] = response + return state + + def _crud_clear_memory_relations(self, state: WorkflowState, step_context: Any) -> WorkflowState: + where_filters = state.get("where") or {} + store = state["store"] + deleted = store.recall_file_entry_repo.clear_relations(where_filters) + state["deleted_relations"] = deleted + return state + + def _crud_clear_recall_files(self, state: WorkflowState, step_context: Any) -> WorkflowState: + where_filters = state.get("where") or {} + store = state["store"] + # Segments hang off files (ADR 0007 L2); clear them alongside their categories. + store.recall_file_segment_repo.clear_segments(where_filters) + deleted = store.recall_file_repo.clear_categories(where_filters) + state["deleted_categories"] = deleted + return state + + def _crud_clear_recall_entries(self, state: WorkflowState, step_context: Any) -> WorkflowState: + where_filters = state.get("where") or {} + store = state["store"] + deleted = store.recall_entry_repo.clear_items(where_filters) + state["deleted_items"] = deleted + return state + + def _crud_clear_memory_resources(self, state: WorkflowState, step_context: Any) -> WorkflowState: + where_filters = state.get("where") or {} + store = state["store"] + deleted = store.resource_repo.clear_resources(where_filters) + state["deleted_resources"] = deleted + return state + + def _crud_build_clear_memory_response(self, state: WorkflowState, step_context: Any) -> WorkflowState: + deleted_categories = state.get("deleted_categories", {}) + deleted_items = state.get("deleted_items", {}) + deleted_resources = state.get("deleted_resources", {}) + deleted_relations = state.get("deleted_relations", []) + response = { + "deleted_categories": [self._model_dump_without_embeddings(cat) for cat in deleted_categories.values()], + "deleted_items": [self._model_dump_without_embeddings(item) for item in deleted_items.values()], + "deleted_resources": [self._model_dump_without_embeddings(res) for res in deleted_resources.values()], + "deleted_relations": [rel.model_dump() for rel in deleted_relations], + } + state["response"] = response + return state + + async def create_recall_entry( + self, + *, + memory_type: EntryType, + memory_content: str, + recall_files: list[str], + user: dict[str, Any] | None = None, + propagate: bool = True, + ) -> dict[str, Any]: + if memory_type not in get_args(EntryType): + msg = f"Invalid memory type: '{memory_type}', must be one of {get_args(EntryType)}" + raise ValueError(msg) + + ctx = self._get_context() + store = self._get_database() + user_scope = self.user_model(**user).model_dump() if user is not None else None + await self._ensure_categories_ready(ctx, store, user_scope) + + state: WorkflowState = { + "memory_payload": { + "type": memory_type, + "content": memory_content, + "categories": recall_files, + }, + "ctx": ctx, + "store": store, + "category_ids": list(ctx.category_ids), + "user": user_scope, + "propagate": propagate, + } + + result = await self._run_workflow("patch_create", state) + response = cast(dict[str, Any] | None, result.get("response")) + if response is None: + msg = "Create memory item workflow failed to produce a response" + raise RuntimeError(msg) + return response + + async def update_recall_entry( + self, + *, + memory_id: str, + memory_type: EntryType | None = None, + memory_content: str | None = None, + recall_files: list[str] | None = None, + user: dict[str, Any] | None = None, + propagate: bool = True, + ) -> dict[str, Any]: + if all((memory_type is None, memory_content is None, recall_files is None)): + msg = "At least one of memory type, memory content, or memory categories is required for UPDATE operation" + raise ValueError(msg) + if memory_type and memory_type not in get_args(EntryType): + msg = f"Invalid memory type: '{memory_type}', must be one of {get_args(EntryType)}" + raise ValueError(msg) + + ctx = self._get_context() + store = self._get_database() + user_scope = self.user_model(**user).model_dump() if user is not None else None + await self._ensure_categories_ready(ctx, store, user_scope) + + state: WorkflowState = { + "memory_id": memory_id, + "memory_payload": { + "type": memory_type, + "content": memory_content, + "categories": recall_files, + }, + "ctx": ctx, + "store": store, + "category_ids": list(ctx.category_ids), + "user": user_scope, + "propagate": propagate, + } + + result = await self._run_workflow("patch_update", state) + response = cast(dict[str, Any] | None, result.get("response")) + if response is None: + msg = "Update memory item workflow failed to produce a response" + raise RuntimeError(msg) + return response + + async def delete_recall_entry( + self, + *, + memory_id: str, + user: dict[str, Any] | None = None, + propagate: bool = True, + ) -> dict[str, Any]: + ctx = self._get_context() + store = self._get_database() + user_scope = self.user_model(**user).model_dump() if user is not None else None + await self._ensure_categories_ready(ctx, store, user_scope) + + state: WorkflowState = { + "memory_id": memory_id, + "ctx": ctx, + "store": store, + "category_ids": list(ctx.category_ids), + "user": user_scope, + "propagate": propagate, + } + + result = await self._run_workflow("patch_delete", state) + response = cast(dict[str, Any] | None, result.get("response")) + if response is None: + msg = "Delete memory item workflow failed to produce a response" + raise RuntimeError(msg) + return response + + def _build_create_recall_entry_workflow(self) -> list[WorkflowStep]: + steps = [ + WorkflowStep( + step_id="create_recall_entry", + role="patch", + handler=self._patch_create_recall_entry, + requires={"memory_payload", "ctx", "store", "user"}, + produces={"recall_entry", "category_updates"}, + capabilities={"db", "llm"}, + config={"embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="persist_index", + role="persist", + handler=self._patch_persist_and_index, + requires={"category_updates", "ctx", "store"}, + produces={"categories"}, + capabilities={"db", "llm"}, + config={"chat_llm_profile": "default"}, + ), + WorkflowStep( + step_id="build_response", + role="emit", + handler=self._patch_build_response, + requires={"recall_entry", "category_updates", "ctx", "store"}, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + @staticmethod + def _list_create_recall_entry_initial_keys() -> set[str]: + return { + "memory_payload", + "ctx", + "store", + "user", + } + + def _build_update_recall_entry_workflow(self) -> list[WorkflowStep]: + steps = [ + WorkflowStep( + step_id="update_recall_entry", + role="patch", + handler=self._patch_update_recall_entry, + requires={"memory_id", "memory_payload", "ctx", "store", "user"}, + produces={"recall_entry", "category_updates"}, + capabilities={"db", "llm"}, + config={"embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="persist_index", + role="persist", + handler=self._patch_persist_and_index, + requires={"category_updates", "ctx", "store"}, + produces={"categories"}, + capabilities={"db", "llm"}, + config={"chat_llm_profile": "default"}, + ), + WorkflowStep( + step_id="build_response", + role="emit", + handler=self._patch_build_response, + requires={"recall_entry", "category_updates", "ctx", "store"}, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + @staticmethod + def _list_update_recall_entry_initial_keys() -> set[str]: + return { + "memory_id", + "memory_payload", + "ctx", + "store", + "user", + } + + def _build_delete_recall_entry_workflow(self) -> list[WorkflowStep]: + steps = [ + WorkflowStep( + step_id="delete_recall_entry", + role="patch", + handler=self._patch_delete_recall_entry, + requires={"memory_id", "ctx", "store", "user"}, + produces={"recall_entry", "category_updates"}, + capabilities={"db"}, + ), + WorkflowStep( + step_id="persist_index", + role="persist", + handler=self._patch_persist_and_index, + requires={"category_updates", "ctx", "store"}, + produces={"categories"}, + capabilities={"db", "llm"}, + config={"chat_llm_profile": "default"}, + ), + WorkflowStep( + step_id="build_response", + role="emit", + handler=self._patch_build_response, + requires={"recall_entry", "category_updates", "ctx", "store"}, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + @staticmethod + def _list_delete_recall_entry_initial_keys() -> set[str]: + return { + "memory_id", + "ctx", + "store", + "user", + } + + async def _patch_create_recall_entry(self, state: WorkflowState, step_context: Any) -> WorkflowState: + memory_payload = state["memory_payload"] + ctx = state["ctx"] + store = state["store"] + user = state["user"] + propagate = state["propagate"] + category_memory_updates: dict[str, tuple[Any, Any]] = {} + + embed_payload = [memory_payload["content"]] + content_embedding = (await self._get_step_embedding_client(step_context).embed(embed_payload))[0] + + item = store.recall_entry_repo.create_item( + memory_type=memory_payload["type"], + summary=memory_payload["content"], + embedding=content_embedding, + user_data=dict(user or {}), + ) + cat_names = memory_payload["categories"] + mapped_cat_ids = self._map_category_names_to_ids(cat_names, ctx) + for cid in mapped_cat_ids: + store.recall_file_entry_repo.link_item_category(item.id, cid, user_data=dict(user or {})) + if propagate: + category_memory_updates[cid] = (None, memory_payload["content"]) + + state.update({ + "recall_entry": item, + "category_updates": category_memory_updates, + }) + return state + + async def _patch_update_recall_entry(self, state: WorkflowState, step_context: Any) -> WorkflowState: + memory_id = state["memory_id"] + memory_payload = state["memory_payload"] + ctx = state["ctx"] + store = state["store"] + user = state["user"] + propagate = state["propagate"] + category_memory_updates: dict[str, tuple[Any, Any]] = {} + + item = store.recall_entry_repo.get_item(memory_id) + if not item: + msg = f"Memory item with id {memory_id} not found" + raise ValueError(msg) + old_content = item.summary + old_item_categories = store.recall_file_entry_repo.get_item_categories(memory_id) + mapped_old_cat_ids = [cat.category_id for cat in old_item_categories] + + if memory_payload["content"]: + embed_payload = [memory_payload["content"]] + content_embedding = (await self._get_step_embedding_client(step_context).embed(embed_payload))[0] + else: + content_embedding = None + + if memory_payload["type"] or memory_payload["content"]: + item = store.recall_entry_repo.update_item( + item_id=memory_id, + memory_type=memory_payload["type"], + summary=memory_payload["content"], + embedding=content_embedding, + ) + self._reconcile_update_categories( + memory_id=memory_id, + new_cat_names=memory_payload["categories"], + mapped_old_cat_ids=mapped_old_cat_ids, + content_changed=bool(memory_payload["content"]), + old_content=old_content, + new_summary=item.summary, + ctx=ctx, + store=store, + user=user, + propagate=propagate, + category_memory_updates=category_memory_updates, + ) + + state.update({ + "recall_entry": item, + "category_updates": category_memory_updates, + }) + return state + + def _reconcile_update_categories( + self, + *, + memory_id: str, + new_cat_names: list[str] | None, + mapped_old_cat_ids: list[str], + content_changed: bool, + old_content: Any, + new_summary: Any, + ctx: Any, + store: Database, + user: dict[str, Any] | None, + propagate: bool, + category_memory_updates: dict[str, tuple[Any, Any]], + ) -> None: + """Sync an item's category links for an UPDATE. + + ``new_cat_names is None`` means the caller omitted categories, so existing + links are left untouched (an empty list, by contrast, clears all links). + """ + if new_cat_names is None: + if propagate and content_changed: + for cid in mapped_old_cat_ids: + category_memory_updates[cid] = (old_content, new_summary) + return + + mapped_new_cat_ids = self._map_category_names_to_ids(new_cat_names, ctx) + old_set, new_set = set(mapped_old_cat_ids), set(mapped_new_cat_ids) + for cid in old_set - new_set: + store.recall_file_entry_repo.unlink_item_category(memory_id, cid) + if propagate: + category_memory_updates[cid] = (old_content, None) + for cid in new_set - old_set: + store.recall_file_entry_repo.link_item_category(memory_id, cid, user_data=dict(user or {})) + if propagate: + category_memory_updates[cid] = (None, new_summary) + if propagate and content_changed: + for cid in old_set & new_set: + category_memory_updates[cid] = (old_content, new_summary) + + async def _patch_delete_recall_entry(self, state: WorkflowState, step_context: Any) -> WorkflowState: + memory_id = state["memory_id"] + store = state["store"] + propagate = state["propagate"] + category_memory_updates: dict[str, tuple[Any, Any]] = {} + + item = store.recall_entry_repo.get_item(memory_id) + if not item: + msg = f"Memory item with id {memory_id} not found" + raise ValueError(msg) + item_categories = store.recall_file_entry_repo.get_item_categories(memory_id) + if propagate: + for cat in item_categories: + category_memory_updates[cat.category_id] = (item.summary, None) + # Remove the item's category relations first so deleting the item never + # leaves orphan edges pointing at a non-existent item. + store.recall_file_entry_repo.unlink_item(memory_id) + store.recall_entry_repo.delete_item(memory_id) + + state.update({ + "recall_entry": item, + "category_updates": category_memory_updates, + }) + return state + + async def _patch_persist_and_index(self, state: WorkflowState, step_context: Any) -> WorkflowState: + llm_client = self._get_step_llm_client(step_context) + await self._patch_category_summaries( + state.get("category_updates", {}), + ctx=state["ctx"], + store=state["store"], + llm_client=llm_client, + ) + return state + + def _patch_build_response(self, state: WorkflowState, step_context: Any) -> WorkflowState: + store = state["store"] + item = self._model_dump_without_embeddings(state["recall_entry"]) + category_updates_ids = list(state.get("category_updates", {}).keys()) + category_updates = [ + self._model_dump_without_embeddings(store.recall_file_repo.categories[c]) for c in category_updates_ids + ] + response = { + "recall_entry": item, + "category_updates": category_updates, + } + state["response"] = response + return state + + def _map_category_names_to_ids(self, names: list[str], ctx: Context) -> list[str]: + if not names: + return [] + mapped: list[str] = [] + seen: set[str] = set() + for name in names: + key = name.strip().lower() + cid = ctx.category_name_to_id.get(key) + if cid and cid not in seen: + mapped.append(cid) + seen.add(cid) + return mapped + + async def _patch_category_summaries( + self, + updates: dict[str, tuple[str | None, str | None]], + ctx: Context, + store: Database, + llm_client: Any | None = None, + ) -> None: + if not updates: + return + tasks = [] + target_ids: list[str] = [] + client = llm_client or self._get_llm_client() + for cid, (content_before, content_after) in updates.items(): + cat = store.recall_file_repo.categories.get(cid) + if not cat or (not content_before and not content_after): + continue + prompt = self._build_category_patch_prompt( + category=cat, content_before=content_before, content_after=content_after + ) + tasks.append(client.chat(prompt)) + target_ids.append(cid) + if not tasks: + return + patches = await asyncio.gather(*tasks) + for cid, patch in zip(target_ids, patches, strict=True): + need_update, summary = self._parse_category_patch_response(patch) + if not need_update: + continue + cat = store.recall_file_repo.categories.get(cid) + store.recall_file_repo.update_category( + category_id=cid, + content=summary.strip(), + ) + + def _build_category_patch_prompt( + self, *, category: RecallFile, content_before: str | None, content_after: str | None + ) -> str: + if content_before and content_after: + update_content = "\n".join([ + "The memory content before:", + content_before, + "The memory content after:", + content_after, + ]) + elif content_before: + update_content = "\n".join([ + "This memory content is discarded:", + content_before, + ]) + elif content_after: + update_content = "\n".join([ + "This memory content is newly added:", + content_after, + ]) + original_content = category.content or "" + prompt = CATEGORY_PATCH_PROMPT + return prompt.format( + category=self._escape_prompt_value(category.name), + original_content=self._escape_prompt_value(original_content or ""), + update_content=self._escape_prompt_value(update_content or ""), + ) + + def _parse_category_patch_response(self, response: str) -> tuple[bool, str]: + try: + data = json.loads(response) + except (json.JSONDecodeError, TypeError): + return False, "" + if not isinstance(data, dict): + return False, "" + if not data.get("updated_content"): + return False, "" + need_update = data.get("need_update", False) + updated_content = data["updated_content"].strip() + if updated_content == "empty": + updated_content = "" + return need_update, updated_content diff --git a/src/memu/app/memorize.py b/src/memu/app/memorize.py new file mode 100644 index 0000000..2f6331f --- /dev/null +++ b/src/memu/app/memorize.py @@ -0,0 +1,1210 @@ +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import pathlib +import re +from collections.abc import Awaitable, Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, cast +from xml.etree.ElementTree import Element + +import defusedxml.ElementTree as ET +from pydantic import BaseModel + +from memu.app.settings import CategoryConfig, CustomPrompt +from memu.database.models import EntryType, RecallEntry, RecallFile, RecallFileEntry, Resource +from memu.preprocess import PreprocessContext, preprocess_resource +from memu.prompts.category_summary import ( + CUSTOM_PROMPT as CATEGORY_SUMMARY_CUSTOM_PROMPT, +) +from memu.prompts.category_summary import ( + PROMPT as CATEGORY_SUMMARY_PROMPT, +) +from memu.prompts.memory_type import ( + CUSTOM_PROMPTS as MEMORY_TYPE_CUSTOM_PROMPTS, +) +from memu.prompts.memory_type import ( + CUSTOM_TYPE_CUSTOM_PROMPTS, + DEFAULT_MEMORY_TYPES, +) +from memu.prompts.memory_type import ( + PROMPTS as MEMORY_TYPE_PROMPTS, +) +from memu.workflow.step import WorkflowState, WorkflowStep + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from memu.app.service import Context + from memu.app.settings import MemorizeConfig, MemoryFilesConfig + from memu.blob.local_fs import LocalFS + from memu.database.interfaces import Database + + +class MemorizeMixin: + if TYPE_CHECKING: + memorize_config: MemorizeConfig + category_configs: list[CategoryConfig] + category_config_map: dict[str, CategoryConfig] + _category_prompt_str: str + fs: LocalFS + _run_workflow: Callable[..., Awaitable[WorkflowState]] + _get_context: Callable[[], Context] + _get_database: Callable[[], Database] + _get_step_llm_client: Callable[[Mapping[str, Any] | None], Any] + _get_step_embedding_client: Callable[[Mapping[str, Any] | None], Any] + _get_embedding_client: Callable[..., Any] + _get_llm_client: Callable[..., Any] + _get_vlm_client: Callable[..., Any] + _model_dump_without_embeddings: Callable[[BaseModel], dict[str, Any]] + _extract_json_blob: Callable[[str], str] + _escape_prompt_value: Callable[[str], str] + user_model: type[BaseModel] + + # Memory file system export (provided by MemoryService). + memory_files_config: MemoryFilesConfig + _build_memory_files: Callable[..., Awaitable[dict[str, Any]]] + + # Provided by CRUDMixin (composed onto MemoryService). + async def _patch_category_summaries( + self, + updates: dict[str, tuple[str | None, str | None]], + ctx: Context, + store: Database, + llm_client: Any | None = None, + ) -> None: ... + + async def memorize( + self, + *, + resource_url: str, + modality: str, + user: dict[str, Any] | None = None, + ) -> dict[str, Any]: + ctx = self._get_context() + store = self._get_database() + user_scope = self.user_model(**user).model_dump() if user is not None else None + await self._ensure_categories_ready(ctx, store, user_scope) + + memory_types = self._resolve_memory_types() + + state: WorkflowState = { + "resource_url": resource_url, + "modality": modality, + "memory_types": memory_types, + "categories_prompt_str": self._category_prompt_str, + "ctx": ctx, + "store": store, + "category_ids": list(ctx.category_ids), + "user": user_scope, + # Legacy single-resource path: force only the provided categories. + "allow_new_categories": False, + # Legacy path does not classify by workspace track. + "resource_track": None, + } + + result = await self._run_workflow("memorize", state) + response = cast(dict[str, Any] | None, result.get("response")) + if response is None: + msg = "Memorize workflow failed to produce a response" + raise RuntimeError(msg) + return response + + def _build_memorize_workflow(self) -> list[WorkflowStep]: + steps = [ + WorkflowStep( + step_id="ingest_resource", + role="ingest", + handler=self._memorize_ingest_resource, + requires={"resource_url", "modality"}, + produces={"local_path", "raw_text"}, + capabilities={"io"}, + ), + WorkflowStep( + step_id="preprocess_multimodal", + role="preprocess", + handler=self._memorize_preprocess_multimodal, + requires={"local_path", "modality", "raw_text"}, + produces={"preprocessed_resources"}, + capabilities={"llm"}, + config={"chat_llm_profile": self.memorize_config.preprocess_llm_profile}, + ), + WorkflowStep( + step_id="extract_entries", + role="extract", + handler=self._memorize_extract_entries, + requires={ + "preprocessed_resources", + "memory_types", + "categories_prompt_str", + "modality", + "resource_url", + }, + produces={"resource_plans"}, + capabilities={"llm"}, + config={"chat_llm_profile": self.memorize_config.memory_extract_llm_profile}, + ), + WorkflowStep( + step_id="dedupe_merge", + role="dedupe_merge", + handler=self._memorize_dedupe_merge, + requires={"resource_plans"}, + produces={"resource_plans"}, + capabilities=set(), + ), + WorkflowStep( + step_id="categorize_entries", + role="categorize", + handler=self._memorize_categorize_entries, + requires={ + "resource_plans", + "ctx", + "store", + "local_path", + "modality", + "user", + "allow_new_categories", + "resource_track", + }, + produces={"resources", "entries", "relations", "file_updates"}, + capabilities={"db", "vector"}, + config={"embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="persist_index", + role="persist", + handler=self._memorize_persist_and_index, + requires={"file_updates", "ctx", "store"}, + produces={"files"}, + capabilities={"db", "llm"}, + config={"chat_llm_profile": self.memorize_config.category_update_llm_profile}, + ), + WorkflowStep( + step_id="build_response", + role="emit", + handler=self._memorize_build_response, + requires={"resources", "entries", "relations", "ctx", "store", "category_ids"}, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + @staticmethod + def _list_memorize_initial_keys() -> set[str]: + return { + "resource_url", + "modality", + "memory_types", + "categories_prompt_str", + "ctx", + "store", + "category_ids", + "user", + "allow_new_categories", + "resource_track", + } + + async def _memorize_ingest_resource(self, state: WorkflowState, step_context: Any) -> WorkflowState: + local_path, raw_text = await self.fs.fetch(state["resource_url"], state["modality"]) + state.update({"local_path": local_path, "raw_text": raw_text}) + return state + + # Modalities whose preprocessing analyzes media via the VLM (vision) client. + _VISION_MODALITIES = frozenset({"image", "video"}) + + async def _memorize_preprocess_multimodal(self, state: WorkflowState, step_context: Any) -> WorkflowState: + modality = state["modality"] + client = self._get_step_llm_client(step_context) + if modality in self._VISION_MODALITIES: + with contextlib.suppress(KeyError): + client = self._get_vlm_client(self.memorize_config.vlm_profile, step_context=step_context) + preprocessed = await self._preprocess_resource_url( + local_path=state["local_path"], + text=state.get("raw_text"), + modality=modality, + llm_client=client, + ) + if not preprocessed: + preprocessed = [{"text": state.get("raw_text"), "caption": None}] + state["preprocessed_resources"] = preprocessed + return state + + async def _memorize_extract_entries(self, state: WorkflowState, step_context: Any) -> WorkflowState: + llm_client = self._get_step_llm_client(step_context) + preprocessed_resources = state.get("preprocessed_resources", []) + resource_plans: list[dict[str, Any]] = [] + total_segments = len(preprocessed_resources) or 1 + + for idx, prep in enumerate(preprocessed_resources): + res_url = self._segment_resource_url(state["resource_url"], idx, total_segments) + text = prep.get("text") + caption = prep.get("caption") + + structured_entries = await self._generate_structured_entries( + resource_url=res_url, + modality=state["modality"], + memory_types=state["memory_types"], + text=text, + categories_prompt_str=state["categories_prompt_str"], + llm_client=llm_client, + ) + + resource_plans.append({ + "resource_url": res_url, + "text": text, + "caption": caption, + "entries": structured_entries, + }) + + state["resource_plans"] = resource_plans + return state + + def _memorize_dedupe_merge(self, state: WorkflowState, step_context: Any) -> WorkflowState: + # Placeholder for future dedup/merge logic + state["resource_plans"] = state.get("resource_plans", []) + return state + + async def _memorize_categorize_entries(self, state: WorkflowState, step_context: Any) -> WorkflowState: + embed_client = self._get_step_embedding_client(step_context) + ctx = state["ctx"] + store = state["store"] + modality = state["modality"] + local_path = state["local_path"] + resources: list[Resource] = [] + created_entries: list[RecallEntry] = [] + relations: list[RecallFileEntry] = [] + file_updates: dict[str, list[tuple[str, str]]] = {} + user_scope = state.get("user", {}) + allow_new_categories = state.get("allow_new_categories", False) + track = state.get("resource_track") + + for plan in state.get("resource_plans", []): + res = await self._create_resource_with_caption( + resource_url=plan["resource_url"], + modality=modality, + local_path=local_path, + caption=plan.get("caption"), + store=store, + embed_client=embed_client, + user=user_scope, + track=track, + ) + resources.append(res) + + plan_entries = plan.get("entries") or [] + if not plan_entries: + continue + + mem_entries, rels, new_file_updates = await self._persist_recall_entries( + resource_id=res.id, + structured_entries=plan_entries, + ctx=ctx, + store=store, + embed_client=embed_client, + user=user_scope, + allow_new_categories=allow_new_categories, + ) + created_entries.extend(mem_entries) + relations.extend(rels) + for file_id, entry_tuples in new_file_updates.items(): + file_updates.setdefault(file_id, []).extend(entry_tuples) + + state.update({ + "resources": resources, + "entries": created_entries, + "relations": relations, + "file_updates": file_updates, + }) + return state + + async def _memorize_persist_and_index(self, state: WorkflowState, step_context: Any) -> WorkflowState: + llm_client = self._get_step_llm_client(step_context) + updated_summaries = await self._update_file_summaries( + state.get("file_updates", {}), + ctx=state["ctx"], + store=state["store"], + llm_client=llm_client, + ) + if self.memorize_config.enable_item_references: + await self._persist_entry_references( + updated_summaries=updated_summaries, + file_updates=state.get("file_updates", {}), + store=state["store"], + ) + return state + + def _memorize_build_response(self, state: WorkflowState, step_context: Any) -> WorkflowState: + ctx = state["ctx"] + store = state["store"] + resources = [self._model_dump_without_embeddings(r) for r in state.get("resources", [])] + entries = [self._model_dump_without_embeddings(entry) for entry in state.get("entries", [])] + relations = [rel.model_dump() for rel in state.get("relations", [])] + category_ids = state.get("category_ids") or list(ctx.category_ids) + files = [self._model_dump_without_embeddings(store.recall_file_repo.categories[c]) for c in category_ids] + + # Legacy response contract: ``memorize`` keeps emitting ``items``/``categories`` + # keys for backward compatibility; the new vocabulary lives in the internal vars. + if len(resources) == 1: + response = { + "resource": resources[0], + "items": entries, + "categories": files, + "relations": relations, + } + else: + response = { + "resources": resources, + "items": entries, + "categories": files, + "relations": relations, + } + state["response"] = response + return state + + def _segment_resource_url(self, base_url: str, idx: int, total_segments: int) -> str: + if total_segments <= 1: + return base_url + path = pathlib.Path(base_url) + return f"{path.stem}_#segment_{idx}{path.suffix}" + + async def _fetch_and_preprocess_resource( + self, resource_url: str, modality: str, llm_client: Any | None = None + ) -> tuple[str, list[dict[str, str | None]]]: + """ + Fetch and preprocess a resource. + + Returns: + Tuple of (local_path, preprocessed_resources) + where preprocessed_resources is a list of dicts with 'text' and 'caption' + """ + local_path, text = await self.fs.fetch(resource_url, modality) + preprocessed_resources = await self._preprocess_resource_url( + local_path=local_path, + text=text, + modality=modality, + llm_client=llm_client, + ) + return local_path, preprocessed_resources + + async def _create_resource_with_caption( + self, + *, + resource_url: str, + modality: str, + local_path: str, + caption: str | None, + store: Database, + embed_client: Any | None = None, + user: Mapping[str, Any] | None = None, + track: str | None = None, + ) -> Resource: + caption_text = caption.strip() if caption else None + if caption_text: + client = embed_client or self._get_embedding_client() + caption_embedding = (await client.embed([caption_text]))[0] + else: + caption_embedding = None + + res = store.resource_repo.create_resource( + url=resource_url, + modality=modality, + local_path=local_path, + caption=caption_text, + embedding=caption_embedding, + user_data=dict(user or {}), + track=track, + ) + # if caption: + # caption_text = caption.strip() + # if caption_text: + # res.caption = caption_text + # client = embed_client or self._get_llm_client() + # res.embedding = (await client.embed([caption_text]))[0] + # res.updated_at = pendulum.now() + return res + + def _resolve_memory_types(self) -> list[EntryType]: + configured_types = self.memorize_config.memory_types or DEFAULT_MEMORY_TYPES + return [cast(EntryType, mtype) for mtype in configured_types] + + def _resolve_summary_prompt(self, modality: str, override: str | None) -> str | None: + memo_settings = self.memorize_config + result = memo_settings.multimodal_preprocess_prompts.get(modality) + if override: + return override + if result is None: + return ( + memo_settings.default_category_summary_prompt + if isinstance(memo_settings.default_category_summary_prompt, str) + else None + ) + return result if isinstance(result, str) else None + + def _resolve_multimodal_preprocess_prompt(self, modality: str) -> str | None: + memo_settings = self.memorize_config + result = memo_settings.multimodal_preprocess_prompts.get(modality) + return result if isinstance(result, str) else None + + @staticmethod + def _resolve_custom_prompt(prompt: str | CustomPrompt, templates: Mapping[str, str]) -> str: + if isinstance(prompt, str): + return prompt + valid_blocks = [ + (block.ordinal, name, block.prompt or templates.get(name)) + for name, block in prompt.items() + if (block.ordinal >= 0 and (block.prompt or templates.get(name))) + ] + if not valid_blocks: + # raise ValueError(f"No valid blocks contained in custom prompt: {prompt}") + return "" + sorted_blocks = sorted(valid_blocks) + return "\n\n".join(block for (_, _, block) in sorted_blocks if block is not None) + + async def _generate_structured_entries( + self, + *, + resource_url: str, + modality: str, + memory_types: list[EntryType], + text: str | None, + categories_prompt_str: str, + segments: list[dict[str, int | str]] | None = None, + llm_client: Any | None = None, + ) -> list[tuple[EntryType, str, list[str]]]: + if not memory_types: + return [] + + client = llm_client or self._get_llm_client() + if text: + entries = await self._generate_text_entries( + resource_text=text, + modality=modality, + memory_types=memory_types, + categories_prompt_str=categories_prompt_str, + segments=segments, + llm_client=client, + ) + return entries + # if entries: + # return entries + # no_result_entry = self._build_no_result_fallback(memory_types[0], resource_url, modality) + # return [no_result_entry] + + return [] + # return self._build_no_text_fallback(memory_types, resource_url, modality) + + async def _generate_text_entries( + self, + *, + resource_text: str, + modality: str, + memory_types: list[EntryType], + categories_prompt_str: str, + segments: list[dict[str, int | str]] | None, + llm_client: Any | None = None, + ) -> list[tuple[EntryType, str, list[str]]]: + if modality == "conversation" and segments: + segment_entries = await self._generate_entries_for_segments( + resource_text=resource_text, + segments=segments, + memory_types=memory_types, + categories_prompt_str=categories_prompt_str, + llm_client=llm_client, + ) + if segment_entries: + return segment_entries + return await self._generate_entries_from_text( + resource_text=resource_text, + memory_types=memory_types, + categories_prompt_str=categories_prompt_str, + llm_client=llm_client, + ) + + async def _generate_entries_for_segments( + self, + *, + resource_text: str, + segments: list[dict[str, int | str]], + memory_types: list[EntryType], + categories_prompt_str: str, + llm_client: Any | None = None, + ) -> list[tuple[EntryType, str, list[str]]]: + entries: list[tuple[EntryType, str, list[str]]] = [] + lines = resource_text.split("\n") + max_idx = len(lines) - 1 + for segment in segments: + start_idx = int(segment.get("start", 0)) + end_idx = int(segment.get("end", max_idx)) + segment_text = self._extract_segment_text(lines, start_idx, end_idx) + if not segment_text: + continue + segment_entries = await self._generate_entries_from_text( + resource_text=segment_text, + memory_types=memory_types, + categories_prompt_str=categories_prompt_str, + llm_client=llm_client, + ) + entries.extend(segment_entries) + return entries + + async def _generate_entries_from_text( + self, + *, + resource_text: str, + memory_types: list[EntryType], + categories_prompt_str: str, + llm_client: Any | None = None, + ) -> list[tuple[EntryType, str, list[str]]]: + if not memory_types: + return [] + client = llm_client or self._get_llm_client() + prompts = [ + self._build_memory_type_prompt( + memory_type=mtype, + resource_text=resource_text, + categories_str=categories_prompt_str, + ) + for mtype in memory_types + ] + valid_prompts = [prompt for prompt in prompts if prompt.strip()] + # These prompts are instructions that request structured output, not text summaries. + tasks = [client.chat(prompt_text) for prompt_text in valid_prompts] + responses = await asyncio.gather(*tasks) + return self._parse_structured_entries(memory_types, responses) + + def _parse_structured_entries( + self, memory_types: list[EntryType], responses: Sequence[str] + ) -> list[tuple[EntryType, str, list[str]]]: + entries: list[tuple[EntryType, str, list[str]]] = [] + for mtype, response in zip(memory_types, responses, strict=True): + parsed = self._parse_memory_type_response_xml(response) + # if not parsed: + # fallback_entry = response.strip() + # if fallback_entry: + # entries.append((mtype, fallback_entry, [])) + # continue + for entry in parsed: + content = (entry.get("content") or "").strip() + if not content: + continue + cat_names = [c.strip() for c in entry.get("categories", []) if isinstance(c, str) and c.strip()] + entries.append((mtype, content, cat_names)) + return entries + + def _extract_segment_text(self, lines: list[str], start_idx: int, end_idx: int) -> str | None: + segment_lines = [] + for line in lines: + match = re.match(r"\[(\d+)\]", line) + if not match: + continue + idx = int(match.group(1)) + if start_idx <= idx <= end_idx: + segment_lines.append(line) + return "\n".join(segment_lines) if segment_lines else None + + def _build_no_text_fallback( + self, memory_types: list[EntryType], resource_url: str, modality: str + ) -> list[tuple[EntryType, str, list[str]]]: + fallback = f"Resource {resource_url} ({modality}) stored. No text summary in v0." + return [(mtype, f"{fallback} (memory type: {mtype}).", []) for mtype in memory_types] + + def _build_no_result_fallback( + self, memory_type: EntryType, resource_url: str, modality: str + ) -> tuple[EntryType, str, list[str]]: + fallback = f"Resource {resource_url} ({modality}) stored. No structured memories generated." + return memory_type, fallback, [] + + async def _persist_recall_entries( + self, + *, + resource_id: str, + structured_entries: list[tuple[EntryType, str, list[str]]], + ctx: Context, + store: Database, + embed_client: Any | None = None, + user: Mapping[str, Any] | None = None, + allow_new_categories: bool = False, + ) -> tuple[list[RecallEntry], list[RecallFileEntry], dict[str, list[tuple[str, str]]]]: + """ + Persist recall entries and track per-file updates. + + Returns: + Tuple of (entries, relations, file_updates) + where file_updates maps file_id -> list of (entry_id, summary) tuples + """ + summary_payloads = [content for _, content, _ in structured_entries] + client = embed_client or self._get_embedding_client() + entry_embeddings = await client.embed(summary_payloads) if summary_payloads else [] + entries: list[RecallEntry] = [] + rels: list[RecallFileEntry] = [] + # Stores (entry_id, summary) tuples for reference support + file_entry_updates: dict[str, list[tuple[str, str]]] = {} + + reinforce = self.memorize_config.enable_item_reinforcement + for (memory_type, summary_text, cat_names), emb in zip(structured_entries, entry_embeddings, strict=True): + entry = store.recall_entry_repo.create_item( + resource_id=resource_id, + memory_type=memory_type, + summary=summary_text, + embedding=emb, + user_data=dict(user or {}), + reinforce=reinforce, + ) + entries.append(entry) + if reinforce and entry.extra.get("reinforcement_count", 1) > 1: + # existing entry + continue + mapped_file_ids = await self._resolve_category_ids( + cat_names, ctx, store, user=user, allow_new=allow_new_categories + ) + for file_id in mapped_file_ids: + rels.append( + store.recall_file_entry_repo.link_item_category(entry.id, file_id, user_data=dict(user or {})) + ) + # Store (entry_id, summary) tuple for reference support + file_entry_updates.setdefault(file_id, []).append((entry.id, summary_text)) + + return entries, rels, file_entry_updates + + def _start_category_initialization(self, ctx: Context, store: Database) -> None: + if ctx.categories_ready: + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + if loop: + ctx.category_init_task = loop.create_task(self._initialize_categories(ctx, store)) + else: + asyncio.run(self._initialize_categories(ctx, store)) + + async def _ensure_categories_ready( + self, ctx: Context, store: Database, user_scope: Mapping[str, Any] | None = None + ) -> None: + if ctx.categories_ready: + return + if ctx.category_init_task: + await ctx.category_init_task + ctx.category_init_task = None + return + await self._initialize_categories(ctx, store, user_scope) + + @staticmethod + def _classify_categories( + configs: list[CategoryConfig], + existing_by_name: dict[str, RecallFile], + ) -> tuple[ + list[tuple[int, CategoryConfig]], + list[tuple[int, CategoryConfig, RecallFile]], + dict[int, RecallFile], + ]: + to_create: list[tuple[int, CategoryConfig]] = [] + to_update: list[tuple[int, CategoryConfig, RecallFile]] = [] + ready: dict[int, RecallFile] = {} + for i, cfg in enumerate(configs): + name = cfg.name.strip() or "Untitled" + description = cfg.description.strip() + ex = existing_by_name.get(name) + if ex is None: + to_create.append((i, cfg)) + elif ex.embedding is None or (ex.description or "") != description: + to_update.append((i, cfg, ex)) + else: + ready[i] = ex + return to_create, to_update, ready + + async def _initialize_categories( + self, ctx: Context, store: Database, user: Mapping[str, Any] | None = None + ) -> None: + if ctx.categories_ready: + return + if not self.category_configs: + ctx.categories_ready = True + return + + user_data = dict(user or {}) + existing = store.recall_file_repo.list_categories(where={**user_data, "track": "memory"}) + existing_by_name: dict[str, RecallFile] = {c.name: c for c in existing.values()} + + to_create, to_update, ready = self._classify_categories(self.category_configs, existing_by_name) + + needs_embed: list[tuple[int, CategoryConfig]] = [] + needs_embed.extend(to_create) + needs_embed.extend((i, cfg) for i, cfg, _ in to_update) + + embed_map: dict[int, list[float]] = {} + if needs_embed: + texts = [self._category_embedding_text(cfg) for _, cfg in needs_embed] + vecs = await self._get_embedding_client("embedding").embed(texts) + for (i, _), vec in zip(needs_embed, vecs, strict=True): + embed_map[i] = vec + + cats: dict[int, RecallFile] = dict(ready) + + for i, cfg in to_create: + name = cfg.name.strip() or "Untitled" + description = cfg.description.strip() + cat = store.recall_file_repo.get_or_create_category( + name=name, description=description, embedding=embed_map[i], user_data=user_data + ) + cats[i] = cat + + for i, cfg, ex in to_update: + description = cfg.description.strip() + cat = store.recall_file_repo.update_category( + category_id=ex.id, description=description, embedding=embed_map[i] + ) + cats[i] = cat + + ctx.category_ids = [] + ctx.category_name_to_id = {} + for i in range(len(self.category_configs)): + cat = cats[i] + ctx.category_ids.append(cat.id) + name = self.category_configs[i].name.strip() or "Untitled" + ctx.category_name_to_id[name.lower()] = cat.id + ctx.categories_ready = True + + @staticmethod + def _category_embedding_text(cat: CategoryConfig) -> str: + name = cat.name.strip() or "Untitled" + desc = cat.description.strip() + return f"{name}: {desc}" if desc else name + + def _map_category_names_to_ids(self, names: list[str], ctx: Context) -> list[str]: + if not names: + return [] + mapped: list[str] = [] + seen: set[str] = set() + for name in names: + key = name.strip().lower() + cid = ctx.category_name_to_id.get(key) + if cid and cid not in seen: + mapped.append(cid) + seen.add(cid) + return mapped + + @staticmethod + def _partition_category_names(names: list[str], ctx: Context) -> tuple[list[str], list[str]]: + """Split proposed names into (known category ids, unknown names) with dedup.""" + known_ids: list[str] = [] + known_seen: set[str] = set() + unknown: list[str] = [] + unknown_seen: set[str] = set() + for name in names: + key = name.strip().lower() + if not key: + continue + cid = ctx.category_name_to_id.get(key) + if cid is not None: + if cid not in known_seen: + known_ids.append(cid) + known_seen.add(cid) + elif key not in unknown_seen: + unknown.append(name.strip()) + unknown_seen.add(key) + return known_ids, unknown + + async def _resolve_category_ids( + self, + names: list[str], + ctx: Context, + store: Database, + *, + user: Mapping[str, Any] | None = None, + allow_new: bool = False, + ) -> list[str]: + """Resolve extractor-proposed category names to ids, optionally creating unknown ones. + + When ``allow_new`` is true this implements the open/adaptive taxonomy: any category + name the extractor proposes is created on first sight and cached in the context. + (Embedding-similarity merging of near-duplicate categories is handled later by + consolidation; here we only do exact-name dedup.) When false, only names matching an + existing category resolve and proposed-but-unknown names are dropped, so the caller is + confined to the provided taxonomy. + """ + if not names: + return [] + user_data = dict(user or {}) + resolved, unknown = self._partition_category_names(names, ctx) + seen: set[str] = set(resolved) + + if unknown and allow_new: + vecs = await self._get_embedding_client("embedding").embed(unknown) + for name, vec in zip(unknown, vecs, strict=True): + cat = store.recall_file_repo.get_or_create_category( + name=name, description="", embedding=vec, user_data=user_data + ) + ctx.category_name_to_id[name.lower()] = cat.id + if cat.id not in ctx.category_ids: + ctx.category_ids.append(cat.id) + if cat.id not in seen: + resolved.append(cat.id) + seen.add(cat.id) + return resolved + + async def _preprocess_resource_url( + self, *, local_path: str, text: str | None, modality: str, llm_client: Any | None = None + ) -> list[dict[str, str | None]]: + """Preprocess a resource by delegating to the per-format ``preprocess`` package. + + Returns a list of preprocessed resources, each with 'text' and 'caption'. + """ + return await preprocess_resource( + modality=modality, + local_path=local_path, + text=text, + ctx=self._build_preprocess_context(), + llm_client=llm_client, + ) + + def _build_preprocess_context(self) -> PreprocessContext: + """Bundle the service dependencies the preprocessors need.""" + return PreprocessContext( + get_llm_client=self._get_llm_client, + get_vlm_client=lambda: self._get_vlm_client(self.memorize_config.vlm_profile), + escape_prompt_value=self._escape_prompt_value, + extract_json_blob=self._extract_json_blob, + resolve_custom_prompt=self._resolve_custom_prompt, + multimodal_preprocess_prompts=self.memorize_config.multimodal_preprocess_prompts, + ) + + def _format_categories_for_prompt(self, categories: list[CategoryConfig]) -> str: + adaptive_hint = ( + "Assign each memory item 1-3 concise, reusable category names (short noun " + "phrases). Reuse a listed category when one fits; otherwise propose a new " + "concise category name. Categories are created automatically." + ) + if not categories: + return "No predefined categories yet.\n" + adaptive_hint + lines = [] + for cat in categories: + name = cat.name.strip() or "Untitled" + desc = cat.description.strip() + lines.append(f"- {name}: {desc}" if desc else f"- {name}") + return "Existing categories (reuse when appropriate):\n" + "\n".join(lines) + "\n\n" + adaptive_hint + + def _add_conversation_indices(self, conversation: str) -> str: + """ + Add [INDEX] markers to each line of the conversation. + + Args: + conversation: Raw conversation text with lines + + Returns: + Conversation with [INDEX] markers prepended to each non-empty line + """ + lines = conversation.split("\n") + indexed_lines = [] + index = 0 + + for line in lines: + stripped = line.strip() + if stripped: # Only index non-empty lines + indexed_lines.append(f"[{index}] {line}") + index += 1 + else: + # Preserve empty lines without indexing + indexed_lines.append(line) + + return "\n".join(indexed_lines) + + def _build_memory_type_prompt(self, *, memory_type: EntryType, resource_text: str, categories_str: str) -> str: + configured_prompt = self.memorize_config.memory_type_prompts.get(memory_type) + if configured_prompt is None: + template = MEMORY_TYPE_PROMPTS.get(memory_type) + elif isinstance(configured_prompt, str): + template = configured_prompt + else: + template = self._resolve_custom_prompt( + configured_prompt, MEMORY_TYPE_CUSTOM_PROMPTS.get(memory_type, CUSTOM_TYPE_CUSTOM_PROMPTS) + ) + if not template: + return resource_text + safe_resource = self._escape_prompt_value(resource_text) + safe_categories = self._escape_prompt_value(categories_str) + return template.format(resource=safe_resource, categories_str=safe_categories) + + def _build_entry_ref_id(self, entry_id: str) -> str: + return entry_id.replace("-", "")[:6] + + def _extract_refs_from_summaries(self, summaries: dict[str, str]) -> set[str]: + """ + Extract all [ref:xxx] references from summary texts. + + Args: + summaries: dict mapping file_id -> summary text + + Returns: + Set of all referenced short IDs (the xxx part from [ref:xxx]) + """ + from memu.utils.references import extract_references + + refs: set[str] = set() + for summary in summaries.values(): + refs.update(extract_references(summary)) + return refs + + async def _persist_entry_references( + self, + *, + updated_summaries: dict[str, str], + file_updates: dict[str, list[tuple[str, str]]], + store: Database, + ) -> None: + """ + Persist ref_id to entries that are referenced in file summaries. + + This function: + 1. Extracts all [ref:xxx] patterns from updated summaries + 2. Builds a mapping of short_id -> full entry_id for all entries in file_updates + 3. For entries whose short_id appears in the references, updates their extra column + with {"ref_id": short_id} + """ + # Extract all referenced short IDs from summaries + referenced_short_ids = self._extract_refs_from_summaries(updated_summaries) + if not referenced_short_ids: + return + + # Build mapping of short_id -> full entry_id for all entries in file_updates + short_id_to_entry_id: dict[str, str] = {} + for entry_tuples in file_updates.values(): + for entry_id, _ in entry_tuples: + short_id = self._build_entry_ref_id(entry_id) + short_id_to_entry_id[short_id] = entry_id + + # Update extra column for referenced entries + for short_id in referenced_short_ids: + matched_entry_id = short_id_to_entry_id.get(short_id) + if matched_entry_id: + store.recall_entry_repo.update_item( + item_id=matched_entry_id, + extra={"ref_id": short_id}, + ) + + def _build_file_summary_prompt( + self, + *, + category: RecallFile, + new_memories: list[str] | list[tuple[str, str]], + ) -> str: + """ + Build the prompt for updating a file summary. + + Args: + category: The file (RecallFile) to update + new_memories: Either list of summary strings (legacy) or list of (entry_id, summary) tuples (with refs) + """ + # Check if references are enabled and we have (id, summary) tuples + enable_refs = getattr(self.memorize_config, "enable_item_references", False) + + if enable_refs: + from memu.prompts.category_summary import ( + CUSTOM_PROMPT_WITH_REFS as category_summary_custom_prompt, + ) + from memu.prompts.category_summary import ( + PROMPT_WITH_REFS as category_summary_prompt, + ) + + tuple_memories = cast(list[tuple[str, str]], new_memories) + new_entries_text = "\n".join( + f"- [{self._build_entry_ref_id(entry_id)}] {summary}" + for entry_id, summary in tuple_memories + if summary.strip() + ) + else: + category_summary_prompt = CATEGORY_SUMMARY_PROMPT + category_summary_custom_prompt = CATEGORY_SUMMARY_CUSTOM_PROMPT + + if new_memories and isinstance(new_memories[0], tuple): + tuple_memories = cast(list[tuple[str, str]], new_memories) + new_entries_text = "\n".join(f"- {summary}" for _entry_id, summary in tuple_memories if summary.strip()) + else: + str_memories = cast(list[str], new_memories) + new_entries_text = "\n".join(f"- {m}" for m in str_memories if m.strip()) + + original = category.content or "" + category_config = self.category_config_map.get(category.name) + configured_prompt = ( + category_config and category_config.summary_prompt + ) or self.memorize_config.default_category_summary_prompt + if configured_prompt is None: + prompt = category_summary_prompt + elif isinstance(configured_prompt, str): + prompt = configured_prompt + else: + prompt = self._resolve_custom_prompt(configured_prompt, category_summary_custom_prompt) + target_length = ( + category_config and category_config.target_length + ) or self.memorize_config.default_category_summary_target_length + return prompt.format( + category=self._escape_prompt_value(category.name), + original_content=self._escape_prompt_value(original or ""), + new_recall_entries_text=self._escape_prompt_value(new_entries_text or "No new entries."), + target_length=target_length, + ) + + async def _update_file_summaries( + self, + updates: dict[str, list[tuple[str, str]]] | dict[str, list[str]], + ctx: Context, + store: Database, + llm_client: Any | None = None, + ) -> dict[str, str]: + """ + Update file summaries based on new entries. + + Returns: + dict mapping file_id -> updated summary text + """ + updated_summaries: dict[str, str] = {} + if not updates: + return updated_summaries + tasks = [] + target_ids: list[str] = [] + client = llm_client or self._get_llm_client() + for cid, memories in updates.items(): + cat = store.recall_file_repo.categories.get(cid) + if not cat or not memories: + continue + prompt = self._build_file_summary_prompt(category=cat, new_memories=memories) + tasks.append(client.chat(prompt)) + target_ids.append(cid) + if not tasks: + return updated_summaries + summaries = await asyncio.gather(*tasks) + for cid, summary in zip(target_ids, summaries, strict=True): + cat = store.recall_file_repo.categories.get(cid) + if not cat: + continue + cleaned_summary = summary.replace("```markdown", "").replace("```", "").strip() + store.recall_file_repo.update_category( + category_id=cid, + content=cleaned_summary, + ) + updated_summaries[cid] = cleaned_summary + return updated_summaries + + def _parse_conversation_preprocess(self, raw: str) -> tuple[str | None, str | None]: + conversation = self._extract_tag_content(raw, "conversation") + summary = self._extract_tag_content(raw, "summary") + return conversation, summary + + @staticmethod + def _extract_tag_content(raw: str, tag: str) -> str | None: + pattern = re.compile(rf"<{tag}>(.*?)", re.IGNORECASE | re.DOTALL) + match = pattern.search(raw) + if not match: + return None + content = match.group(1).strip() + return content or None + + def _parse_memory_type_response(self, raw: str) -> list[dict[str, Any]]: + if not raw: + return [] + raw = raw.strip() + if not raw: + return [] + payload = None + try: + payload = json.loads(raw) + except json.JSONDecodeError: + try: + blob = self._extract_json_blob(raw) + payload = json.loads(blob) + except Exception: + return [] + if not isinstance(payload, dict): + return [] + items = payload.get("memories_items") + if not isinstance(items, list): + return [] + normalized: list[dict[str, Any]] = [] + for entry in items: + if not isinstance(entry, dict): + continue + normalized.append(entry) + return normalized + + def _find_xml_boundaries(self, raw: str) -> tuple[int, int, str] | None: + """Find the start index, end index, and closing tag for XML root element.""" + root_tags = ["item", "profile", "behaviors", "events", "knowledge", "skills"] + for tag in root_tags: + opening = f"<{tag}>" + closing = f"" + start_idx = raw.find(opening) + if start_idx != -1: + end_idx = raw.rfind(closing) + if end_idx != -1: + return (start_idx, end_idx, closing) + return None + + def _parse_memory_element(self, memory_elem: Element) -> dict[str, Any] | None: + """Parse a single memory XML element into a dict.""" + memory_dict: dict[str, Any] = {} + + content_elem = memory_elem.find("content") + if content_elem is not None and content_elem.text: + memory_dict["content"] = content_elem.text.strip() + + categories_elem = memory_elem.find("categories") + if categories_elem is not None: + categories = [cat_elem.text.strip() for cat_elem in categories_elem.findall("category") if cat_elem.text] + memory_dict["categories"] = categories + + if memory_dict.get("content") and memory_dict.get("categories"): + return memory_dict + return None + + def _parse_memory_type_response_xml(self, raw: str) -> list[dict[str, Any]]: + """ + Parse XML memory extraction output into a list of memory items. + + Expected XML format (root tag varies by memory type): + + + ... + + ... + + + + """ + if not raw or not raw.strip(): + return [] + raw = raw.strip() + + try: + boundaries = self._find_xml_boundaries(raw) + if boundaries is None: + logger.warning("Could not find valid root tag in XML response") + return [] + + start_idx, end_idx, end_tag = boundaries + xml_content = raw[start_idx : end_idx + len(end_tag)] + xml_content = xml_content.replace("&", "&") + + try: + root = ET.fromstring(xml_content) + except ET.ParseError: + # Some LLMs emit one per memory rather than a single root + # element wrapping all memories, resulting in "junk after document + # element" when the slice contains multiple top-level tags. Wrap + # the content in a synthetic root element and retry. + root = ET.fromstring(f"<_root_>{xml_content}") + + result: list[dict[str, Any]] = [] + + for memory_elem in root.iter("memory"): + parsed = self._parse_memory_element(memory_elem) + if parsed: + result.append(parsed) + + except ET.ParseError: + logger.exception("Failed to parse XML") + return [] + else: + return result diff --git a/src/memu/app/memorize_workspace.py b/src/memu/app/memorize_workspace.py new file mode 100644 index 0000000..2df24c0 --- /dev/null +++ b/src/memu/app/memorize_workspace.py @@ -0,0 +1,834 @@ +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import pathlib +from collections.abc import Awaitable, Callable, Mapping +from typing import TYPE_CHECKING, Any, ClassVar, cast + +from pydantic import BaseModel + +from memu.app.settings import CategoryConfig, CustomPrompt +from memu.blob.folder import diff_folder, load_manifest, manifest_from_scan, save_manifest, scan_folder +from memu.database.models import EntryType, RecallFile, Resource +from memu.preprocess import PreprocessContext, preprocess_resource +from memu.prompts.memory_fs import ( + CONTENT_PLACEHOLDER, + DESCRIPTION_PLACEHOLDER, + EXISTING_PLACEHOLDER, + NAME_PLACEHOLDER, + ROUTE_PROMPTS, + SYNTHESIS_PROMPTS, +) +from memu.prompts.memory_type import DEFAULT_MEMORY_TYPES +from memu.workflow.step import WorkflowState, WorkflowStep + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from memu.app.service import Context + from memu.app.settings import MemorizeConfig, MemoryFilesConfig + from memu.blob.local_fs import LocalFS + from memu.database.interfaces import Database + + +class MemorizeWorkspaceMixin: + if TYPE_CHECKING: + memorize_config: MemorizeConfig + category_configs: list[CategoryConfig] + _category_prompt_str: str + fs: LocalFS + _run_workflow: Callable[..., Awaitable[WorkflowState]] + _get_context: Callable[[], Context] + _get_database: Callable[[], Database] + _get_step_llm_client: Callable[[Mapping[str, Any] | None], Any] + _get_step_embedding_client: Callable[[Mapping[str, Any] | None], Any] + _get_embedding_client: Callable[..., Any] + _get_llm_client: Callable[..., Any] + _get_vlm_client: Callable[..., Any] + _model_dump_without_embeddings: Callable[[BaseModel], dict[str, Any]] + _extract_json_blob: Callable[[str], str] + _escape_prompt_value: Callable[[str], str] + user_model: type[BaseModel] + + # Memory file system export (provided by MemoryService). + memory_files_config: MemoryFilesConfig + _build_memory_files: Callable[..., Awaitable[dict[str, Any]]] + + # Provided by CRUDMixin (composed onto MemoryService). + async def _patch_category_summaries( + self, + updates: dict[str, tuple[str | None, str | None]], + ctx: Context, + store: Database, + llm_client: Any | None = None, + ) -> None: ... + + async def memorize_workspace( + self, + *, + folder: str, + user: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Sync a folder of source files into memory by diffing an input manifest. + + Scans ``folder`` recursively, infers each file's modality by extension + (unsupported extensions are skipped), and diffs against the sidecar + ``.memu_manifest.json`` to find added/modified/deleted files. Modified and + deleted files have their previously extracted memory cascade-deleted (with + affected category summaries recomputed); added and modified files are + (re)memorized by submitting each one through the single-file + :meth:`memorize` workflow. The manifest is then rewritten. + + ``memorize`` itself is left untouched: this is purely an additive, + directory-oriented entry point built on top of it. + """ + ctx = self._get_context() + store = self._get_database() + user_scope = self.user_model(**user).model_dump() if user is not None else None + await self._ensure_categories_ready(ctx, store, user_scope) + + root = pathlib.Path(folder).resolve() + scanned = scan_folder(root) + manifest = load_manifest(root) + diff = diff_folder(scanned, manifest) + + # 1. Cascade-delete memory for files that were modified or removed. + stale_urls = {sf.abs_path for sf in diff.modified} + stale_urls.update(str(root / rel) for rel in diff.deleted) + removed_resources = await self._cascade_delete_by_urls(stale_urls, ctx=ctx, store=store, user_scope=user_scope) + + # 2. (Re)memorize added and modified files; each file maps to one Resource. + changed_resources: list[Resource] = [] + entries: list[dict[str, Any]] = [] + files: list[dict[str, Any]] = [] + for scanned_file in [*diff.added, *diff.modified]: + result = await self._memorize_one( + resource_url=scanned_file.abs_path, + modality=scanned_file.modality, + user_scope=user_scope, + ctx=ctx, + store=store, + track=self._classify_track(scanned_file.rel_path), + ) + changed_resources.extend(cast("list[Resource]", result.get("resources") or [])) + # The inner single-file ``memorize`` keeps its legacy response keys + # (``items``/``categories``); translate them to the new vocabulary here. + response = cast("dict[str, Any]", result.get("response") or {}) + entries.extend(response.get("items", [])) + # Files reflect the cumulative scoped state, so the latest wins. + if response.get("categories"): + files = response["categories"] + + # 3. Refresh the memory file tree (full rebuild when anything was removed). + await self._update_memory_files(changed_resources, user_scope, force_full=diff.has_removals) + + # 4. Persist the updated input manifest. + save_manifest(root, manifest_from_scan(scanned)) + + return { + "folder": str(root), + "added": [sf.rel_path for sf in diff.added], + "modified": [sf.rel_path for sf in diff.modified], + "deleted": list(diff.deleted), + "resources": [self._model_dump_without_embeddings(r) for r in changed_resources], + "removed_resources": [self._model_dump_without_embeddings(r) for r in removed_resources], + "entries": entries, + "files": files, + } + + async def _memorize_one( + self, + *, + resource_url: str, + modality: str, + user_scope: dict[str, Any] | None, + ctx: Context, + store: Database, + track: str | None = None, + ) -> WorkflowState: + """Run the memorize workflow for a single file (one file -> one Resource). + + This mirrors :meth:`memorize` but returns the full workflow state (so the + workspace sync can collect the created resources) and takes an already + resolved ``user_scope``/``ctx``/``store`` to avoid re-resolving them per file. + """ + memory_types = self._resolve_memory_types() + state: WorkflowState = { + "resource_url": resource_url, + "modality": modality, + "memory_types": memory_types, + "categories_prompt_str": self._category_prompt_str, + "ctx": ctx, + "store": store, + "category_ids": list(ctx.category_ids), + "user": user_scope, + # Workspace sync path: let the extractor grow the taxonomy. + "allow_new_categories": True, + # Which workspace track this file belongs to (chat/skill/workspace). + "resource_track": track, + } + # The workspace path runs its own workflow (memorize + per-file skill + # generation); single-file ``memorize`` stays untouched (ADR 0006). + result = await self._run_workflow("memorize_workspace", state) + if result.get("response") is None: + msg = "Memorize workflow failed to produce a response" + raise RuntimeError(msg) + return result + + @staticmethod + def _classify_track(rel_path: str) -> str: + """Classify a workspace file into a track by its top-level folder. + + Files under ``chat/`` are the ``"chat"`` track, files under ``agent/`` are + the ``"skill"`` track, and everything else is the ``"workspace"`` track. + ``rel_path`` is the posix path relative to the scanned folder root. + """ + top = rel_path.split("/", 1)[0] + if top == "chat": + return "chat" + if top == "agent": + return "skill" + return "workspace" + + async def _cascade_delete_by_urls( + self, + urls: set[str], + *, + ctx: Context, + store: Database, + user_scope: dict[str, Any] | None, + ) -> list[Resource]: + """Delete resources (and their items/relations) whose url is in ``urls``. + + Affected category summaries are recomputed so the structured memory stays + consistent after a source file is changed or removed. + """ + if not urls: + return [] + where = user_scope or None + targets = [res for res in store.resource_repo.list_resources(where=where).values() if res.url in urls] + if not targets: + return [] + target_ids = {res.id for res in targets} + + # Discarded entry summaries per file, used to recompute summaries. Only the + # legacy entry-plane path (single-file ``memorize``) populates these. + file_discards: dict[str, list[str]] = {} + for entry in store.recall_entry_repo.list_items(where=where).values(): + if entry.resource_id not in target_ids: + continue + for relation in store.recall_file_entry_repo.get_item_categories(entry.id): + store.recall_file_entry_repo.unlink_item_category(entry.id, relation.category_id) + file_discards.setdefault(relation.category_id, []).append(entry.summary) + store.recall_entry_repo.delete_item(entry.id) + + for res in targets: + # Drop the resource -> file provenance links for the new synthesis path. + # NOTE (ADR 0007 phase 1 open issue): we do not rebuild the affected files + # from their remaining linked resources, so their content may go stale after + # a source change/delete. Tolerated for now. + store.recall_file_resource_repo.unlink_resource(res.id) + store.resource_repo.delete_resource(res.id) + + updates: dict[str, tuple[str | None, str | None]] = { + cid: ("\n".join(s for s in summaries if s and s.strip()), None) + for cid, summaries in file_discards.items() + if any(s and s.strip() for s in summaries) + } + if updates: + await self._patch_category_summaries(updates, ctx=ctx, store=store, llm_client=self._get_llm_client()) + return targets + + async def _update_memory_files( + self, + changed_resources: list[Resource], + user_scope: dict[str, Any] | None, + *, + force_full: bool = False, + ) -> None: + """Refresh the memory file tree after a workspace sync (init or incremental). + + Gated behind ``memory_files_config.enabled`` so a sync without the export + feature configured is a no-op. When any file was modified or deleted + (``force_full``), the tree is rebuilt from the full scoped store so stale + skills/entries do not linger; otherwise an incremental update merges the + just-created resources. Best-effort: the structured memory is already + persisted, so an export error must not fail the sync. + """ + if not getattr(self.memory_files_config, "enabled", False): + return + if not changed_resources and not force_full: + return + try: + await self._build_memory_files(user_scope, changed=None if force_full else changed_resources) + except Exception: + logger.exception("Memory file export failed after workspace memorize") + + def _build_memorize_workspace_workflow(self) -> list[WorkflowStep]: + """The workspace memorize pipeline: direct resource -> file synthesis (ADR 0007 phase 1). + + Unlike single-file :meth:`memorize` (``resource -> entry -> file``), the + workspace path synthesizes files straight from the preprocessed source and + creates no ``RecallEntry``. After ``preprocess`` it: + + - ``create_resource`` — one file maps to one :class:`Resource` (caption/embedding + for INDEX recall), for every track including ``workspace`` (resource-only). + - ``synthesize_files`` — for the ``chat`` and ``skill`` tracks only, route the + source to the files to update/create then synthesize each file's body, upserting + ``RecallFile`` and recording ``resource -> file`` provenance. ``workspace`` is a + no-op here. Retrieval over these files is deferred (ADR 0007 phase 2). + """ + synthesis_profile = getattr(self.memory_files_config, "synthesis_llm_profile", "default") + return [ + WorkflowStep( + step_id="ingest_resource", + role="ingest", + handler=self._memorize_ingest_resource, + requires={"resource_url", "modality"}, + produces={"local_path", "raw_text"}, + capabilities={"io"}, + ), + WorkflowStep( + step_id="preprocess_multimodal", + role="preprocess", + handler=self._memorize_preprocess_multimodal, + requires={"local_path", "modality", "raw_text"}, + produces={"preprocessed_resources"}, + capabilities={"llm"}, + config={"chat_llm_profile": self.memorize_config.preprocess_llm_profile}, + ), + WorkflowStep( + step_id="create_resource", + role="persist", + handler=self._memorize_ws_create_resource, + requires={ + "preprocessed_resources", + "modality", + "local_path", + "resource_url", + "store", + "user", + "resource_track", + }, + produces={"resources"}, + capabilities={"db", "vector"}, + config={"embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="synthesize_files", + role="synthesize_files", + handler=self._memorize_ws_synthesize_files, + requires={"resources", "preprocessed_resources", "resource_track", "store", "user"}, + produces={"files"}, + capabilities={"llm", "db", "vector"}, + config={"chat_llm_profile": synthesis_profile, "embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="build_response", + role="emit", + handler=self._memorize_ws_build_response, + requires={"resources", "files"}, + produces={"response"}, + capabilities=set(), + ), + ] + + @staticmethod + def _list_memorize_initial_keys() -> set[str]: + return { + "resource_url", + "modality", + "memory_types", + "categories_prompt_str", + "ctx", + "store", + "category_ids", + "user", + "allow_new_categories", + "resource_track", + } + + async def _memorize_ingest_resource(self, state: WorkflowState, step_context: Any) -> WorkflowState: + local_path, raw_text = await self.fs.fetch(state["resource_url"], state["modality"]) + state.update({"local_path": local_path, "raw_text": raw_text}) + return state + + # Modalities whose preprocessing analyzes media via the VLM (vision) client. + _VISION_MODALITIES = frozenset({"image", "video"}) + + async def _memorize_preprocess_multimodal(self, state: WorkflowState, step_context: Any) -> WorkflowState: + modality = state["modality"] + client = self._get_step_llm_client(step_context) + if modality in self._VISION_MODALITIES: + with contextlib.suppress(KeyError): + client = self._get_vlm_client(self.memorize_config.vlm_profile, step_context=step_context) + preprocessed = await self._preprocess_resource_url( + local_path=state["local_path"], + text=state.get("raw_text"), + modality=modality, + llm_client=client, + ) + if not preprocessed: + preprocessed = [{"text": state.get("raw_text"), "caption": None}] + state["preprocessed_resources"] = preprocessed + return state + + @staticmethod + def _format_skill_source_content(preprocessed_resources: list[dict[str, Any]]) -> str: + """Flatten a source's preprocessed segments into a single text block.""" + parts = [ + " ".join((prep.get("text") or "").split()) + for prep in preprocessed_resources + if (prep.get("text") or "").strip() + ] + return "\n\n".join(parts) + + # --- Workspace resource -> file path (ADR 0007 phase 1) ------------------- + + # Maps a workspace ``resource_track`` to the ``RecallFile.track`` it synthesizes + # into. ``workspace`` has no entry (resource-only), so it is absent. + _TRACK_TO_FILE_TRACK: ClassVar[dict[str, str]] = {"chat": "memory", "skill": "skill"} + + async def _memorize_ws_create_resource(self, state: WorkflowState, step_context: Any) -> WorkflowState: + """Create the single ``Resource`` for this file (one file -> one resource). + + Runs for every track; the ``workspace`` track stops here (resource-only). The + caption is the joined per-segment captions, embedded for INDEX/resource recall. + """ + embed_client = self._get_step_embedding_client(step_context) + store = state["store"] + preprocessed = state.get("preprocessed_resources") or [] + captions = [(prep.get("caption") or "").strip() for prep in preprocessed] + caption = "\n\n".join(c for c in captions if c) or None + res = await self._create_resource_with_caption( + resource_url=state["resource_url"], + modality=state["modality"], + local_path=state["local_path"], + caption=caption, + store=store, + embed_client=embed_client, + user=state.get("user", {}), + track=state.get("resource_track"), + ) + state["resources"] = [res] + return state + + async def _memorize_ws_synthesize_files(self, state: WorkflowState, step_context: Any) -> WorkflowState: + """Synthesize this source into ``RecallFile``s for the chat/skill tracks. + + Two steps: (a) route the source to the set of files to update/create given the + existing files' names+descriptions, and (b) synthesize each target file's body in + parallel. Persists each file and a ``resource -> file`` provenance link. The + ``workspace`` track (and any source with no content) is a no-op. + """ + track = state.get("resource_track") + file_track = self._TRACK_TO_FILE_TRACK.get(track or "") + resources = state.get("resources") or [] + content = self._format_skill_source_content(state.get("preprocessed_resources") or []) + if file_track is None or not resources or not content: + state["files"] = [] + return state + + store = state["store"] + user_scope = dict(state.get("user") or {}) + llm_client = self._get_step_llm_client(step_context) + embed_client = self._get_step_embedding_client(step_context) + resource = resources[0] + + existing = store.recall_file_repo.list_categories(where={**user_scope, "track": file_track}) + ops = await self._route_source_to_files( + file_track=file_track, content=content, existing=existing, llm_client=llm_client + ) + touched = await self._synthesize_file_ops( + ops=ops, + file_track=file_track, + content=content, + existing=existing, + resource=resource, + store=store, + user_scope=user_scope, + llm_client=llm_client, + embed_client=embed_client, + ) + await self._sync_file_segments( + files=touched, + file_track=file_track, + store=store, + user_scope=user_scope, + embed_client=embed_client, + ) + state["files"] = touched + return state + + @staticmethod + def _segment_texts_for_file(file: RecallFile, file_track: str) -> list[str]: + """Compute the searchable segment texts for a synthesized file (ADR 0007 L2 items). + + The slicing rule is track-specific: + + - ``skill``: a single ``name: ...\\ndescription: ...`` segment for the whole skill. + - ``memory``: one segment per content line, skipping blank lines and markdown + headings (lines starting with one or more ``#``). + + Texts are stripped and de-duplicated while preserving order so a repeated line is + embedded only once. + """ + if file_track == "skill": + return [f"name: {file.name}\ndescription: {file.description}"] + + texts: list[str] = [] + for line in (file.content or "").split("\n"): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + texts.append(stripped) + return list(dict.fromkeys(texts)) + + async def _sync_file_segments( + self, + *, + files: list[RecallFile], + file_track: str, + store: Database, + user_scope: dict[str, Any], + embed_client: Any, + ) -> None: + """Reconcile each file's stored segments with its freshly computed segment texts. + + Diffs the new segment texts against the existing ones and does a drop-and-add on the + difference only: segments whose text disappeared are deleted, and only genuinely new + texts are embedded and inserted. Unchanged lines keep their existing embedding, so an + edit that touches a few lines does not re-embed the whole file. + """ + for file in files: + new_texts = self._segment_texts_for_file(file, file_track) + existing = store.recall_file_segment_repo.list_segments_for_file(file.id) + existing_texts = {seg.text for seg in existing} + new_set = set(new_texts) + + for seg in existing: + if seg.text not in new_set: + store.recall_file_segment_repo.delete_segment(seg.id) + + to_add = [text for text in new_texts if text not in existing_texts] + if not to_add: + continue + vecs = await embed_client.embed(to_add) + for text, vec in zip(to_add, vecs, strict=True): + store.recall_file_segment_repo.create_segment( + recall_file_id=file.id, track=file_track, text=text, embedding=vec, user_data=dict(user_scope) + ) + + async def _route_source_to_files( + self, + *, + file_track: str, + content: str, + existing: Mapping[str, RecallFile], + llm_client: Any, + ) -> list[dict[str, str]]: + """Ask the model which existing files to update / what new files to create.""" + existing_text = self._format_existing_files(existing) or "(none)" + prompt = ( + ROUTE_PROMPTS[file_track] + .replace(EXISTING_PLACEHOLDER, existing_text) + .replace(CONTENT_PLACEHOLDER, self._escape_prompt_value(content)) + ) + return self._parse_file_ops(await llm_client.chat(prompt), existing) + + async def _synthesize_file_ops( + self, + *, + ops: list[dict[str, str]], + file_track: str, + content: str, + existing: Mapping[str, RecallFile], + resource: Resource, + store: Database, + user_scope: dict[str, Any], + llm_client: Any, + embed_client: Any, + ) -> list[RecallFile]: + """Synthesize each routed file's body (in parallel) and persist file + link.""" + existing_by_name = {f.name: f for f in existing.values()} + # Resolve ops to unique targets (dedup by name; last op's description wins). + targets: list[dict[str, Any]] = [] + by_name: dict[str, dict[str, Any]] = {} + for op in ops: + name = op["name"] + ex = existing_by_name.get(name) + description = (op.get("description") or (ex.description if ex else "") or "").strip() + target = by_name.get(name) + if target is None: + target = {"name": name, "description": description, "existing": ex} + by_name[name] = target + targets.append(target) + elif description: + target["description"] = description + if not targets: + return [] + + prompts = [ + SYNTHESIS_PROMPTS[file_track] + .replace(NAME_PLACEHOLDER, self._escape_prompt_value(t["name"])) + .replace(DESCRIPTION_PLACEHOLDER, self._escape_prompt_value(t["description"])) + .replace( + EXISTING_PLACEHOLDER, self._escape_prompt_value((t["existing"].content if t["existing"] else "") or "") + ) + .replace(CONTENT_PLACEHOLDER, self._escape_prompt_value(content)) + for t in targets + ] + bodies = await asyncio.gather(*[llm_client.chat(prompt) for prompt in prompts]) + + # Embed name+description for the files being created. + creates = [t for t in targets if t["existing"] is None] + create_vecs: dict[str, list[float]] = {} + if creates: + emb_texts = [f"{t['name']}: {t['description']}" if t["description"] else t["name"] for t in creates] + vecs = await embed_client.embed(emb_texts) + for t, vec in zip(creates, vecs, strict=True): + create_vecs[t["name"]] = vec + + touched: list[RecallFile] = [] + for target, body in zip(targets, bodies, strict=True): + cleaned = body.replace("```markdown", "").replace("```", "").strip() + file = target["existing"] + if file is None: + file = store.recall_file_repo.get_or_create_category( + name=target["name"], + description=target["description"], + embedding=create_vecs[target["name"]], + user_data=user_scope, + track=file_track, + ) + file = store.recall_file_repo.update_category(category_id=file.id, content=cleaned) + store.recall_file_resource_repo.link_resource_category(resource.id, file.id, user_data=dict(user_scope)) + touched.append(file) + return touched + + @staticmethod + def _format_existing_files(existing: Mapping[str, RecallFile]) -> str: + """Render existing files as ``- name: description`` lines for the router prompt.""" + return "\n".join( + f"- {f.name}: {f.description}" if f.description else f"- {f.name}" + for f in sorted(existing.values(), key=lambda f: f.name) + ) + + def _parse_file_ops(self, raw: str, existing: Mapping[str, RecallFile]) -> list[dict[str, str]]: + """Parse the router's JSON array into validated ``{op, name, description}`` dicts. + + ``update`` ops naming an unknown file are dropped (we never update a file that + does not exist); ``create``/``update`` are otherwise kept with a stripped name. + """ + if not raw: + return [] + start = raw.find("[") + end = raw.rfind("]") + if start == -1 or end == -1 or end <= start: + return [] + try: + parsed = json.loads(raw[start : end + 1]) + except (json.JSONDecodeError, TypeError): + return [] + if not isinstance(parsed, list): + return [] + existing_names = {f.name for f in existing.values()} + ops: list[dict[str, str]] = [] + for entry in parsed: + if not isinstance(entry, dict): + continue + op = entry.get("op") + name = entry.get("name") + if op not in {"update", "create"} or not isinstance(name, str) or not name.strip(): + continue + name = name.strip() + if op == "update" and name not in existing_names: + continue + description = entry.get("description") + description = description.strip() if isinstance(description, str) else "" + ops.append({"op": op, "name": name, "description": description}) + return ops + + def _memorize_ws_build_response(self, state: WorkflowState, step_context: Any) -> WorkflowState: + """Emit the workspace response (no entries; ``categories`` carries touched files).""" + resources = [self._model_dump_without_embeddings(r) for r in state.get("resources", [])] + files = [self._model_dump_without_embeddings(f) for f in state.get("files", [])] + # Keep the legacy response contract (``items``/``categories``); items is always + # empty on this path since the entry plane is gone. + base: dict[str, Any] = {"items": [], "categories": files, "relations": []} + if len(resources) == 1: + state["response"] = {"resource": resources[0], **base} + else: + state["response"] = {"resources": resources, **base} + return state + + async def _create_resource_with_caption( + self, + *, + resource_url: str, + modality: str, + local_path: str, + caption: str | None, + store: Database, + embed_client: Any | None = None, + user: Mapping[str, Any] | None = None, + track: str | None = None, + ) -> Resource: + caption_text = caption.strip() if caption else None + if caption_text: + client = embed_client or self._get_embedding_client() + caption_embedding = (await client.embed([caption_text]))[0] + else: + caption_embedding = None + + res = store.resource_repo.create_resource( + url=resource_url, + modality=modality, + local_path=local_path, + caption=caption_text, + embedding=caption_embedding, + user_data=dict(user or {}), + track=track, + ) + return res + + def _resolve_memory_types(self) -> list[EntryType]: + configured_types = self.memorize_config.memory_types or DEFAULT_MEMORY_TYPES + return [cast(EntryType, mtype) for mtype in configured_types] + + @staticmethod + def _resolve_custom_prompt(prompt: str | CustomPrompt, templates: Mapping[str, str]) -> str: + if isinstance(prompt, str): + return prompt + valid_blocks = [ + (block.ordinal, name, block.prompt or templates.get(name)) + for name, block in prompt.items() + if (block.ordinal >= 0 and (block.prompt or templates.get(name))) + ] + if not valid_blocks: + # raise ValueError(f"No valid blocks contained in custom prompt: {prompt}") + return "" + sorted_blocks = sorted(valid_blocks) + return "\n\n".join(block for (_, _, block) in sorted_blocks if block is not None) + + async def _preprocess_resource_url( + self, *, local_path: str, text: str | None, modality: str, llm_client: Any | None = None + ) -> list[dict[str, str | None]]: + """Preprocess a resource by delegating to the per-format ``preprocess`` package. + + Returns a list of preprocessed resources, each with 'text' and 'caption'. + """ + return await preprocess_resource( + modality=modality, + local_path=local_path, + text=text, + ctx=self._build_preprocess_context(), + llm_client=llm_client, + ) + + def _build_preprocess_context(self) -> PreprocessContext: + """Bundle the service dependencies the preprocessors need.""" + return PreprocessContext( + get_llm_client=self._get_llm_client, + get_vlm_client=lambda: self._get_vlm_client(self.memorize_config.vlm_profile), + escape_prompt_value=self._escape_prompt_value, + extract_json_blob=self._extract_json_blob, + resolve_custom_prompt=self._resolve_custom_prompt, + multimodal_preprocess_prompts=self.memorize_config.multimodal_preprocess_prompts, + ) + + async def _ensure_categories_ready( + self, ctx: Context, store: Database, user_scope: Mapping[str, Any] | None = None + ) -> None: + if ctx.categories_ready: + return + if ctx.category_init_task: + await ctx.category_init_task + ctx.category_init_task = None + return + await self._initialize_categories(ctx, store, user_scope) + + @staticmethod + def _classify_categories( + configs: list[CategoryConfig], + existing_by_name: dict[str, RecallFile], + ) -> tuple[ + list[tuple[int, CategoryConfig]], + list[tuple[int, CategoryConfig, RecallFile]], + dict[int, RecallFile], + ]: + to_create: list[tuple[int, CategoryConfig]] = [] + to_update: list[tuple[int, CategoryConfig, RecallFile]] = [] + ready: dict[int, RecallFile] = {} + for i, cfg in enumerate(configs): + name = cfg.name.strip() or "Untitled" + description = cfg.description.strip() + ex = existing_by_name.get(name) + if ex is None: + to_create.append((i, cfg)) + elif ex.embedding is None or (ex.description or "") != description: + to_update.append((i, cfg, ex)) + else: + ready[i] = ex + return to_create, to_update, ready + + async def _initialize_categories( + self, ctx: Context, store: Database, user: Mapping[str, Any] | None = None + ) -> None: + if ctx.categories_ready: + return + if not self.category_configs: + ctx.categories_ready = True + return + + user_data = dict(user or {}) + existing = store.recall_file_repo.list_categories(where={**user_data, "track": "memory"}) + existing_by_name: dict[str, RecallFile] = {c.name: c for c in existing.values()} + + to_create, to_update, ready = self._classify_categories(self.category_configs, existing_by_name) + + needs_embed: list[tuple[int, CategoryConfig]] = [] + needs_embed.extend(to_create) + needs_embed.extend((i, cfg) for i, cfg, _ in to_update) + + embed_map: dict[int, list[float]] = {} + if needs_embed: + texts = [self._category_embedding_text(cfg) for _, cfg in needs_embed] + vecs = await self._get_embedding_client("embedding").embed(texts) + for (i, _), vec in zip(needs_embed, vecs, strict=True): + embed_map[i] = vec + + cats: dict[int, RecallFile] = dict(ready) + + for i, cfg in to_create: + name = cfg.name.strip() or "Untitled" + description = cfg.description.strip() + cat = store.recall_file_repo.get_or_create_category( + name=name, description=description, embedding=embed_map[i], user_data=user_data + ) + cats[i] = cat + + for i, cfg, ex in to_update: + description = cfg.description.strip() + cat = store.recall_file_repo.update_category( + category_id=ex.id, description=description, embedding=embed_map[i] + ) + cats[i] = cat + + ctx.category_ids = [] + ctx.category_name_to_id = {} + for i in range(len(self.category_configs)): + cat = cats[i] + ctx.category_ids.append(cat.id) + name = self.category_configs[i].name.strip() or "Untitled" + ctx.category_name_to_id[name.lower()] = cat.id + ctx.categories_ready = True + + @staticmethod + def _category_embedding_text(cat: CategoryConfig) -> str: + name = cat.name.strip() or "Untitled" + desc = cat.description.strip() + return f"{name}: {desc}" if desc else name diff --git a/src/memu/app/memory_files.py b/src/memu/app/memory_files.py new file mode 100644 index 0000000..82a2ecf --- /dev/null +++ b/src/memu/app/memory_files.py @@ -0,0 +1,101 @@ +"""Memory file-system build orchestration, extracted from ``MemoryService``. + +This collaborator owns the markdown "memory file system" artifact layer: the +exporter, the LLM synthesizer, and the write lock. ``MemoryService`` keeps a +thin ``_build_memory_files`` delegate so this stays an internal composition +detail, but the init-vs-incremental synthesis policy lives here rather than +bloating the service composition root. +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any + +from memu.memory_fs import ExistingArtifacts, ExportResult, MemoryFileExporter, MemorySynthesizer + +if TYPE_CHECKING: + from collections.abc import Callable + + from memu.app.settings import MemoryFilesConfig + from memu.database.interfaces import Database + + +class MemoryFilesBuilder: + """Initialize or incrementally update the markdown memory file tree. + + Holds the exporter/synthesizer/lock so the build policy is testable and + decoupled from the rest of the service. Writes are serialized through a + single lock so concurrent exports never interleave on the output directory. + """ + + def __init__(self, *, config: MemoryFilesConfig) -> None: + self.config = config + self.exporter = MemoryFileExporter(config.output_dir) + self.synthesizer = MemorySynthesizer() + self.lock = asyncio.Lock() + + async def build( + self, + database: Database, + where: dict[str, Any] | None, + *, + changed: list[Any] | None, + make_client: Callable[[str], Any], + ) -> dict[str, Any]: + """Initialize or incrementally update the memory file tree. + + ``changed`` is the list of just-memorized ``Resource`` objects driving an + incremental update. When it is ``None`` (or no prior tree exists), the + tree is (re)initialized from the full scoped store. ``make_client`` builds + the LLM client used for synthesis from a profile name. + + LLM work only happens when ``synthesize=True``: the root overview documents + MEMORY.md and SKILL.md are synthesized (MEMORY.md from the per-source + descriptions, SKILL.md from the skill-track files' bodies). Otherwise both are + left as ``None`` so the exporter renders each deterministically as a link index + over its per-file payloads. The ``memory/`` and ``skill/`` trees themselves are + always rendered deterministically from the DB (ADR 0006). + """ + memory_body: str | None = None + skill_body: str | None = None + + if self.config.synthesize: + # MEMORY.md trunk: the just-changed sources for an incremental update, + # otherwise the full in-scope store. SKILL.md is synthesized from the full + # skill set regardless (the workflow already merges skills per source). + is_update = changed is not None and self.exporter.artifacts_exist() + if is_update: + descriptions = MemoryFileExporter._build_descriptions(changed) # type: ignore[arg-type] + else: + resources = list(database.resource_repo.list_resources(where=where or None).values()) + descriptions = MemoryFileExporter._build_descriptions(resources) + skill_files = list( + database.recall_file_repo.list_categories(where={**(where or {}), "track": "skill"}).values() + ) + + # An incremental update merges into the prior overviews; a full + # (re)initialization starts from empty existing state. + existing = await asyncio.to_thread(self.exporter.read_existing) if is_update else ExistingArtifacts() + client = make_client(self.config.synthesis_llm_profile) + memory_body, skill_body = await asyncio.gather( + self.synthesizer.synthesize_memory( + descriptions, existing_memory=existing.memory_body, chat=client.chat + ), + self.synthesizer.synthesize_skill_overview( + skill_files, existing_skill=existing.skill_body, chat=client.chat + ), + ) + + async with self.lock: + result: ExportResult = await asyncio.to_thread( + self.exporter.export, + database, + where=where, + memory_body=memory_body, + skill_body=skill_body, + ) + return result.to_dict() + + +__all__ = ["MemoryFilesBuilder"] diff --git a/src/memu/app/retrieve.py b/src/memu/app/retrieve.py new file mode 100644 index 0000000..1e17c82 --- /dev/null +++ b/src/memu/app/retrieve.py @@ -0,0 +1,1427 @@ +from __future__ import annotations + +import json +import logging +import re +from collections.abc import Awaitable, Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, cast + +from pydantic import BaseModel + +from memu.prompts.retrieve.llm_category_ranker import PROMPT as LLM_CATEGORY_RANKER_PROMPT +from memu.prompts.retrieve.llm_item_ranker import PROMPT as LLM_ITEM_RANKER_PROMPT +from memu.prompts.retrieve.llm_resource_ranker import PROMPT as LLM_RESOURCE_RANKER_PROMPT +from memu.prompts.retrieve.pre_retrieval_decision import SYSTEM_PROMPT as PRE_RETRIEVAL_SYSTEM_PROMPT +from memu.prompts.retrieve.pre_retrieval_decision import USER_PROMPT as PRE_RETRIEVAL_USER_PROMPT +from memu.vector import cosine_topk +from memu.workflow.step import WorkflowState, WorkflowStep + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from memu.app.service import Context + from memu.app.settings import RetrieveConfig + from memu.database.interfaces import Database + + +class RetrieveMixin: + if TYPE_CHECKING: + retrieve_config: RetrieveConfig + _run_workflow: Callable[..., Awaitable[WorkflowState]] + _get_context: Callable[[], Context] + _get_database: Callable[[], Database] + _ensure_categories_ready: Callable[[Context, Database], Awaitable[None]] + _get_step_llm_client: Callable[[Mapping[str, Any] | None], Any] + _get_step_embedding_client: Callable[[Mapping[str, Any] | None], Any] + _get_llm_client: Callable[..., Any] + _get_embedding_client: Callable[..., Any] + _model_dump_without_embeddings: Callable[[BaseModel], dict[str, Any]] + _extract_json_blob: Callable[[str], str] + _escape_prompt_value: Callable[[str], str] + user_model: type[BaseModel] + + async def retrieve( + self, + queries: list[dict[str, Any]], + where: dict[str, Any] | None = None, + ) -> dict[str, Any]: + if not queries: + raise ValueError("empty_queries") + ctx = self._get_context() + store = self._get_database() + original_query = self._extract_query_text(queries[-1]) + # await self._ensure_categories_ready(ctx, store) + where_filters = self._normalize_where(where) + + context_queries_objs = queries[:-1] if len(queries) > 1 else [] + + route_intention = self.retrieve_config.route_intention + retrieve_file = self.retrieve_config.category.enabled + retrieve_entry = self.retrieve_config.item.enabled + retrieve_resource = self.retrieve_config.resource.enabled + sufficiency_check = self.retrieve_config.sufficiency_check + + workflow_name = "retrieve_llm" if self.retrieve_config.method == "llm" else "retrieve_rag" + + state: WorkflowState = { + "method": self.retrieve_config.method, + "original_query": original_query, + "context_queries": context_queries_objs, + "route_intention": route_intention, + "skip_rewrite": len(queries) == 1, + "retrieve_file": retrieve_file, + "retrieve_entry": retrieve_entry, + "retrieve_resource": retrieve_resource, + "sufficiency_check": sufficiency_check, + "ctx": ctx, + "store": store, + "where": where_filters, + } + + result = await self._run_workflow(workflow_name, state) + response = cast(dict[str, Any] | None, result.get("response")) + if response is None: + msg = "Retrieve workflow failed to produce a response" + raise RuntimeError(msg) + return response + + def _normalize_where(self, where: Mapping[str, Any] | None) -> dict[str, Any]: + """Validate and clean the `where` scope filters against the configured user model.""" + if not where: + return {} + + valid_fields = set(getattr(self.user_model, "model_fields", {}).keys()) + cleaned: dict[str, Any] = {} + + for raw_key, value in where.items(): + if value is None: + continue + field = raw_key.split("__", 1)[0] + if field not in valid_fields: + msg = f"Unknown filter field '{field}' for current user scope" + raise ValueError(msg) + cleaned[raw_key] = value + + return cleaned + + @staticmethod + def _memory_where(where_filters: Mapping[str, Any] | None) -> dict[str, Any]: + """Scope a category ``where`` filter to the memory track. + + Retrieval ranks/recalls memory categories only; skill-track ``RecallFile``s + (ADR 0006) live in the same table and must be excluded. + """ + return {**(where_filters or {}), "track": "memory"} + + @staticmethod + def _memory_categories(store: Database) -> dict[str, Any]: + """The category cache filtered to the memory track (skills excluded).""" + return { + cid: c for cid, c in store.recall_file_repo.categories.items() if getattr(c, "track", "memory") == "memory" + } + + def _build_rag_retrieve_workflow(self) -> list[WorkflowStep]: + steps = [ + WorkflowStep( + step_id="route_intention", + role="route_intention", + handler=self._rag_route_intention, + requires={"route_intention", "original_query", "context_queries", "skip_rewrite"}, + produces={"needs_retrieval", "rewritten_query", "active_query", "next_step_query"}, + capabilities={"llm"}, + config={"chat_llm_profile": self.retrieve_config.sufficiency_check_llm_profile}, + ), + WorkflowStep( + step_id="route_file", + role="route_file", + handler=self._rag_route_file, + requires={"retrieve_file", "needs_retrieval", "active_query", "ctx", "store", "where"}, + produces={"file_hits", "file_summary_lookup", "query_vector"}, + capabilities={"vector"}, + config={"embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="sufficiency_after_file", + role="sufficiency_check", + handler=self._rag_file_sufficiency, + requires={ + "retrieve_file", + "needs_retrieval", + "active_query", + "context_queries", + "file_hits", + "ctx", + "store", + "where", + }, + produces={"next_step_query", "proceed_to_entries", "query_vector"}, + capabilities={"llm"}, + config={ + "chat_llm_profile": self.retrieve_config.sufficiency_check_llm_profile, + "embed_llm_profile": "embedding", + }, + ), + WorkflowStep( + step_id="recall_entries", + role="recall_entries", + handler=self._rag_recall_entries, + requires={ + "needs_retrieval", + "proceed_to_entries", + "ctx", + "store", + "where", + "active_query", + "query_vector", + }, + produces={"entry_hits", "query_vector"}, + capabilities={"vector"}, + config={"embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="sufficiency_after_entries", + role="sufficiency_check", + handler=self._rag_entry_sufficiency, + requires={ + "needs_retrieval", + "active_query", + "context_queries", + "entry_hits", + "ctx", + "store", + "where", + }, + produces={"next_step_query", "proceed_to_resources", "query_vector"}, + capabilities={"llm"}, + config={ + "chat_llm_profile": self.retrieve_config.sufficiency_check_llm_profile, + "embed_llm_profile": "embedding", + }, + ), + WorkflowStep( + step_id="recall_resources", + role="recall_resources", + handler=self._rag_recall_resources, + requires={ + "needs_retrieval", + "proceed_to_resources", + "ctx", + "store", + "where", + "active_query", + "query_vector", + }, + produces={"resource_hits", "query_vector"}, + capabilities={"vector"}, + config={"embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="build_context", + role="build_context", + handler=self._rag_build_context, + requires={"needs_retrieval", "original_query", "rewritten_query", "ctx", "store", "where"}, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + def _list_retrieve_initial_keys(self) -> set[str]: + return { + "method", + "original_query", + "context_queries", + "route_intention", + "skip_rewrite", + "retrieve_file", + "retrieve_entry", + "retrieve_resource", + "sufficiency_check", + "ctx", + "store", + "where", + } + + async def _rag_route_intention(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("route_intention"): + state.update({ + "needs_retrieval": True, + "rewritten_query": state["original_query"], + "active_query": state["original_query"], + "next_step_query": None, + "proceed_to_entries": False, + "proceed_to_resources": False, + }) + return state + + llm_client = self._get_step_llm_client(step_context) + needs_retrieval, rewritten_query = await self._decide_if_retrieval_needed( + state["original_query"], + state["context_queries"], + retrieved_content=None, + llm_client=llm_client, + ) + if state.get("skip_rewrite"): + rewritten_query = state["original_query"] + + state.update({ + "needs_retrieval": needs_retrieval, + "rewritten_query": rewritten_query, + "active_query": rewritten_query, + "next_step_query": None, + "proceed_to_entries": False, + "proceed_to_resources": False, + }) + return state + + async def _rag_route_file(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("retrieve_file") or not state.get("needs_retrieval"): + state["file_hits"] = [] + state["file_summary_lookup"] = {} + state["query_vector"] = None + return state + + embed_client = self._get_step_embedding_client(step_context) + store = state["store"] + where_filters = state.get("where") or {} + file_pool = store.recall_file_repo.list_categories(self._memory_where(where_filters)) + qvec = (await embed_client.embed([state["active_query"]]))[0] + hits, summary_lookup = await self._rank_files_by_summary( + qvec, + self.retrieve_config.category.top_k, + state["ctx"], + store, + embed_client=embed_client, + categories=file_pool, + ) + state.update({ + "query_vector": qvec, + "file_hits": hits, + "file_summary_lookup": summary_lookup, + "file_pool": file_pool, + }) + return state + + async def _rag_file_sufficiency(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("needs_retrieval"): + state["proceed_to_entries"] = False + return state + if not state.get("retrieve_file") or not state.get("sufficiency_check"): + state["proceed_to_entries"] = True + return state + + retrieved_content = "" + store = state["store"] + where_filters = state.get("where") or {} + file_pool = state.get("file_pool") or store.recall_file_repo.list_categories(self._memory_where(where_filters)) + hits = state.get("file_hits") or [] + if hits: + retrieved_content = self._format_file_content( + hits, + state.get("file_summary_lookup", {}), + store, + categories=file_pool, + ) + + llm_client = self._get_step_llm_client(step_context) + needs_more, rewritten_query = await self._decide_if_retrieval_needed( + state["active_query"], + state["context_queries"], + retrieved_content=retrieved_content or "No content retrieved yet.", + llm_client=llm_client, + ) + state["next_step_query"] = rewritten_query + state["active_query"] = rewritten_query + state["proceed_to_entries"] = needs_more + if needs_more: + embed_client = self._get_step_embedding_client(step_context) + state["query_vector"] = (await embed_client.embed([state["active_query"]]))[0] + return state + + def _extract_referenced_entry_ids(self, state: WorkflowState) -> set[str]: + """Extract entry IDs from file summary references.""" + from memu.utils.references import extract_references + + file_hits = state.get("file_hits") or [] + summary_lookup = state.get("file_summary_lookup", {}) + file_pool = state.get("file_pool") or {} + referenced_entry_ids: set[str] = set() + + for cid, _score in file_hits: + # Get summary from lookup or category + summary = summary_lookup.get(cid) + if not summary: + cat = file_pool.get(cid) + if cat: + summary = cat.content + if summary: + refs = extract_references(summary) + referenced_entry_ids.update(refs) + + return referenced_entry_ids + + async def _rag_recall_entries(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("retrieve_entry") or not state.get("needs_retrieval") or not state.get("proceed_to_entries"): + state["entry_hits"] = [] + return state + + store = state["store"] + where_filters = state.get("where") or {} + entries_pool = store.recall_entry_repo.list_items(where_filters) + qvec = state.get("query_vector") + if qvec is None: + embed_client = self._get_step_embedding_client(step_context) + qvec = (await embed_client.embed([state["active_query"]]))[0] + state["query_vector"] = qvec + state["entry_hits"] = store.recall_entry_repo.vector_search_items( + qvec, + self.retrieve_config.item.top_k, + where=where_filters, + ranking=self.retrieve_config.item.ranking, + recency_decay_days=self.retrieve_config.item.recency_decay_days, + ) + state["entry_pool"] = entries_pool + return state + + async def _rag_entry_sufficiency(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("needs_retrieval"): + state["proceed_to_resources"] = False + return state + if not state.get("retrieve_entry") or not state.get("sufficiency_check"): + state["proceed_to_resources"] = True + return state + + store = state["store"] + where_filters = state.get("where") or {} + entries_pool = state.get("entry_pool") or store.recall_entry_repo.list_items(where_filters) + retrieved_content = "" + hits = state.get("entry_hits") or [] + if hits: + retrieved_content = self._format_entry_content(hits, store, items=entries_pool) + + llm_client = self._get_step_llm_client(step_context) + needs_more, rewritten_query = await self._decide_if_retrieval_needed( + state["active_query"], + state["context_queries"], + retrieved_content=retrieved_content or "No content retrieved yet.", + llm_client=llm_client, + ) + state["next_step_query"] = rewritten_query + state["active_query"] = rewritten_query + state["proceed_to_resources"] = needs_more + if needs_more: + embed_client = self._get_step_embedding_client(step_context) + state["query_vector"] = (await embed_client.embed([state["active_query"]]))[0] + return state + + async def _rag_recall_resources(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if ( + not state.get("needs_retrieval") + or not state.get("retrieve_resource") + or not state.get("proceed_to_resources") + ): + state["resource_hits"] = [] + return state + + store = state["store"] + where_filters = state.get("where") or {} + resource_pool = store.resource_repo.list_resources(where_filters) + state["resource_pool"] = resource_pool + if not resource_pool: + state["resource_hits"] = [] + return state + + qvec = state.get("query_vector") + if qvec is None: + embed_client = self._get_step_embedding_client(step_context) + qvec = (await embed_client.embed([state["active_query"]]))[0] + state["query_vector"] = qvec + state["resource_hits"] = store.resource_repo.vector_search_resources( + qvec, self.retrieve_config.resource.top_k, where=where_filters + ) + return state + + def _rag_build_context(self, state: WorkflowState, _: Any) -> WorkflowState: + response = { + "needs_retrieval": bool(state.get("needs_retrieval")), + "original_query": state["original_query"], + "rewritten_query": state.get("rewritten_query", state["original_query"]), + "next_step_query": state.get("next_step_query"), + "categories": [], + "items": [], + "resources": [], + } + if state.get("needs_retrieval"): + store = state["store"] + where_filters = state.get("where") or {} + files_pool = state.get("file_pool") or store.recall_file_repo.list_categories( + self._memory_where(where_filters) + ) + entries_pool = state.get("entry_pool") or store.recall_entry_repo.list_items(where_filters) + resources_pool = state.get("resource_pool") or store.resource_repo.list_resources(where_filters) + response["categories"] = self._materialize_hits( + state.get("file_hits", []), + files_pool, + ) + response["items"] = self._materialize_hits(state.get("entry_hits", []), entries_pool) + response["resources"] = self._materialize_hits( + state.get("resource_hits", []), + resources_pool, + ) + state["response"] = response + return state + + def _build_llm_retrieve_workflow(self) -> list[WorkflowStep]: + steps = [ + WorkflowStep( + step_id="route_intention", + role="route_intention", + handler=self._llm_route_intention, + requires={"original_query", "context_queries", "skip_rewrite"}, + produces={"needs_retrieval", "rewritten_query", "active_query", "next_step_query"}, + capabilities={"llm"}, + config={"llm_profile": self.retrieve_config.sufficiency_check_llm_profile}, + ), + WorkflowStep( + step_id="route_file", + role="route_file", + handler=self._llm_route_file, + requires={"needs_retrieval", "active_query", "ctx", "store", "where"}, + produces={"file_hits"}, + capabilities={"llm"}, + config={"llm_profile": self.retrieve_config.llm_ranking_llm_profile}, + ), + WorkflowStep( + step_id="sufficiency_after_file", + role="sufficiency_check", + handler=self._llm_file_sufficiency, + requires={"needs_retrieval", "active_query", "context_queries", "file_hits"}, + produces={"next_step_query", "proceed_to_entries"}, + capabilities={"llm"}, + config={"llm_profile": self.retrieve_config.sufficiency_check_llm_profile}, + ), + WorkflowStep( + step_id="recall_entries", + role="recall_entries", + handler=self._llm_recall_entries, + requires={ + "needs_retrieval", + "proceed_to_entries", + "ctx", + "store", + "where", + "active_query", + "file_hits", + }, + produces={"entry_hits"}, + capabilities={"llm"}, + config={"llm_profile": self.retrieve_config.llm_ranking_llm_profile}, + ), + WorkflowStep( + step_id="sufficiency_after_entries", + role="sufficiency_check", + handler=self._llm_entry_sufficiency, + requires={"needs_retrieval", "active_query", "context_queries", "entry_hits"}, + produces={"next_step_query", "proceed_to_resources"}, + capabilities={"llm"}, + config={"llm_profile": self.retrieve_config.sufficiency_check_llm_profile}, + ), + WorkflowStep( + step_id="recall_resources", + role="recall_resources", + handler=self._llm_recall_resources, + requires={ + "needs_retrieval", + "proceed_to_resources", + "active_query", + "ctx", + "store", + "where", + "entry_hits", + "file_hits", + }, + produces={"resource_hits"}, + capabilities={"llm"}, + config={"llm_profile": self.retrieve_config.llm_ranking_llm_profile}, + ), + WorkflowStep( + step_id="build_context", + role="build_context", + handler=self._llm_build_context, + requires={"needs_retrieval", "original_query", "rewritten_query"}, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + async def _llm_route_intention(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("route_intention"): + state.update({ + "needs_retrieval": True, + "rewritten_query": state["original_query"], + "active_query": state["original_query"], + "next_step_query": None, + "proceed_to_entries": False, + "proceed_to_resources": False, + }) + return state + + llm_client = self._get_step_llm_client(step_context) + needs_retrieval, rewritten_query = await self._decide_if_retrieval_needed( + state["original_query"], + state["context_queries"], + retrieved_content=None, + llm_client=llm_client, + ) + if state.get("skip_rewrite"): + rewritten_query = state["original_query"] + + state.update({ + "needs_retrieval": needs_retrieval, + "rewritten_query": rewritten_query, + "active_query": rewritten_query, + "next_step_query": None, + "proceed_to_entries": False, + "proceed_to_resources": False, + }) + return state + + async def _llm_route_file(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("needs_retrieval"): + state["file_hits"] = [] + return state + llm_client = self._get_step_llm_client(step_context) + store = state["store"] + where_filters = state.get("where") or {} + file_pool = store.recall_file_repo.list_categories(self._memory_where(where_filters)) + hits = await self._llm_rank_files( + state["active_query"], + self.retrieve_config.category.top_k, + state["ctx"], + store, + llm_client=llm_client, + categories=file_pool, + ) + state["file_hits"] = hits + state["file_pool"] = file_pool + return state + + async def _llm_file_sufficiency(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("needs_retrieval"): + state["proceed_to_entries"] = False + return state + if not state.get("retrieve_file") or not state.get("sufficiency_check"): + state["proceed_to_entries"] = True + return state + + retrieved_content = "" + hits = state.get("file_hits") or [] + if hits: + retrieved_content = self._format_llm_file_content(hits) + + llm_client = self._get_step_llm_client(step_context) + needs_more, rewritten_query = await self._decide_if_retrieval_needed( + state["active_query"], + state["context_queries"], + retrieved_content=retrieved_content or "No content retrieved yet.", + llm_client=llm_client, + ) + state["next_step_query"] = rewritten_query + state["active_query"] = rewritten_query + state["proceed_to_entries"] = needs_more + return state + + async def _llm_recall_entries(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("needs_retrieval") or not state.get("proceed_to_entries"): + state["entry_hits"] = [] + return state + + where_filters = state.get("where") or {} + file_hits = state.get("file_hits", []) + category_ids = [cat["id"] for cat in file_hits] + llm_client = self._get_step_llm_client(step_context) + store = state["store"] + + use_refs = getattr(self.retrieve_config.item, "use_category_references", False) + ref_ids: list[str] = [] + if use_refs and file_hits: + # Extract all ref_ids from category summaries + from memu.utils.references import extract_references + + for cat in file_hits: + summary = cat.get("summary") or "" + ref_ids.extend(extract_references(summary)) + if ref_ids: + # Query items by ref_ids + entries_pool = store.recall_entry_repo.list_items_by_ref_ids(ref_ids, where_filters) + else: + entries_pool = store.recall_entry_repo.list_items(where_filters) + + relations = store.recall_file_entry_repo.list_relations(where_filters) + file_pool = state.get("file_pool") or store.recall_file_repo.list_categories(self._memory_where(where_filters)) + state["entry_hits"] = await self._llm_rank_entries( + state["active_query"], + self.retrieve_config.item.top_k, + category_ids, + state.get("file_hits", []), + state["ctx"], + store, + llm_client=llm_client, + categories=file_pool, + items=entries_pool, + relations=relations, + ) + state["entry_pool"] = entries_pool + state["relation_pool"] = relations + return state + + async def _llm_entry_sufficiency(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("needs_retrieval"): + state["proceed_to_resources"] = False + return state + if not state.get("retrieve_entry") or not state.get("sufficiency_check"): + state["proceed_to_resources"] = True + return state + + retrieved_content = "" + hits = state.get("entry_hits") or [] + if hits: + retrieved_content = self._format_llm_entry_content(hits) + + llm_client = self._get_step_llm_client(step_context) + needs_more, rewritten_query = await self._decide_if_retrieval_needed( + state["active_query"], + state["context_queries"], + retrieved_content=retrieved_content or "No content retrieved yet.", + llm_client=llm_client, + ) + state["next_step_query"] = rewritten_query + state["active_query"] = rewritten_query + state["proceed_to_resources"] = needs_more + return state + + async def _llm_recall_resources(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("needs_retrieval") or not state.get("proceed_to_resources"): + state["resource_hits"] = [] + return state + + llm_client = self._get_step_llm_client(step_context) + store = state["store"] + where_filters = state.get("where") or {} + resource_pool = store.resource_repo.list_resources(where_filters) + entries_pool = state.get("entry_pool") or store.recall_entry_repo.list_items(where_filters) + state["resource_hits"] = await self._llm_rank_resources( + state["active_query"], + self.retrieve_config.resource.top_k, + state.get("file_hits", []), + state.get("entry_hits", []), + state["ctx"], + store, + llm_client=llm_client, + items=entries_pool, + resources=resource_pool, + ) + state["resource_pool"] = resource_pool + return state + + def _llm_build_context(self, state: WorkflowState, _: Any) -> WorkflowState: + response = { + "needs_retrieval": bool(state.get("needs_retrieval")), + "original_query": state["original_query"], + "rewritten_query": state.get("rewritten_query", state["original_query"]), + "next_step_query": state.get("next_step_query"), + "categories": [], + "items": [], + "resources": [], + } + if state.get("needs_retrieval"): + response["categories"] = list(state.get("file_hits") or []) + response["items"] = list(state.get("entry_hits") or []) + response["resources"] = list(state.get("resource_hits") or []) + state["response"] = response + return state + + async def _rank_files_by_summary( + self, + query_vec: list[float], + top_k: int, + ctx: Context, + store: Database, + embed_client: Any | None = None, + categories: Mapping[str, Any] | None = None, + ) -> tuple[list[tuple[str, float]], dict[str, str]]: + file_pool = categories if categories is not None else self._memory_categories(store) + entries = [(cid, cat.content) for cid, cat in file_pool.items() if cat.content] + if not entries: + return [], {} + summary_texts = [summary for _, summary in entries] + client = embed_client or self._get_embedding_client() + summary_embeddings = await client.embed(summary_texts) + corpus = [(cid, emb) for (cid, _), emb in zip(entries, summary_embeddings, strict=True)] + hits = cosine_topk(query_vec, corpus, k=top_k) + summary_lookup = dict(entries) + return hits, summary_lookup + + async def _decide_if_retrieval_needed( + self, + query: str, + context_queries: list[dict[str, Any]] | None, + retrieved_content: str | None = None, + system_prompt: str | None = None, + llm_client: Any | None = None, + ) -> tuple[bool, str]: + """ + Decide if the query requires memory retrieval (or MORE retrieval) and rewrite it with context. + + Args: + query: The current query string + context_queries: List of previous query objects with role and content + retrieved_content: Content retrieved so far (if checking for sufficiency) + system_prompt: Optional system prompt override + + Returns: + Tuple of (needs_retrieval: bool, rewritten_query: str) + - needs_retrieval: True if retrieval/more retrieval is needed + - rewritten_query: The rewritten query for the next step + """ + history_text = self._format_query_context(context_queries) + content_text = retrieved_content or "No content retrieved yet." + + prompt = self.retrieve_config.sufficiency_check_prompt or PRE_RETRIEVAL_USER_PROMPT + user_prompt = prompt.format( + query=self._escape_prompt_value(query), + conversation_history=self._escape_prompt_value(history_text), + retrieved_content=self._escape_prompt_value(content_text), + ) + + sys_prompt = system_prompt or PRE_RETRIEVAL_SYSTEM_PROMPT + client = llm_client or self._get_llm_client() + response = await client.chat(user_prompt, system_prompt=sys_prompt) + decision = self._extract_decision(response) + rewritten = self._extract_rewritten_query(response) or query + + return decision == "RETRIEVE", rewritten + + def _format_query_context(self, queries: list[dict[str, Any]] | None) -> str: + """Format query context for prompts, including role information""" + if not queries: + return "No query context." + + lines = [] + for q in queries: + if isinstance(q, str): + # Backward compatibility + lines.append(f"- {q}") + elif isinstance(q, dict): + role = q.get("role", "user") + content = q.get("content") + if isinstance(content, dict): + text = content.get("text", "") + elif isinstance(content, str): + text = content + else: + text = str(content) + lines.append(f"- [{role}]: {text}") + else: + lines.append(f"- {q!s}") + + return "\n".join(lines) + + @staticmethod + def _extract_query_text(query: dict[str, Any]) -> str: + """ + Extract text content from query message structure. + + Args: + query: Query in format {"role": "user", "content": {"text": "..."}} + + Returns: + The extracted text string + """ + if isinstance(query, str): + # Backward compatibility: if it's already a string, return it + return query + + if not isinstance(query, dict): + raise TypeError("INVALID") + + content = query.get("content") + if isinstance(content, dict): + text = content.get("text", "") + if not text: + raise ValueError("EMPTY") + return str(text) + elif isinstance(content, str): + # Also support {"role": "user", "content": "text"} format + return content + else: + raise TypeError("INVALID") + + def _extract_decision(self, raw: str) -> str: + """Extract RETRIEVE or NO_RETRIEVE decision from LLM response""" + if not raw: + return "RETRIEVE" # Default to retrieve if uncertain + + match = re.search(r"(.*?)", raw, re.IGNORECASE | re.DOTALL) + if match: + decision = match.group(1).strip().upper() + if "NO_RETRIEVE" in decision or "NO RETRIEVE" in decision: + return "NO_RETRIEVE" + if "RETRIEVE" in decision: + return "RETRIEVE" + + upper = raw.strip().upper() + if "NO_RETRIEVE" in upper or "NO RETRIEVE" in upper: + return "NO_RETRIEVE" + + return "RETRIEVE" # Default to retrieve + + def _extract_rewritten_query(self, raw: str) -> str | None: + """Extract rewritten query from LLM response""" + match = re.search(r"(.*?)", raw, re.IGNORECASE | re.DOTALL) + if match: + return match.group(1).strip() + return None + + async def _embedding_based_retrieve( + self, + query: str, + top_k: int, + context_queries: list[dict[str, Any]] | None, + ctx: Context, + store: Database, + llm_client: Any | None = None, + embed_client: Any | None = None, + where: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + """Embedding-based retrieval with query rewriting and judging at each tier""" + where_filters = self._normalize_where(where) + file_pool = store.recall_file_repo.list_categories(self._memory_where(where_filters)) + entries_pool = store.recall_entry_repo.list_items(where_filters) + resource_pool = store.resource_repo.list_resources(where_filters) + client = llm_client or self._get_llm_client() + embed = embed_client or self._get_embedding_client() + current_query = query + qvec = (await embed.embed([current_query]))[0] + response: dict[str, Any] = {"resources": [], "items": [], "categories": [], "next_step_query": None} + content_sections: list[str] = [] + + # Tier 1: Categories + cat_hits, summary_lookup = await self._rank_files_by_summary( + qvec, + top_k, + ctx, + store, + embed_client=embed, + categories=file_pool, + ) + if cat_hits: + response["categories"] = self._materialize_hits(cat_hits, file_pool) + content_sections.append(self._format_file_content(cat_hits, summary_lookup, store, categories=file_pool)) + + needs_more, current_query = await self._decide_if_retrieval_needed( + current_query, + context_queries, + retrieved_content="\n\n".join(content_sections), + llm_client=client, + ) + response["next_step_query"] = current_query + if not needs_more: + return response + # Re-embed with rewritten query + qvec = (await embed.embed([current_query]))[0] + + # Tier 2: Items + entry_hits = store.recall_entry_repo.vector_search_items(qvec, top_k, where=where_filters) + if entry_hits: + response["items"] = self._materialize_hits(entry_hits, entries_pool) + content_sections.append(self._format_entry_content(entry_hits, store, items=entries_pool)) + + needs_more, current_query = await self._decide_if_retrieval_needed( + current_query, + context_queries, + retrieved_content="\n\n".join(content_sections), + llm_client=client, + ) + response["next_step_query"] = current_query + if not needs_more: + return response + # Re-embed with rewritten query + qvec = (await embed.embed([current_query]))[0] + + # Tier 3: Resources + if resource_pool: + res_hits = store.resource_repo.vector_search_resources(qvec, top_k, where=where_filters) + if res_hits: + response["resources"] = self._materialize_hits(res_hits, resource_pool) + content_sections.append(self._format_resource_content(res_hits, store, resources=resource_pool)) + + return response + + def _materialize_hits(self, hits: Sequence[tuple[str, float]], pool: dict[str, Any]) -> list[dict[str, Any]]: + out = [] + for _id, score in hits: + obj = pool.get(_id) + if not obj: + continue + data = self._model_dump_without_embeddings(obj) + data["score"] = float(score) + out.append(data) + return out + + def _format_file_content( + self, + hits: list[tuple[str, float]], + summaries: dict[str, str], + store: Database, + categories: Mapping[str, Any] | None = None, + ) -> str: + file_pool = categories if categories is not None else self._memory_categories(store) + lines = [] + for cid, score in hits: + cat = file_pool.get(cid) + if not cat: + continue + summary = summaries.get(cid) or cat.content or "" + lines.append(f"Category: {cat.name}\nSummary: {summary}\nScore: {score:.3f}") + return "\n\n".join(lines).strip() + + def _format_entry_content( + self, hits: list[tuple[str, float]], store: Database, items: Mapping[str, Any] | None = None + ) -> str: + entry_pool = items if items is not None else store.recall_entry_repo.items + lines = [] + for iid, score in hits: + item = entry_pool.get(iid) + if not item: + continue + lines.append(f"Memory Item ({item.memory_type}): {item.summary}\nScore: {score:.3f}") + return "\n\n".join(lines).strip() + + def _format_resource_content( + self, hits: list[tuple[str, float]], store: Database, resources: Mapping[str, Any] | None = None + ) -> str: + resource_pool = resources if resources is not None else store.resource_repo.resources + lines = [] + for rid, score in hits: + res = resource_pool.get(rid) + if not res: + continue + caption = res.caption or f"Resource {res.url}" + lines.append(f"Resource: {caption}\nScore: {score:.3f}") + return "\n\n".join(lines).strip() + + def _extract_judgement(self, raw: str) -> str: + if not raw: + return "MORE" + match = re.search(r"(.*?)", raw, re.IGNORECASE | re.DOTALL) + if match: + token = match.group(1).strip().upper() + if "ENOUGH" in token: + return "ENOUGH" + if "MORE" in token: + return "MORE" + upper = raw.strip().upper() + if "ENOUGH" in upper: + return "ENOUGH" + return "MORE" + + async def _llm_based_retrieve( + self, + query: str, + top_k: int, + context_queries: list[dict[str, Any]] | None, + ctx: Context, + store: Database, + llm_client: Any | None = None, + where: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + """ + LLM-based retrieval that uses language model to search and rank results + in a hierarchical manner, with query rewriting and judging at each tier. + + Flow: + 1. Search categories with LLM, judge + rewrite query + 2. If needs more, search items from relevant categories, judge + rewrite + 3. If needs more, search resources related to context + """ + where_filters = self._normalize_where(where) + file_pool = store.recall_file_repo.list_categories(self._memory_where(where_filters)) + entries_pool = store.recall_entry_repo.list_items(where_filters) + relations = store.recall_file_entry_repo.list_relations(where_filters) + resource_pool = store.resource_repo.list_resources(where_filters) + current_query = query + client = llm_client or self._get_llm_client() + response: dict[str, Any] = {"resources": [], "items": [], "categories": [], "next_step_query": None} + content_sections: list[str] = [] + + # Tier 1: Search and rank categories + file_hits = await self._llm_rank_files( + current_query, + top_k, + ctx, + store, + llm_client=client, + categories=file_pool, + ) + if file_hits: + response["categories"] = file_hits + content_sections.append(self._format_llm_file_content(file_hits)) + + needs_more, current_query = await self._decide_if_retrieval_needed( + current_query, + context_queries, + retrieved_content="\n\n".join(content_sections), + llm_client=client, + ) + response["next_step_query"] = current_query + if not needs_more: + return response + + # Tier 2: Search memory items from relevant categories + relevant_file_ids = [cat["id"] for cat in file_hits] + entry_hits = await self._llm_rank_entries( + current_query, + top_k, + relevant_file_ids, + file_hits, + ctx, + store, + llm_client=client, + categories=file_pool, + items=entries_pool, + relations=relations, + ) + if entry_hits: + response["items"] = entry_hits + content_sections.append(self._format_llm_entry_content(entry_hits)) + + needs_more, current_query = await self._decide_if_retrieval_needed( + current_query, + context_queries, + retrieved_content="\n\n".join(content_sections), + llm_client=client, + ) + response["next_step_query"] = current_query + if not needs_more: + return response + + # Tier 3: Search resources related to the context + resource_hits = await self._llm_rank_resources( + current_query, + top_k, + file_hits, + entry_hits, + ctx, + store, + llm_client=client, + items=entries_pool, + resources=resource_pool, + ) + if resource_hits: + response["resources"] = resource_hits + content_sections.append(self._format_llm_resource_content(resource_hits)) + + return response + + def _format_files_for_llm( + self, + store: Database, + category_ids: list[str] | None = None, + categories: Mapping[str, Any] | None = None, + ) -> str: + """Format categories for LLM consumption""" + categories_to_format = categories if categories is not None else self._memory_categories(store) + if category_ids: + categories_to_format = {cid: cat for cid, cat in categories_to_format.items() if cid in category_ids} + + if not categories_to_format: + return "No categories available." + + lines = [] + for cid, cat in categories_to_format.items(): + lines.append(f"ID: {cid}") + lines.append(f"Name: {cat.name}") + if cat.description: + lines.append(f"Description: {cat.description}") + if cat.content: + lines.append(f"Summary: {cat.content}") + lines.append("---") + + return "\n".join(lines) + + def _format_entries_for_llm( + self, + store: Database, + category_ids: list[str] | None = None, + items: Mapping[str, Any] | None = None, + relations: Sequence[Any] | None = None, + ) -> str: + """Format memory items for LLM consumption, optionally filtered by category""" + entry_pool = items if items is not None else store.recall_entry_repo.items + relation_pool = relations if relations is not None else store.recall_file_entry_repo.relations + items_to_format = [] + seen_item_ids = set() + + if category_ids: + # Get items that belong to the specified categories + for rel in relation_pool: + if rel.category_id in category_ids: + item = entry_pool.get(rel.item_id) + if item and item.id not in seen_item_ids: + items_to_format.append(item) + seen_item_ids.add(item.id) + else: + items_to_format = list(entry_pool.values()) + + if not items_to_format: + return "No memory items available." + + lines = [] + for item in items_to_format: + lines.append(f"ID: {item.id}") + lines.append(f"Type: {item.memory_type}") + lines.append(f"Summary: {item.summary}") + lines.append("---") + + return "\n".join(lines) + + def _format_resources_for_llm( + self, + store: Database, + item_ids: list[str] | None = None, + items: Mapping[str, Any] | None = None, + resources: Mapping[str, Any] | None = None, + ) -> str: + """Format resources for LLM consumption, optionally filtered by related items""" + resource_pool = resources if resources is not None else store.resource_repo.resources + entry_pool = items if items is not None else store.recall_entry_repo.items + resources_to_format = [] + + if item_ids: + # Get resources that are related to the specified items + resource_ids = {entry_pool[iid].resource_id for iid in item_ids if iid in entry_pool and iid is not None} + resources_to_format = [ + resource_pool[rid] for rid in resource_ids if rid in resource_pool and rid is not None + ] + else: + resources_to_format = list(resource_pool.values()) + + if not resources_to_format: + return "No resources available." + + lines = [] + for res in resources_to_format: + lines.append(f"ID: {res.id}") + lines.append(f"URL: {res.url}") + lines.append(f"Modality: {res.modality}") + if res.caption: + lines.append(f"Caption: {res.caption}") + lines.append("---") + + return "\n".join(lines) + + async def _llm_rank_files( + self, + query: str, + top_k: int, + ctx: Context, + store: Database, + llm_client: Any | None = None, + categories: Mapping[str, Any] | None = None, + ) -> list[dict[str, Any]]: + """Use LLM to rank categories based on query relevance""" + file_pool = categories if categories is not None else self._memory_categories(store) + if not file_pool: + return [] + + categories_data = self._format_files_for_llm(store, categories=file_pool) + prompt = LLM_CATEGORY_RANKER_PROMPT.format( + query=self._escape_prompt_value(query), + top_k=top_k, + categories_data=self._escape_prompt_value(categories_data), + ) + + client = llm_client or self._get_llm_client() + llm_response = await client.chat(prompt) + return self._parse_llm_file_response(llm_response, store, categories=file_pool) + + async def _llm_rank_entries( + self, + query: str, + top_k: int, + category_ids: list[str], + file_hits: list[dict[str, Any]], + ctx: Context, + store: Database, + llm_client: Any | None = None, + categories: Mapping[str, Any] | None = None, + items: Mapping[str, Any] | None = None, + relations: Sequence[Any] | None = None, + ) -> list[dict[str, Any]]: + """Use LLM to rank memory items from relevant categories""" + if not category_ids: + logger.debug("[LLM Rank Items] No category_ids provided") + return [] + + entry_pool = items if items is not None else store.recall_entry_repo.items + items_data = self._format_entries_for_llm(store, category_ids, items=entry_pool, relations=relations) + if items_data == "No memory items available.": + return [] + + # Format relevant categories for context + relevant_categories_info = "\n".join([ + f"- {cat['name']}: {cat.get('summary', cat.get('description', ''))}" for cat in file_hits + ]) + + prompt = LLM_ITEM_RANKER_PROMPT.format( + query=self._escape_prompt_value(query), + top_k=top_k, + relevant_categories=self._escape_prompt_value(relevant_categories_info), + items_data=self._escape_prompt_value(items_data), + ) + + client = llm_client or self._get_llm_client() + llm_response = await client.chat(prompt) + return self._parse_llm_entry_response(llm_response, store, items=entry_pool) + + async def _llm_rank_resources( + self, + query: str, + top_k: int, + file_hits: list[dict[str, Any]], + entry_hits: list[dict[str, Any]], + ctx: Context, + store: Database, + llm_client: Any | None = None, + items: Mapping[str, Any] | None = None, + resources: Mapping[str, Any] | None = None, + ) -> list[dict[str, Any]]: + """Use LLM to rank resources related to the context""" + # Get item IDs to filter resources + item_ids = [item["id"] for item in entry_hits] + if not item_ids: + return [] + + entry_pool = items if items is not None else store.recall_entry_repo.items + resource_pool = resources if resources is not None else store.resource_repo.resources + resources_data = self._format_resources_for_llm(store, item_ids, items=entry_pool, resources=resource_pool) + if resources_data == "No resources available.": + return [] + + # Build context info + context_parts = [] + if file_hits: + context_parts.append("Relevant Categories:") + context_parts.extend([f"- {cat['name']}" for cat in file_hits]) + if entry_hits: + context_parts.append("\nRelevant Memory Items:") + context_parts.extend([f"- {item.get('summary', '')[:100]}..." for item in entry_hits[:3]]) + + context_info = "\n".join(context_parts) + prompt = LLM_RESOURCE_RANKER_PROMPT.format( + query=self._escape_prompt_value(query), + top_k=top_k, + context_info=self._escape_prompt_value(context_info), + resources_data=self._escape_prompt_value(resources_data), + ) + + client = llm_client or self._get_llm_client() + llm_response = await client.chat(prompt) + return self._parse_llm_resource_response(llm_response, store, resources=resource_pool) + + def _parse_llm_file_response( + self, raw_response: str, store: Database, categories: Mapping[str, Any] | None = None + ) -> list[dict[str, Any]]: + """Parse LLM category ranking response""" + file_pool = categories if categories is not None else self._memory_categories(store) + results = [] + try: + json_blob = self._extract_json_blob(raw_response) + parsed = json.loads(json_blob) + + if "categories" in parsed and isinstance(parsed["categories"], list): + category_ids = parsed["categories"] + # Return categories in the order provided by LLM (already sorted by relevance) + for cat_id in category_ids: + if isinstance(cat_id, str): + cat = file_pool.get(cat_id) + if cat: + cat_data = self._model_dump_without_embeddings(cat) + results.append(cat_data) + except Exception as e: + logger.warning(f"Failed to parse LLM category ranking response: {e}") + + return results + + def _parse_llm_entry_response( + self, raw_response: str, store: Database, items: Mapping[str, Any] | None = None + ) -> list[dict[str, Any]]: + """Parse LLM item ranking response""" + entry_pool = items if items is not None else store.recall_entry_repo.items + results = [] + try: + json_blob = self._extract_json_blob(raw_response) + parsed = json.loads(json_blob) + + if "items" in parsed and isinstance(parsed["items"], list): + item_ids = parsed["items"] + # Return items in the order provided by LLM (already sorted by relevance) + for item_id in item_ids: + if isinstance(item_id, str): + mem_item = entry_pool.get(item_id) + if mem_item: + item_data = self._model_dump_without_embeddings(mem_item) + results.append(item_data) + except Exception as e: + logger.warning(f"Failed to parse LLM item ranking response: {e}") + + return results + + def _parse_llm_resource_response( + self, raw_response: str, store: Database, resources: Mapping[str, Any] | None = None + ) -> list[dict[str, Any]]: + """Parse LLM resource ranking response""" + resource_pool = resources if resources is not None else store.resource_repo.resources + results = [] + try: + json_blob = self._extract_json_blob(raw_response) + parsed = json.loads(json_blob) + + if "resources" in parsed and isinstance(parsed["resources"], list): + resource_ids = parsed["resources"] + # Return resources in the order provided by LLM (already sorted by relevance) + for res_id in resource_ids: + if isinstance(res_id, str): + res = resource_pool.get(res_id) + if res: + res_data = self._model_dump_without_embeddings(res) + results.append(res_data) + except Exception as e: + logger.warning(f"Failed to parse LLM resource ranking response: {e}") + + return results + + def _format_llm_file_content(self, hits: list[dict[str, Any]]) -> str: + """Format LLM-ranked category content for judger""" + lines = [] + for cat in hits: + summary = cat.get("summary", "") or cat.get("description", "") + lines.append(f"Category: {cat['name']}\nSummary: {summary}") + return "\n\n".join(lines).strip() + + def _format_llm_entry_content(self, hits: list[dict[str, Any]]) -> str: + """Format LLM-ranked item content for judger""" + lines = [] + for item in hits: + lines.append(f"Memory Item ({item['memory_type']}): {item['summary']}") + return "\n\n".join(lines).strip() + + def _format_llm_resource_content(self, hits: list[dict[str, Any]]) -> str: + """Format LLM-ranked resource content for judger""" + lines = [] + for res in hits: + caption = res.get("caption", "") or f"Resource {res['url']}" + lines.append(f"Resource: {caption}") + return "\n\n".join(lines).strip() diff --git a/src/memu/app/retrieve_workspace.py b/src/memu/app/retrieve_workspace.py new file mode 100644 index 0000000..b07be8f --- /dev/null +++ b/src/memu/app/retrieve_workspace.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, cast + +from pydantic import BaseModel + +from memu.vector import cosine_topk +from memu.workflow.step import WorkflowState, WorkflowStep + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from memu.app.settings import RetrieveWorkspaceConfig + from memu.database.interfaces import Database + + +class RetrieveWorkspaceMixin: + if TYPE_CHECKING: + retrieve_workspace_config: RetrieveWorkspaceConfig + _run_workflow: Callable[..., Awaitable[WorkflowState]] + _get_database: Callable[[], Database] + _get_step_embedding_client: Callable[[Mapping[str, Any] | None], Any] + _model_dump_without_embeddings: Callable[[BaseModel], dict[str, Any]] + user_model: type[BaseModel] + + async def retrieve_workspace( + self, + query: str, + where: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Single-shot, LLM-free retrieval over the segment/file/resource layers. + + Mirrors the relation between :meth:`memorize` and ``memorize_workspace``: + a simpler entry point built on the same store and workflow machinery. The + query is embedded once and used to rank two layers by vector similarity — + no intention routing, sufficiency checks, or summarization: + + * ``segments``: :class:`RecallFileSegment` slices ranked by embedding, + ``file.top_k`` of them. + * ``files``: the :class:`RecallFile`\\ s pointed to by those segments — not + a ranked search, just a roll-up. Each file's score is the max score of + the segments that point to it. + * ``resources``: workspace-track resources ranked by embedding, + ``resource.top_k`` of them. + + The entry layer is disabled here (its config is retained but ignored). + Returns ``segments``, ``files``, and ``resources``. + """ + if not query or not query.strip(): + raise ValueError("empty_query") + store = self._get_database() + where_filters = self._normalize_where(where) + config = self.retrieve_workspace_config + + state: WorkflowState = { + "query": query, + "store": store, + "where": where_filters, + "retrieve_file": config.file.enabled, + "retrieve_resource": config.resource.enabled, + } + + result = await self._run_workflow("retrieve_workspace", state) + response = cast(dict[str, Any] | None, result.get("response")) + if response is None: + msg = "Retrieve workspace workflow failed to produce a response" + raise RuntimeError(msg) + return response + + def _normalize_where(self, where: Mapping[str, Any] | None) -> dict[str, Any]: + """Validate and clean the `where` scope filters against the configured user model.""" + if not where: + return {} + + valid_fields = set(getattr(self.user_model, "model_fields", {}).keys()) + cleaned: dict[str, Any] = {} + + for raw_key, value in where.items(): + if value is None: + continue + field = raw_key.split("__", 1)[0] + if field not in valid_fields: + msg = f"Unknown filter field '{field}' for current user scope" + raise ValueError(msg) + cleaned[raw_key] = value + + return cleaned + + def _build_retrieve_workspace_workflow(self) -> list[WorkflowStep]: + """The simple embedding-only workspace retrieval pipeline. + + A segment recall step ranks :class:`RecallFileSegment` slices by embedding; + a file roll-up step gathers the files those segments point to; a resource + recall step ranks workspace-track resources by embedding. A terminal step + assembles the response. None of the routing/sufficiency machinery of + ``retrieve_rag`` applies. The query vector is embedded by the first recall + step and reused downstream. + """ + steps = [ + WorkflowStep( + step_id="recall_segments", + role="recall_segments", + handler=self._ws_recall_segments, + requires={"retrieve_file", "query", "store", "where"}, + produces={"segment_hits", "segment_pool", "query_vector"}, + capabilities={"vector"}, + config={"embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="collect_files", + role="collect_files", + handler=self._ws_collect_files, + requires={"retrieve_file", "segment_hits", "segment_pool", "store", "where"}, + produces={"file_hits", "file_pool", "file_resource_urls"}, + capabilities=set(), + ), + WorkflowStep( + step_id="recall_resources", + role="recall_resources", + handler=self._ws_recall_resources, + requires={"retrieve_resource", "query", "store", "where", "query_vector"}, + produces={"resource_hits", "resource_pool", "query_vector"}, + capabilities={"vector"}, + config={"embed_llm_profile": "embedding"}, + ), + WorkflowStep( + step_id="build_response", + role="build_context", + handler=self._ws_build_response, + requires={ + "segment_hits", + "segment_pool", + "file_hits", + "file_pool", + "file_resource_urls", + "resource_hits", + "resource_pool", + }, + produces={"response"}, + capabilities=set(), + ), + ] + return steps + + @staticmethod + def _list_retrieve_workspace_initial_keys() -> set[str]: + return {"query", "store", "where", "retrieve_file", "retrieve_resource"} + + async def _ws_query_vector(self, state: WorkflowState, step_context: Any) -> list[float]: + """Embed the query once and cache it on the state for reuse across steps.""" + cached = state.get("query_vector") + if cached is not None: + return cast(list[float], cached) + embed_client = self._get_step_embedding_client(step_context) + qvec = (await embed_client.embed([state["query"]]))[0] + state["query_vector"] = qvec + return cast(list[float], qvec) + + async def _ws_recall_segments(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("retrieve_file"): + state["segment_hits"] = [] + state["segment_pool"] = {} + state.setdefault("query_vector", None) + return state + + store = state["store"] + # The segment repo has no vector search, so rank the stored segment + # embeddings directly, mirroring how files used to be ranked. Optionally + # scope to the requested tracks via the denormalized segment ``track``. + segment_where = dict(state.get("where") or {}) + tracks = self.retrieve_workspace_config.file.tracks + if tracks: + segment_where["track__in"] = list(tracks) + segment_pool = {seg.id: seg for seg in store.recall_file_segment_repo.list_segments(segment_where)} + qvec = await self._ws_query_vector(state, step_context) + state["segment_hits"] = cosine_topk( + qvec, + [(sid, seg.embedding) for sid, seg in segment_pool.items()], + k=self.retrieve_workspace_config.file.top_k, + ) + state["segment_pool"] = segment_pool + return state + + async def _ws_collect_files(self, state: WorkflowState, _: Any) -> WorkflowState: + """Roll the ranked segments up to their files (no ranked file search). + + Every file pointed to by a top segment is returned; a file's score is the + max score across the segments that point to it. + """ + segment_hits = state.get("segment_hits") or [] + segment_pool = state.get("segment_pool") or {} + store = state["store"] + where_filters = state.get("where") or {} + file_pool = store.recall_file_repo.list_categories(where_filters) + + file_scores: dict[str, float] = {} + for seg_id, score in segment_hits: + seg = segment_pool.get(seg_id) + if seg is None: + continue + fid = seg.recall_file_id + if fid not in file_pool: + continue + score = float(score) + if fid not in file_scores or score > file_scores[fid]: + file_scores[fid] = score + + # Preserve descending-score order so the response reads best-first. + state["file_hits"] = sorted(file_scores.items(), key=lambda kv: kv[1], reverse=True) + state["file_pool"] = file_pool + state["file_resource_urls"] = self._ws_collect_file_resource_urls(store, where_filters, file_pool) + return state + + @staticmethod + def _ws_collect_file_resource_urls( + store: Database, where_filters: dict[str, Any], file_pool: dict[str, Any] + ) -> dict[str, list[str]]: + """Map each file id to the URLs of the resources linked to it. + + Resolves the ``RecallFileResource`` link table (file -> resource) and the + resource records (resource -> url) within the current scope, surfacing url + strings only — the raw resource/link ids are not exposed to callers. + """ + resources = store.resource_repo.list_resources(where_filters) + file_resource_urls: dict[str, list[str]] = {} + for rel in store.recall_file_resource_repo.list_relations(where_filters): + if rel.file_id not in file_pool: + continue + resource = resources.get(rel.resource_id) + if resource is None: + continue + file_resource_urls.setdefault(rel.file_id, []).append(resource.url) + return file_resource_urls + + async def _ws_recall_resources(self, state: WorkflowState, step_context: Any) -> WorkflowState: + if not state.get("retrieve_resource"): + state["resource_hits"] = [] + state["resource_pool"] = {} + return state + + store = state["store"] + # Workspace retrieval only surfaces resources ingested by + # ``memorize_workspace`` (track="workspace"); other tracks are excluded. + resource_where = {**(state.get("where") or {}), "track": "workspace"} + resource_pool = store.resource_repo.list_resources(resource_where) + qvec = await self._ws_query_vector(state, step_context) + state["resource_hits"] = store.resource_repo.vector_search_resources( + qvec, self.retrieve_workspace_config.resource.top_k, where=resource_where + ) + state["resource_pool"] = resource_pool + return state + + def _ws_build_response(self, state: WorkflowState, _: Any) -> WorkflowState: + files = self._materialize_hits(state.get("file_hits", []), state.get("file_pool", {})) + file_resource_urls = state.get("file_resource_urls", {}) + for file in files: + file["resource_urls"] = file_resource_urls.get(file["id"], []) + state["response"] = { + "segments": self._materialize_hits(state.get("segment_hits", []), state.get("segment_pool", {})), + "files": files, + "resources": self._materialize_hits(state.get("resource_hits", []), state.get("resource_pool", {})), + } + return state + + def _materialize_hits(self, hits: Sequence[tuple[str, float]], pool: dict[str, Any]) -> list[dict[str, Any]]: + out = [] + for _id, score in hits: + obj = pool.get(_id) + if not obj: + continue + data = self._model_dump_without_embeddings(obj) + data["score"] = float(score) + out.append(data) + return out diff --git a/src/memu/app/service.py b/src/memu/app/service.py new file mode 100644 index 0000000..6e1a017 --- /dev/null +++ b/src/memu/app/service.py @@ -0,0 +1,492 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal, TypeVar + +from pydantic import BaseModel + +from memu.app.client_pool import ClientPool +from memu.app.crud import CRUDMixin +from memu.app.memorize import MemorizeMixin +from memu.app.memorize_workspace import MemorizeWorkspaceMixin +from memu.app.memory_files import MemoryFilesBuilder +from memu.app.retrieve import RetrieveMixin +from memu.app.retrieve_workspace import RetrieveWorkspaceMixin +from memu.app.settings import ( + BlobConfig, + CategoryConfig, + DatabaseConfig, + EmbeddingConfig, + EmbeddingProfilesConfig, + LLMConfig, + LLMProfilesConfig, + MemorizeConfig, + MemoryFilesConfig, + RetrieveConfig, + RetrieveWorkspaceConfig, + UserConfig, + VLMConfig, + embedding_config_from_llm, + vlm_config_from_llm, +) +from memu.blob.local_fs import LocalFS +from memu.database.factory import build_database +from memu.database.interfaces import Database +from memu.embedding.gateway import build_embedding_client +from memu.llm.gateway import build_llm_client +from memu.llm.wrapper import ( + LLMCallMetadata, + LLMClientWrapper, + LLMInterceptorHandle, + LLMInterceptorRegistry, +) +from memu.vlm.gateway import build_vlm_client +from memu.workflow.interceptor import WorkflowInterceptorHandle, WorkflowInterceptorRegistry +from memu.workflow.pipeline import PipelineManager +from memu.workflow.runner import WorkflowRunner, resolve_workflow_runner +from memu.workflow.step import WorkflowState, WorkflowStep + +TConfigModel = TypeVar("TConfigModel", bound=BaseModel) + + +@dataclass +class Context: + categories_ready: bool = False + category_ids: list[str] = field(default_factory=list) + category_name_to_id: dict[str, str] = field(default_factory=dict) + category_init_task: asyncio.Task | None = None + + +class MemoryService(MemorizeMixin, MemorizeWorkspaceMixin, RetrieveMixin, RetrieveWorkspaceMixin, CRUDMixin): + def __init__( + self, + *, + llm_profiles: LLMProfilesConfig | dict[str, Any] | None = None, + blob_config: BlobConfig | dict[str, Any] | None = None, + database_config: DatabaseConfig | dict[str, Any] | None = None, + memorize_config: MemorizeConfig | dict[str, Any] | None = None, + retrieve_config: RetrieveConfig | dict[str, Any] | None = None, + retrieve_workspace_config: RetrieveWorkspaceConfig | dict[str, Any] | None = None, + workflow_runner: WorkflowRunner | str | None = None, + user_config: UserConfig | dict[str, Any] | None = None, + memory_files_config: MemoryFilesConfig | dict[str, Any] | None = None, + embedding_profiles: EmbeddingProfilesConfig | dict[str, Any] | None = None, + ): + self.llm_profiles = self._validate_config(llm_profiles, LLMProfilesConfig) + self.user_config = self._validate_config(user_config, UserConfig) + self.user_model = self.user_config.model + self.llm_config = self._validate_config(self.llm_profiles.default, LLMConfig) + self.blob_config = self._validate_config(blob_config, BlobConfig) + self.database_config = self._validate_config(database_config, DatabaseConfig) + self.memorize_config = self._validate_config(memorize_config, MemorizeConfig) + self.retrieve_config = self._validate_config(retrieve_config, RetrieveConfig) + self.retrieve_workspace_config = self._validate_config(retrieve_workspace_config, RetrieveWorkspaceConfig) + self.memory_files_config = self._validate_config(memory_files_config, MemoryFilesConfig) + + self.fs = LocalFS(self.blob_config.resources_dir) + self.category_configs: list[CategoryConfig] = list(self.memorize_config.recall_files or []) + self.category_config_map: dict[str, CategoryConfig] = {cfg.name: cfg for cfg in self.category_configs} + self._category_prompt_str = self._format_categories_for_prompt(self.category_configs) + + self._context = Context(categories_ready=not bool(self.category_configs)) + + self.database: Database = build_database( + config=self.database_config, + user_model=self.user_model, + ) + # We need the concrete user scope (user_id: xxx) to initialize the categories + # self._start_category_initialization(self._context, self.database) + + # VLM (vision-language) profiles are derived from the LLM profiles so + # image/video vision reuses the same provider/credentials with a stronger + # multimodal model (see ``vlm_config_from_llm``). + self.vlm_profiles: dict[str, VLMConfig] = { + name: vlm_config_from_llm(cfg) for name, cfg in self.llm_profiles.profiles.items() + } + + # Embedding profiles: use explicit profiles when supplied (e.g. to point + # vectorization at a dedicated provider such as jina/voyage), otherwise + # derive them from the LLM profiles so existing configs keep working. + if embedding_profiles is None: + self.embedding_profiles: dict[str, EmbeddingConfig] = { + name: embedding_config_from_llm(cfg) for name, cfg in self.llm_profiles.profiles.items() + } + else: + self.embedding_profiles = self._validate_config(embedding_profiles, EmbeddingProfilesConfig).profiles + + # Lazy client pools (one per capability), keyed by profile name. Each + # builds and caches a concrete client on first use, sharing the same + # get/cache/build bookkeeping instead of three hand-rolled caches. + self._llm_pool: ClientPool[LLMConfig, Any] = ClientPool( + profiles=self.llm_profiles.profiles, builder=build_llm_client, label="llm" + ) + self._vlm_pool: ClientPool[VLMConfig, Any] = ClientPool( + profiles=self.vlm_profiles, builder=build_vlm_client, label="vlm" + ) + self._embedding_pool: ClientPool[EmbeddingConfig, Any] = ClientPool( + profiles=self.embedding_profiles, builder=build_embedding_client, label="embedding" + ) + self._llm_interceptors = LLMInterceptorRegistry() + self._workflow_interceptors = WorkflowInterceptorRegistry() + + self._workflow_runner = resolve_workflow_runner(workflow_runner) + + # Optional markdown "memory file system" artifact layer (additive, read-only). + # The build/synthesis policy lives in MemoryFilesBuilder; the exporter and + # synthesizer are re-exposed as attributes for back-compat with callers/tests. + self._memory_files = MemoryFilesBuilder(config=self.memory_files_config) + self._memory_file_exporter = self._memory_files.exporter + self._memory_synthesizer = self._memory_files.synthesizer + + self._pipelines = PipelineManager( + available_capabilities={"llm", "vector", "db", "io", "vision"}, + llm_profiles=set(self.llm_profiles.profiles.keys()), + ) + self._register_pipelines() + + @staticmethod + def _llm_call_metadata(profile: str, step_context: Mapping[str, Any] | None) -> LLMCallMetadata: + if not isinstance(step_context, Mapping): + return LLMCallMetadata(profile) + operation = None + for key in ("operation", "workflow_name"): + value = step_context.get(key) + if isinstance(value, str) and value.strip(): + operation = value.strip() + break + step_id = step_context.get("step_id") if isinstance(step_context.get("step_id"), str) else None + trace_id = step_context.get("trace_id") if isinstance(step_context.get("trace_id"), str) else None + tags = step_context.get("tags") if isinstance(step_context.get("tags"), Mapping) else None + return LLMCallMetadata(profile=profile, operation=operation, step_id=step_id, trace_id=trace_id, tags=tags) + + def _wrap_llm_client( + self, + client: Any, + *, + profile: str | None = None, + step_context: Mapping[str, Any] | None = None, + ) -> Any: + cfg = self._llm_pool.config(profile) + provider = cfg.provider if cfg is not None else None + metadata = self._llm_call_metadata(profile or "default", step_context) + return LLMClientWrapper( + client, + registry=self._llm_interceptors, + metadata=metadata, + provider=provider, + chat_model=getattr(client, "chat_model", None), + ) + + def _get_llm_client(self, profile: str | None = None, step_context: Mapping[str, Any] | None = None) -> Any: + base_client = self._llm_pool.get(profile) + return self._wrap_llm_client(base_client, profile=profile, step_context=step_context) + + def _get_vlm_client(self, profile: str | None = None, step_context: Mapping[str, Any] | None = None) -> Any: + base_client = self._vlm_pool.get(profile) + cfg = self._vlm_pool.config(profile) + provider = cfg.provider if cfg is not None else None + metadata = self._llm_call_metadata(profile or "default", step_context) + return LLMClientWrapper( + base_client, + registry=self._llm_interceptors, + metadata=metadata, + provider=provider, + chat_model=getattr(base_client, "vlm_model", None), + ) + + def _get_embedding_client(self, profile: str | None = None, step_context: Mapping[str, Any] | None = None) -> Any: + base_client = self._embedding_pool.get(profile) + cfg = self._embedding_pool.config(profile) + provider = cfg.provider if cfg is not None else None + metadata = self._llm_call_metadata(profile or "default", step_context) + return LLMClientWrapper( + base_client, + registry=self._llm_interceptors, + metadata=metadata, + provider=provider, + embed_model=getattr(base_client, "embed_model", None), + ) + + @property + def llm_client(self) -> Any: + """Default LLM client (lazy).""" + return self._get_llm_client() + + @property + def workflow_runner(self) -> WorkflowRunner: + """Current workflow runner backend.""" + return self._workflow_runner + + @staticmethod + def _llm_profile_from_context( + step_context: Mapping[str, Any] | None, task: Literal["chat", "embedding"] = "chat" + ) -> str | None: + if not isinstance(step_context, Mapping): + return None + step_cfg = step_context.get("step_config") + if not isinstance(step_cfg, Mapping): + return None + if task == "chat": + profile = step_cfg.get("chat_llm_profile", step_cfg.get("llm_profile")) + elif task == "embedding": + profile = step_cfg.get("embed_llm_profile", step_cfg.get("llm_profile")) + else: + raise ValueError(task) + if isinstance(profile, str) and profile.strip(): + return profile.strip() + return None + + def _get_step_llm_client(self, step_context: Mapping[str, Any] | None) -> Any: + profile = self._llm_profile_from_context(step_context, task="chat") or "default" + return self._get_llm_client(profile, step_context=step_context) + + def _get_step_embedding_client(self, step_context: Mapping[str, Any] | None) -> Any: + profile = self._llm_profile_from_context(step_context, task="embedding") or "embedding" + return self._get_embedding_client(profile, step_context=step_context) + + def intercept_before_llm_call( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + priority: int = 0, + where: Mapping[str, Any] | Callable[..., Any] | None = None, + ) -> LLMInterceptorHandle: + return self._llm_interceptors.register_before(fn, name=name, priority=priority, where=where) + + def intercept_after_llm_call( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + priority: int = 0, + where: Mapping[str, Any] | Callable[..., Any] | None = None, + ) -> LLMInterceptorHandle: + return self._llm_interceptors.register_after(fn, name=name, priority=priority, where=where) + + def intercept_on_error_llm_call( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + priority: int = 0, + where: Mapping[str, Any] | Callable[..., Any] | None = None, + ) -> LLMInterceptorHandle: + return self._llm_interceptors.register_on_error(fn, name=name, priority=priority, where=where) + + def intercept_before_workflow_step( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + ) -> WorkflowInterceptorHandle: + """ + Register an interceptor to be called before each workflow step. + + The interceptor receives (step_context: WorkflowStepContext, state: WorkflowState). + """ + return self._workflow_interceptors.register_before(fn, name=name) + + def intercept_after_workflow_step( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + ) -> WorkflowInterceptorHandle: + """ + Register an interceptor to be called after each workflow step. + + The interceptor receives (step_context: WorkflowStepContext, state: WorkflowState). + """ + return self._workflow_interceptors.register_after(fn, name=name) + + def intercept_on_error_workflow_step( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + ) -> WorkflowInterceptorHandle: + """ + Register an interceptor to be called when a workflow step raises an exception. + + The interceptor receives (step_context: WorkflowStepContext, state: WorkflowState, error: Exception). + """ + return self._workflow_interceptors.register_on_error(fn, name=name) + + def _get_context(self) -> Context: + return self._context + + def _get_database(self) -> Database: + return self.database + + def _provider_summary(self) -> dict[str, Any]: + vector_provider = None + if self.database_config.vector_index: + vector_provider = self.database_config.vector_index.provider + return { + "llm_profiles": list(self.llm_profiles.profiles.keys()), + "storage": { + "metadata_store": self.database_config.metadata_store.provider, + "vector_index": vector_provider, + }, + } + + def _register_pipelines(self) -> None: + memo_workflow = self._build_memorize_workflow() + memo_initial_keys = self._list_memorize_initial_keys() + self._pipelines.register("memorize", memo_workflow, initial_state_keys=memo_initial_keys) + # Workspace memorize: memory steps + per-file skill generation (ADR 0006). + workspace_workflow = self._build_memorize_workspace_workflow() + self._pipelines.register("memorize_workspace", workspace_workflow, initial_state_keys=memo_initial_keys) + rag_workflow = self._build_rag_retrieve_workflow() + retrieve_initial_keys = self._list_retrieve_initial_keys() + self._pipelines.register("retrieve_rag", rag_workflow, initial_state_keys=retrieve_initial_keys) + llm_workflow = self._build_llm_retrieve_workflow() + self._pipelines.register("retrieve_llm", llm_workflow, initial_state_keys=retrieve_initial_keys) + # Simple embedding-only workspace retrieval: segment recall + file roll-up + resource recall. + workspace_retrieve_workflow = self._build_retrieve_workspace_workflow() + self._pipelines.register( + "retrieve_workspace", + workspace_retrieve_workflow, + initial_state_keys=self._list_retrieve_workspace_initial_keys(), + ) + patch_create_workflow = self._build_create_recall_entry_workflow() + patch_create_initial_keys = CRUDMixin._list_create_recall_entry_initial_keys() + self._pipelines.register("patch_create", patch_create_workflow, initial_state_keys=patch_create_initial_keys) + patch_update_workflow = self._build_update_recall_entry_workflow() + patch_update_initial_keys = CRUDMixin._list_update_recall_entry_initial_keys() + self._pipelines.register("patch_update", patch_update_workflow, initial_state_keys=patch_update_initial_keys) + patch_delete_workflow = self._build_delete_recall_entry_workflow() + patch_delete_initial_keys = CRUDMixin._list_delete_recall_entry_initial_keys() + self._pipelines.register("patch_delete", patch_delete_workflow, initial_state_keys=patch_delete_initial_keys) + crud_list_items_workflow = self._build_list_recall_entries_workflow() + crud_list_memories_initial_keys = CRUDMixin._list_list_memories_initial_keys() + self._pipelines.register( + "crud_list_recall_entries", crud_list_items_workflow, initial_state_keys=crud_list_memories_initial_keys + ) + crud_list_categories_workflow = self._build_list_recall_files_workflow() + self._pipelines.register( + "crud_list_recall_files", + crud_list_categories_workflow, + initial_state_keys=crud_list_memories_initial_keys, + ) + crud_clear_memory_workflow = self._build_clear_memory_workflow() + crud_clear_memory_initial_keys = CRUDMixin._list_clear_memories_initial_keys() + self._pipelines.register( + "crud_clear_memory", crud_clear_memory_workflow, initial_state_keys=crud_clear_memory_initial_keys + ) + + async def _run_workflow(self, workflow_name: str, initial_state: WorkflowState) -> WorkflowState: + """Execute a workflow through the configured runner backend.""" + steps = self._pipelines.build(workflow_name) + runner_context = {"workflow_name": workflow_name} + return await self._workflow_runner.run( + workflow_name, + steps, + initial_state, + runner_context, + interceptor_registry=self._workflow_interceptors, + ) + + async def export_memory_files(self, *, user: dict[str, Any] | None = None) -> dict[str, Any]: + """Render the (optionally scoped) memory store into browsable markdown files. + + Read-only against the database; only artifacts whose rendered content + changed since the last export are rewritten (diff detection via a sidecar + manifest). Returns a summary of written/unchanged/removed relative paths. + + Requires ``memory_files_config.enabled=True``. + """ + if not self.memory_files_config.enabled: + msg = "Memory files are disabled; set memory_files_config.enabled=True to use export_memory_files()." + raise RuntimeError(msg) + where = self.user_model(**user).model_dump() if user is not None else None + # No changed set => full (re)initialization of the tree. + return await self._build_memory_files(where, changed=None) + + async def _build_memory_files( + self, + where: dict[str, Any] | None, + *, + changed: list[Any] | None, + ) -> dict[str, Any]: + """Initialize or incrementally update the memory file tree. + + Thin delegate to :class:`MemoryFilesBuilder`; ``changed`` is the list of + just-memorized ``Resource`` objects driving an incremental update (``None`` + forces a full (re)initialization from the scoped store). + """ + return await self._memory_files.build( + self.database, + where, + changed=changed, + make_client=self._get_llm_client, + ) + + @staticmethod + def _extract_json_blob(raw: str) -> str: + start = raw.find("{") + end = raw.rfind("}") + if start == -1 or end == -1 or end <= start: + msg = "No JSON object found" + raise ValueError(msg) + return raw[start : end + 1] + + @staticmethod + def _escape_prompt_value(value: str) -> str: + return value.replace("{", "{{").replace("}", "}}") + + def _model_dump_without_embeddings(self, obj: BaseModel) -> dict[str, Any]: + data = obj.model_dump(exclude={"embedding"}) + return data + + @staticmethod + def _validate_config( + config: Mapping[str, Any] | BaseModel | None, + model_type: type[TConfigModel], + ) -> TConfigModel: + if isinstance(config, model_type): + return config + if config is None: + return model_type() + return model_type.model_validate(config) + + def configure_pipeline(self, *, step_id: str, configs: Mapping[str, Any], pipeline: str = "memorize") -> int: + revision = self._pipelines.config_step(pipeline, step_id, dict(configs)) + return revision + + def insert_step_after( + self, + *, + target_step_id: str, + new_step: WorkflowStep, + pipeline: str = "memorize", + ) -> int: + revision = self._pipelines.insert_after(pipeline, target_step_id, new_step) + return revision + + def insert_step_before( + self, + *, + target_step_id: str, + new_step: WorkflowStep, + pipeline: str = "memorize", + ) -> int: + revision = self._pipelines.insert_before(pipeline, target_step_id, new_step) + return revision + + def replace_step( + self, + *, + target_step_id: str, + new_step: WorkflowStep, + pipeline: str = "memorize", + ) -> int: + revision = self._pipelines.replace_step(pipeline, target_step_id, new_step) + return revision + + def remove_step(self, *, target_step_id: str, pipeline: str = "memorize") -> int: + revision = self._pipelines.remove_step(pipeline, target_step_id) + return revision diff --git a/src/memu/app/settings.py b/src/memu/app/settings.py new file mode 100644 index 0000000..efd06ff --- /dev/null +++ b/src/memu/app/settings.py @@ -0,0 +1,594 @@ +from collections.abc import Mapping +from typing import Annotated, Any, Literal + +from pydantic import AfterValidator, BaseModel, BeforeValidator, Field, RootModel, StringConstraints, model_validator + +from memu.prompts.category_summary import ( + DEFAULT_CATEGORY_SUMMARY_PROMPT_ORDINAL, +) +from memu.prompts.category_summary import ( + PROMPT as CATEGORY_SUMMARY_PROMPT, +) +from memu.prompts.memory_type import ( + DEFAULT_MEMORY_CUSTOM_PROMPT_ORDINAL, + DEFAULT_MEMORY_TYPES, +) +from memu.prompts.memory_type import ( + PROMPTS as DEFAULT_MEMORY_TYPE_PROMPTS, +) + + +def normalize_value(v: str) -> str: + if isinstance(v, str): + return v.strip().lower() + return v + + +Normalize = BeforeValidator(normalize_value) + + +def _default_memory_types() -> list[str]: + return list(DEFAULT_MEMORY_TYPES) + + +def _default_memory_type_prompts() -> "dict[str, str | CustomPrompt]": + return dict(DEFAULT_MEMORY_TYPE_PROMPTS) + + +class PromptBlock(BaseModel): + label: str | None = None + ordinal: int = Field(default=0) + prompt: str | None = None + + +class CustomPrompt(RootModel[dict[str, PromptBlock]]): + root: dict[str, PromptBlock] = Field(default_factory=dict) + + def get(self, key: str, default: PromptBlock | None = None) -> PromptBlock | None: + return self.root.get(key, default) + + def items(self) -> list[tuple[str, PromptBlock]]: + return list(self.root.items()) + + +def complete_prompt_blocks(prompt: CustomPrompt, default_blocks: Mapping[str, int]) -> CustomPrompt: + for key, ordinal in default_blocks.items(): + if key not in prompt.root: + prompt.root[key] = PromptBlock(ordinal=ordinal) + return prompt + + +CompleteEntryTypePrompt = AfterValidator(lambda v: complete_prompt_blocks(v, DEFAULT_MEMORY_CUSTOM_PROMPT_ORDINAL)) + + +CompleteCategoryPrompt = AfterValidator(lambda v: complete_prompt_blocks(v, DEFAULT_CATEGORY_SUMMARY_PROMPT_ORDINAL)) + + +class CategoryConfig(BaseModel): + name: str + description: str = "" + target_length: int | None = None + summary_prompt: str | Annotated[CustomPrompt, CompleteCategoryPrompt] | None = None + + +class LazyLLMSource(BaseModel): + source: str | None = Field(default=None, description="default source for lazyllm client backend") + llm_source: str | None = Field(default=None, description="LLM source for lazyllm client backend") + embed_source: str | None = Field(default=None, description="Embedding source for lazyllm client backend") + vlm_source: str | None = Field(default=None, description="VLM source for lazyllm client backend") + stt_source: str | None = Field(default=None, description="STT source for lazyllm client backend") + vlm_model: str = Field(default="qwen-vl-plus", description="Vision language model for lazyllm client backend") + stt_model: str = Field(default="qwen-audio-turbo", description="Speech-to-text model for lazyllm client backend") + + +# Per-provider defaults: provider -> (base_url, api_key_env_or_value, chat_model). +# Used by ``LLMConfig.set_provider_defaults`` to swap OpenAI defaults when a provider is selected. +# Each provider defaults to its latest small/fast model (verified June 2026). +_PROVIDER_DEFAULTS: dict[str, tuple[str, str, str]] = { + "grok": ("https://api.x.ai/v1", "XAI_API_KEY", "grok-4-1-fast"), + "claude": ("https://api.anthropic.com", "ANTHROPIC_API_KEY", "claude-haiku-4-5"), + "deepseek": ("https://api.deepseek.com/v1", "DEEPSEEK_API_KEY", "deepseek-v4-flash"), + "kimi": ("https://api.moonshot.cn/v1", "MOONSHOT_API_KEY", "kimi-k2.7-code-highspeed"), + "minimax": ("https://api.minimax.io/v1", "MINIMAX_API_KEY", "MiniMax-M3"), + "doubao": ("https://ark.cn-beijing.volces.com", "ARK_API_KEY", "doubao-seed-2.0-lite"), + "openrouter": ("https://openrouter.ai", "OPENROUTER_API_KEY", "openai/gpt-5.4-mini"), +} + + +class LLMConfig(BaseModel): + provider: str = Field( + default="openai", + description="Identifier for the LLM provider implementation (used by HTTP client backend).", + ) + base_url: str = Field(default="https://api.openai.com/v1") + api_key: str = Field(default="OPENAI_API_KEY") + chat_model: str = Field(default="gpt-5.4-mini") + client_backend: str = Field( + default="sdk", + description=( + "Which LLM client backend to use: 'sdk' (official OpenAI SDK), " + "'anthropic' (official Anthropic/Claude SDK), 'httpx' (raw HTTP, supports " + "all providers in memu.llm.backends), or 'lazyllm_backend' (Qwen, Doubao, " + "SiliconFlow, etc.)." + ), + ) + lazyllm_source: LazyLLMSource = Field(default=LazyLLMSource()) + endpoint_overrides: dict[str, str] = Field( + default_factory=dict, + description="Optional overrides for HTTP endpoints (keys: 'chat'/'summary').", + ) + # Backward-compat bridge: embedding is owned by the dedicated + # ``memu.embedding`` clients (``EmbeddingConfig``/``embedding_profiles``). The + # LLM/chat clients no longer embed. These fields are only consumed by + # ``embedding_config_from_llm`` to derive an embedding profile from an LLM + # profile when no explicit ``embedding_profiles`` is supplied. Prefer setting + # ``embedding_profiles`` directly. + embed_model: str = Field( + default="text-embedding-3-small", + description="Embedding model used to derive an embedding profile from this LLM profile (bridge only).", + ) + embed_batch_size: int = Field( + default=1, + description="Embedding batch size used to derive an embedding profile from this LLM profile (bridge only).", + ) + + @model_validator(mode="after") + def set_provider_defaults(self) -> "LLMConfig": + # Per-provider defaults for the HTTP client backend. Each entry only + # overrides a field when it still holds the OpenAI default, so explicit + # user values are always preserved. + defaults = _PROVIDER_DEFAULTS.get(self.provider) + if defaults is not None: + base_url, api_key, chat_model = defaults + if self.base_url == "https://api.openai.com/v1": + self.base_url = base_url + if self.api_key == "OPENAI_API_KEY": + self.api_key = api_key + if self.chat_model == "gpt-5.4-mini": + self.chat_model = chat_model + return self + + +class VLMConfig(BaseModel): + """Configuration for a vision-language (multimodal) model client. + + Sibling to :class:`LLMConfig` but scoped to the ``vision`` capability used by + image/video preprocessing. Defaults to each provider's latest VLM model (see + ``memu.vlm.VLM_PROVIDER_DEFAULTS``) instead of the small/fast chat default. + """ + + provider: str = Field( + default="openai", + description="Identifier for the VLM provider implementation (used by HTTP client backend).", + ) + base_url: str = Field(default="https://api.openai.com/v1") + api_key: str = Field(default="OPENAI_API_KEY") + vlm_model: str = Field(default="gpt-5.4", description="Vision-language model used for image/video understanding.") + client_backend: str = Field( + default="sdk", + description=( + "Which VLM client backend to use: 'sdk' (official OpenAI SDK), " + "'anthropic' (official Anthropic/Claude SDK), or 'httpx' (raw HTTP, " + "supports all providers in memu.vlm.backends)." + ), + ) + endpoint_overrides: dict[str, str] = Field( + default_factory=dict, + description="Optional overrides for HTTP endpoints (key: 'vision').", + ) + + @model_validator(mode="after") + def set_provider_defaults(self) -> "VLMConfig": + # base_url/api_key reuse the shared per-provider HTTP defaults; vlm_model + # comes from the VLM-specific defaults. Each field is only overridden + # while it still holds the OpenAI default, so explicit values survive. + from memu.vlm.defaults import default_vlm_model + + defaults = _PROVIDER_DEFAULTS.get(self.provider) + if defaults is not None: + base_url, api_key, _chat_model = defaults + if self.base_url == "https://api.openai.com/v1": + self.base_url = base_url + if self.api_key == "OPENAI_API_KEY": + self.api_key = api_key + if self.vlm_model == "gpt-5.4": + resolved = default_vlm_model(self.provider) + if resolved is not None: + self.vlm_model = resolved + return self + + +def vlm_config_from_llm(llm: "LLMConfig") -> "VLMConfig": + """Derive a :class:`VLMConfig` from an :class:`LLMConfig`. + + Reuses the LLM provider/credentials/transport so vision steps work with zero + extra configuration, swapping only the model for the provider's latest VLM + (falling back to the LLM chat model when the provider has no known VLM). + """ + from memu.vlm.defaults import default_vlm_model + + # The anthropic SDK backend leaves ``provider`` at its generic default, so + # map it explicitly to resolve the right VLM model. + provider = "claude" if llm.client_backend == "anthropic" else llm.provider + vlm_model = default_vlm_model(provider) or llm.chat_model + return VLMConfig( + provider=provider, + base_url=llm.base_url, + api_key=llm.api_key, + vlm_model=vlm_model, + client_backend=llm.client_backend, + endpoint_overrides=dict(llm.endpoint_overrides), + ) + + +class EmbeddingConfig(BaseModel): + """Configuration for an embedding (vectorization) model client. + + Sibling to :class:`LLMConfig`/:class:`VLMConfig` but scoped to the embedding + capability used by vector search. Defaults to OpenAI's + ``text-embedding-3-small``; embedding-only providers (Jina, Voyage) bring + their own ``base_url``/``api_key`` via provider defaults (see + ``memu.embedding.defaults``). + """ + + provider: str = Field( + default="openai", + description="Identifier for the embedding provider implementation (used by HTTP client backend).", + ) + base_url: str = Field(default="https://api.openai.com/v1") + api_key: str = Field(default="OPENAI_API_KEY") + embed_model: str = Field( + default="text-embedding-3-small", + description="Embedding model used for vectorization.", + ) + embed_batch_size: int = Field( + default=1, + description="Maximum batch size for embedding API calls (used by the SDK client backend).", + ) + client_backend: str = Field( + default="sdk", + description=( + "Which embedding client backend to use: 'sdk' (official OpenAI SDK), " + "'httpx' (raw HTTP, supports all providers in memu.embedding.backends, " + "e.g. openai/jina/voyage/doubao/openrouter), or 'lazyllm_backend'." + ), + ) + lazyllm_source: LazyLLMSource = Field(default=LazyLLMSource()) + endpoint_overrides: dict[str, str] = Field( + default_factory=dict, + description="Optional overrides for HTTP endpoints (key: 'embeddings').", + ) + + @model_validator(mode="after") + def set_provider_defaults(self) -> "EmbeddingConfig": + from memu.embedding.defaults import EMBEDDING_PROVIDER_ENDPOINTS, default_embedding_model + + # base_url/api_key: reuse the shared chat per-provider defaults when the + # provider is also a chat provider; otherwise fall back to the + # embedding-only endpoint table (Jina, Voyage). Each field is only + # overridden while it still holds the OpenAI default, so explicit values + # survive. + endpoint = _PROVIDER_DEFAULTS.get(self.provider) + base_url = endpoint[0] if endpoint is not None else None + api_key = endpoint[1] if endpoint is not None else None + if base_url is None: + embed_endpoint = EMBEDDING_PROVIDER_ENDPOINTS.get(self.provider) + if embed_endpoint is not None: + base_url, api_key = embed_endpoint + if base_url is not None and self.base_url == "https://api.openai.com/v1": + self.base_url = base_url + if api_key is not None and self.api_key == "OPENAI_API_KEY": + self.api_key = api_key + if self.embed_model == "text-embedding-3-small": + resolved = default_embedding_model(self.provider) + if resolved is not None: + self.embed_model = resolved + return self + + +def embedding_config_from_llm(llm: "LLMConfig") -> "EmbeddingConfig": + """Derive an :class:`EmbeddingConfig` from an :class:`LLMConfig`. + + Reuses the LLM provider/credentials/transport/embed model so vectorization + works with zero extra configuration when no dedicated embedding profile is + supplied. The ``anthropic`` transport has no embeddings API; the derived + config preserves that so the embedding gateway raises a clear error. + """ + return EmbeddingConfig( + provider=llm.provider, + base_url=llm.base_url, + api_key=llm.api_key, + embed_model=llm.embed_model, + embed_batch_size=llm.embed_batch_size, + client_backend=llm.client_backend, + lazyllm_source=llm.lazyllm_source, + endpoint_overrides=dict(llm.endpoint_overrides), + ) + + +class BlobConfig(BaseModel): + provider: str = Field(default="local") + resources_dir: str = Field(default="./data/resources") + + +class MemoryFilesConfig(BaseModel): + """Render structured memory into a browsable markdown "memory file system". + + Purely additive and read-only against the store; disabled by default so it + never changes existing memorize/retrieve behavior. When enabled, the tree is + refreshed by ``memorize_workspace`` (and on demand via ``export_memory_files``). + """ + + enabled: bool = Field( + default=False, + description="Enable rendering structured memory into browsable markdown files.", + ) + output_dir: str = Field( + default="./data/memory", + description=( + "Directory where the memory markdown tree (the INDEX.md/MEMORY.md/SKILL.md root " + "indexes plus the resource/, memory/, and skill/ directories) is written." + ), + ) + synthesize: bool = Field( + default=False, + description=( + "Synthesize MEMORY.md and skill docs from per-source descriptions via the LLM " + "instead of rendering already-extracted records. INDEX.md stays deterministic." + ), + ) + synthesis_llm_profile: str = Field( + default="default", + description="LLM profile used when synthesize=True.", + ) + + +class RetrieveCategoryConfig(BaseModel): + enabled: bool = Field(default=True, description="Whether to enable category retrieval.") + top_k: int = Field(default=5, description="Total number of categories to retrieve.") + + +class RetrieveItemConfig(BaseModel): + enabled: bool = Field(default=True, description="Whether to enable item retrieval.") + top_k: int = Field(default=5, description="Total number of items to retrieve.") + # Reference-aware retrieval + use_category_references: bool = Field( + default=False, + description="When category retrieval is insufficient, follow [ref:ITEM_ID] citations to fetch referenced items.", + ) + # Salience-aware retrieval settings + ranking: Literal["similarity", "salience"] = Field( + default="similarity", + description="Ranking strategy: 'similarity' (cosine only) or 'salience' (weighted by reinforcement + recency).", + ) + recency_decay_days: float = Field( + default=30.0, + description="Half-life in days for recency decay in salience scoring. After this many days, recency factor is ~0.5.", + ) + + +class RetrieveResourceConfig(BaseModel): + enabled: bool = Field(default=True, description="Whether to enable resource retrieval.") + top_k: int = Field(default=5, description="Total number of resources to retrieve.") + + +class RetrieveFileConfig(BaseModel): + enabled: bool = Field(default=True, description="Whether to enable file retrieval.") + top_k: int = Field(default=5, description="Total number of files to retrieve.") + tracks: list[str] | None = Field( + default=None, + description="Optional file tracks (e.g. ['memory', 'skill']) to filter on. None means all tracks.", + ) + + +class RetrieveEntryConfig(BaseModel): + enabled: bool = Field(default=True, description="Whether to enable entry retrieval.") + top_k: int = Field(default=5, description="Total number of entries to retrieve.") + + +class RetrieveWorkspaceConfig(BaseModel): + """Configure the simple embedding-only workspace retrieval (``retrieve_workspace``). + + Unlike :class:`RetrieveConfig`, this drives a single-shot, LLM-free path that + embeds the query once and ranks each of the file/entry/resource layers by + vector similarity. No routing, sufficiency check, or summarization is involved. + """ + + method: Annotated[Literal["rag"], Normalize] = "rag" + file: RetrieveFileConfig = Field(default=RetrieveFileConfig()) + entry: RetrieveEntryConfig = Field(default=RetrieveEntryConfig()) + resource: RetrieveResourceConfig = Field(default=RetrieveResourceConfig()) + + +class RetrieveConfig(BaseModel): + """Configure retrieval behavior for `MemoryUser.retrieve`. + + Attributes: + method: Retrieval strategy. Use "rag" for embedding-based vector search or + "llm" to delegate ranking to the LLM. + top_k: Maximum number of results to return per category (and per stage), + controlling breadth of the retrieved context. + """ + + method: Annotated[Literal["rag", "llm"], Normalize] = "rag" + # top_k: int = Field( + # default=5, + # description="Maximum number of results to return per category.", + # ) + route_intention: bool = Field( + default=True, description="Whether to route intention (judge needs retrieval & rewrite query)." + ) + # route_intention_prompt: str = Field(default="", description="User prompt for route intention.") + # route_intention_llm_profile: str = Field(default="default", description="LLM profile for route intention.") + category: RetrieveCategoryConfig = Field(default=RetrieveCategoryConfig()) + item: RetrieveItemConfig = Field(default=RetrieveItemConfig()) + resource: RetrieveResourceConfig = Field(default=RetrieveResourceConfig()) + sufficiency_check: bool = Field(default=True, description="Whether to check sufficiency after each tier.") + sufficiency_check_prompt: str = Field(default="", description="User prompt for sufficiency check.") + sufficiency_check_llm_profile: str = Field(default="default", description="LLM profile for sufficiency check.") + llm_ranking_llm_profile: str = Field(default="default", description="LLM profile for LLM ranking.") + + +class MemorizeConfig(BaseModel): + category_assign_threshold: float = Field(default=0.25) + multimodal_preprocess_prompts: dict[str, str | CustomPrompt] = Field( + default_factory=dict, + description="Optional mapping of modality -> preprocess system prompt.", + ) + preprocess_llm_profile: str = Field(default="default", description="LLM profile for preprocess.") + vlm_profile: str = Field( + default="default", + description="LLM profile whose provider/credentials back the VLM client used for image/video vision.", + ) + memory_types: list[str] = Field( + default_factory=_default_memory_types, + description="Ordered list of memory types (profile/event/knowledge/behavior by default).", + ) + memory_type_prompts: dict[str, str | Annotated[CustomPrompt, CompleteEntryTypePrompt]] = Field( + default_factory=_default_memory_type_prompts, + description="User prompt overrides for each memory type extraction.", + ) + memory_extract_llm_profile: str = Field(default="default", description="LLM profile for memory extract.") + recall_files: list[CategoryConfig] = Field( + default_factory=list, + description=( + "Optional seed categories. The kernel presets no taxonomy: categories are " + "discovered adaptively from ingested content. Provide seeds only to guide " + "(not constrain) the taxonomy; an empty list means fully open/adaptive." + ), + ) + # default_category_summary_prompt: str | CustomPrompt = Field( + default_category_summary_prompt: str | Annotated[CustomPrompt, CompleteCategoryPrompt] = Field( + default=CATEGORY_SUMMARY_PROMPT, + description="Default system prompt for auto-generated category summaries.", + ) + default_category_summary_target_length: int = Field( + default=400, + description="Target max length for auto-generated category summaries.", + ) + category_update_llm_profile: str = Field(default="default", description="LLM profile for category summary.") + # Reference tracking for category summaries + enable_item_references: bool = Field( + default=False, + description="Enable inline [ref:ITEM_ID] citations in category summaries linking to source memory items.", + ) + enable_item_reinforcement: bool = Field( + default=False, + description="Enable reinforcement tracking for memory items.", + ) + + +class PatchConfig(BaseModel): + pass + + +class DefaultUserModel(BaseModel): + user_id: str | None = None + # Agent/session scoping for multi-agent and multi-session memory filtering + agent_id: str | None = None + # session_id: str | None = None + + +class UserConfig(BaseModel): + model: type[BaseModel] = Field(default=DefaultUserModel) + + +Key = Annotated[str, StringConstraints(min_length=1)] + + +class LLMProfilesConfig(RootModel[dict[Key, LLMConfig]]): + root: dict[str, LLMConfig] = Field(default_factory=lambda: {"default": LLMConfig()}) + + def get(self, key: str, default: LLMConfig | None = None) -> LLMConfig | None: + return self.root.get(key, default) + + @model_validator(mode="before") + @classmethod + def ensure_default(cls, data: Any) -> Any: + # if data is None: + # return {"default": LLMConfig()} + # if isinstance(data, dict) and "default" not in data: + # data = dict(data) + # data["default"] = LLMConfig() + # return data + if data is None: + data = {} + elif isinstance(data, dict): + data = dict(data) + else: + return data + if "default" not in data: + data["default"] = LLMConfig() + if "embedding" not in data: + data["embedding"] = data["default"] + return data + + @property + def profiles(self) -> dict[str, LLMConfig]: + return self.root + + @property + def default(self) -> LLMConfig: + return self.root.get("default", LLMConfig()) + + +class EmbeddingProfilesConfig(RootModel[dict[Key, EmbeddingConfig]]): + """Named embedding profiles, mirroring :class:`LLMProfilesConfig`. + + When no explicit embedding profiles are supplied, the service derives them + from the LLM profiles (see ``embedding_config_from_llm``) so existing + configs keep vectorizing through the same provider/credentials. + """ + + root: dict[str, EmbeddingConfig] = Field(default_factory=lambda: {"default": EmbeddingConfig()}) + + def get(self, key: str, default: EmbeddingConfig | None = None) -> EmbeddingConfig | None: + return self.root.get(key, default) + + @model_validator(mode="before") + @classmethod + def ensure_default(cls, data: Any) -> Any: + if data is None: + data = {} + elif isinstance(data, dict): + data = dict(data) + else: + return data + if "default" not in data: + data["default"] = EmbeddingConfig() + if "embedding" not in data: + data["embedding"] = data["default"] + return data + + @property + def profiles(self) -> dict[str, EmbeddingConfig]: + return self.root + + @property + def default(self) -> EmbeddingConfig: + return self.root.get("default", EmbeddingConfig()) + + +class MetadataStoreConfig(BaseModel): + provider: Annotated[Literal["inmemory", "postgres", "sqlite"], Normalize] = "inmemory" + ddl_mode: Annotated[Literal["create", "validate"], Normalize] = "create" + dsn: str | None = Field(default=None, description="Database connection string (required for postgres/sqlite).") + + +class VectorIndexConfig(BaseModel): + provider: Annotated[Literal["bruteforce", "pgvector", "none"], Normalize] = "bruteforce" + dsn: str | None = Field(default=None, description="Postgres connection string when provider=pgvector.") + + +class DatabaseConfig(BaseModel): + metadata_store: MetadataStoreConfig = Field(default_factory=MetadataStoreConfig) + vector_index: VectorIndexConfig | None = Field(default=None) + + def model_post_init(self, __context: Any) -> None: + if self.vector_index is None: + if self.metadata_store.provider == "postgres": + self.vector_index = VectorIndexConfig(provider="pgvector", dsn=self.metadata_store.dsn) + else: + self.vector_index = VectorIndexConfig(provider="bruteforce") + elif self.vector_index.provider == "pgvector" and self.vector_index.dsn is None: + self.vector_index = self.vector_index.model_copy(update={"dsn": self.metadata_store.dsn}) diff --git a/src/memu/blob/__init__.py b/src/memu/blob/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/memu/blob/document_text.py b/src/memu/blob/document_text.py new file mode 100644 index 0000000..30a8657 --- /dev/null +++ b/src/memu/blob/document_text.py @@ -0,0 +1,65 @@ +"""Document text extraction for the ``document`` modality. + +Plain-text documents (``.txt``/``.md``/``.json``/...) are read directly as +UTF-8. Rich office/binary formats (PDF, Word, PowerPoint, Excel, HTML, EPub, +...) are converted to Markdown via Microsoft's `MarkItDown +`_ so downstream memory extraction +receives clean, structure-preserving text instead of raw bytes. + +MarkItDown is an optional dependency (``pip install 'memu-py[document]'``); it is +imported lazily so installations that never ingest rich documents stay slim. +""" + +from __future__ import annotations + +import logging +import pathlib + +logger = logging.getLogger(__name__) + +# Rich formats that MarkItDown converts to Markdown. Anything not listed here is +# read directly as UTF-8 text. +MARKITDOWN_EXTENSIONS: frozenset[str] = frozenset({ + ".pdf", + ".docx", + ".doc", + ".pptx", + ".ppt", + ".xlsx", + ".xls", + ".html", + ".htm", + ".epub", +}) + + +def is_rich_document(path: str | pathlib.Path) -> bool: + """Whether ``path`` is a rich format that requires MarkItDown conversion.""" + return pathlib.Path(path).suffix.lower() in MARKITDOWN_EXTENSIONS + + +def extract_document_text(path: str | pathlib.Path) -> str: + """Extract document text, converting rich formats to Markdown via MarkItDown. + + Plain-text files are read directly as UTF-8. PDF/Office/HTML/EPub files are + routed through MarkItDown's local-file converter (the narrowest API, per its + security guidance). Raises ``RuntimeError`` with an actionable message when a + rich format is encountered but MarkItDown is not installed. + """ + p = pathlib.Path(path) + if is_rich_document(p): + return _convert_with_markitdown(p) + return p.read_text(encoding="utf-8") + + +def _convert_with_markitdown(path: pathlib.Path) -> str: + try: + from markitdown import MarkItDown + except ImportError as exc: # pragma: no cover - exercised via tests with monkeypatch + msg = f"Converting '{path.name}' requires MarkItDown. Install it with: pip install 'memu-py[document]'" + raise RuntimeError(msg) from exc + + # ``convert_local`` is the narrowest converter for trusted local files. + converter = MarkItDown(enable_plugins=False) + result = converter.convert_local(str(path)) + return result.text_content or "" diff --git a/src/memu/blob/folder.py b/src/memu/blob/folder.py new file mode 100644 index 0000000..8f64414 --- /dev/null +++ b/src/memu/blob/folder.py @@ -0,0 +1,185 @@ +"""Folder scanning, modality inference, and input-side manifest diffing. + +The folder-based ``memorize`` entry point treats a directory as the unit of +ingestion. This module is responsible for the *input* side of the sync: + +- recursively scan a folder for ingestible files, +- infer each file's modality from its extension (skipping unknown types), +- maintain a sidecar ``.memu_manifest.json`` (``relative path -> content hash``) + so each call can tell which files were added, modified, or deleted. + +This is independent of the *output* side manifest used by ``memu.memory_fs`` +(which hashes the rendered markdown artifacts). +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import pathlib +from dataclasses import dataclass, field + +logger = logging.getLogger(__name__) + +MANIFEST_FILENAME = ".memu_manifest.json" + +# Extension -> modality. Ambiguous extensions (.json, .webm) are mapped to a +# single sensible default and can be made configurable later if needed. +EXT_MODALITY: dict[str, str] = { + ".json": "conversation", + ".txt": "document", + ".md": "document", + ".text": "document", + # Rich document formats converted to Markdown via MarkItDown on ingest. + ".pdf": "document", + ".docx": "document", + ".doc": "document", + ".pptx": "document", + ".ppt": "document", + ".xlsx": "document", + ".xls": "document", + ".html": "document", + ".htm": "document", + ".epub": "document", + ".jpg": "image", + ".jpeg": "image", + ".png": "image", + ".gif": "image", + ".webp": "image", + ".mp4": "video", + ".mov": "video", + ".mkv": "video", + ".avi": "video", + ".mp3": "audio", + ".wav": "audio", + ".m4a": "audio", + ".mpeg": "audio", + ".mpga": "audio", +} + + +@dataclass(frozen=True) +class ScannedFile: + """A single ingestible file discovered in the scanned folder.""" + + rel_path: str + abs_path: str + modality: str + content_hash: str + + +@dataclass +class FolderDiff: + """The added/modified/deleted sets between a scan and the prior manifest.""" + + added: list[ScannedFile] = field(default_factory=list) + modified: list[ScannedFile] = field(default_factory=list) + deleted: list[str] = field(default_factory=list) + + @property + def has_changes(self) -> bool: + return bool(self.added or self.modified or self.deleted) + + @property + def has_removals(self) -> bool: + """Whether any file was modified or deleted (i.e. stale memory exists).""" + return bool(self.modified or self.deleted) + + +def infer_modality(path: str | pathlib.Path) -> str | None: + """Infer modality from a file extension, or None if unsupported.""" + return EXT_MODALITY.get(pathlib.Path(path).suffix.lower()) + + +def compute_file_hash(path: str | pathlib.Path) -> str: + """Content hash (sha256) of a file, streamed to stay memory-friendly.""" + digest = hashlib.sha256() + with pathlib.Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(65536), b""): + digest.update(chunk) + return digest.hexdigest() + + +def scan_folder(folder: str | pathlib.Path) -> dict[str, ScannedFile]: + """Recursively scan a folder for ingestible files keyed by relative path. + + Hidden files/dirs (dot-prefixed), the manifest itself, and files with an + unknown extension are skipped (the latter is logged). + """ + root = pathlib.Path(folder).resolve() + scanned: dict[str, ScannedFile] = {} + if not root.is_dir(): + msg = f"memorize() expects an existing folder, got: {folder}" + raise NotADirectoryError(msg) + + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == MANIFEST_FILENAME: + continue + rel = path.relative_to(root) + if any(part.startswith(".") for part in rel.parts): + continue + modality = infer_modality(path) + if modality is None: + logger.warning("Skipping file with unsupported extension: %s", rel) + continue + rel_str = rel.as_posix() + scanned[rel_str] = ScannedFile( + rel_path=rel_str, + abs_path=str(path), + modality=modality, + content_hash=compute_file_hash(path), + ) + return scanned + + +def load_manifest(folder: str | pathlib.Path) -> dict[str, str]: + """Load the sidecar input manifest (``relative path -> content hash``).""" + manifest_path = pathlib.Path(folder).resolve() / MANIFEST_FILENAME + if not manifest_path.exists(): + return {} + try: + data = json.loads(manifest_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return {} + if not isinstance(data, dict): + return {} + return {str(key): str(value) for key, value in data.items()} + + +def save_manifest(folder: str | pathlib.Path, manifest: dict[str, str]) -> None: + """Persist the input manifest into the scanned folder.""" + manifest_path = pathlib.Path(folder).resolve() / MANIFEST_FILENAME + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + + +def diff_folder(scanned: dict[str, ScannedFile], manifest: dict[str, str]) -> FolderDiff: + """Compute added/modified/deleted relative to a prior manifest.""" + diff = FolderDiff() + for rel, scanned_file in scanned.items(): + prior_hash = manifest.get(rel) + if prior_hash is None: + diff.added.append(scanned_file) + elif prior_hash != scanned_file.content_hash: + diff.modified.append(scanned_file) + diff.deleted = [rel for rel in manifest if rel not in scanned] + return diff + + +def manifest_from_scan(scanned: dict[str, ScannedFile]) -> dict[str, str]: + """Build the manifest payload to persist from a fresh scan.""" + return {rel: scanned_file.content_hash for rel, scanned_file in scanned.items()} + + +__all__ = [ + "EXT_MODALITY", + "MANIFEST_FILENAME", + "FolderDiff", + "ScannedFile", + "compute_file_hash", + "diff_folder", + "infer_modality", + "load_manifest", + "manifest_from_scan", + "save_manifest", +] diff --git a/src/memu/blob/local_fs.py b/src/memu/blob/local_fs.py new file mode 100644 index 0000000..c4d4f06 --- /dev/null +++ b/src/memu/blob/local_fs.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import asyncio +import pathlib +import shutil +from urllib.parse import parse_qs, urlparse + +import httpx + +from memu.blob.document_text import extract_document_text + +# Modalities whose ingested text is read straight from the file as UTF-8. +_PLAIN_TEXT_MODALITIES = frozenset({"conversation", "text"}) + + +class LocalFS: + def __init__(self, base_dir: str): + self.base = pathlib.Path(base_dir) + self.base.mkdir(parents=True, exist_ok=True) + + def _get_filename_from_url(self, url: str, modality: str) -> str: + """ + Extract a clean filename from URL, handling query parameters. + + Args: + url: The URL to parse + modality: The resource modality (for extension inference) + + Returns: + A clean filename without query parameters + """ + parsed = urlparse(url) + path = parsed.path + + # Get base filename from path + filename = pathlib.Path(path).name + + # If filename has no extension or is just a script name (like grab.php), + # try to get the real extension from query parameters or use modality + if not filename or "." not in filename or filename.endswith(".php"): + # Check for 'type' parameter in query string (e.g., ?type=mp3) + query_params = parse_qs(parsed.query) + if "type" in query_params: + ext = query_params["type"][0] + # Generate a filename based on the ID if available + filename = f"audio_{query_params['id'][0]}.{ext}" if "id" in query_params else f"resource.{ext}" + else: + # Use modality to infer extension + ext_map = { + "audio": "mp3", + "video": "mp4", + "image": "jpg", + "document": "txt", + } + ext = ext_map.get(modality, "bin") + filename = f"resource.{ext}" + + # Remove any remaining query parameters from filename + filename = filename.split("?")[0] + + return filename + + async def fetch(self, url: str, modality: str) -> tuple[str, str | None]: + # Local path + p = pathlib.Path(url) + if p.exists(): + dst = self.base / p.name + if str(p.resolve()) != str(dst.resolve()): + shutil.copyfile(p, dst) + text = await self._read_text(dst, modality) + return str(dst), text + + # HTTP - get clean filename + filename = self._get_filename_from_url(url, modality) + dst = self.base / filename + + async with httpx.AsyncClient(timeout=60) as client: + r = await client.get(url) + r.raise_for_status() + dst.write_bytes(r.content) + text = await self._read_text(dst, modality) + return str(dst), text + + @staticmethod + async def _read_text(path: pathlib.Path, modality: str) -> str | None: + """Resolve the ingested text for a fetched file based on its modality. + + - ``document``: extract via MarkItDown for rich formats (PDF/Office/...), + or read plain text directly; runs off-loop since extraction is blocking. + - ``conversation``/``text``: read the file as UTF-8. + - everything else (image/audio/video): no inline text. + """ + if modality == "document": + return await asyncio.to_thread(extract_document_text, path) + if modality in _PLAIN_TEXT_MODALITIES: + return path.read_text(encoding="utf-8") + return None diff --git a/src/memu/cli.py b/src/memu/cli.py new file mode 100644 index 0000000..7cccc9d --- /dev/null +++ b/src/memu/cli.py @@ -0,0 +1,253 @@ +"""Command-line interface for memU. + +A thin wrapper over :class:`memu.app.MemoryService` exposing the four public +entry points (``memorize``, ``memorize-workspace``, ``retrieve``, +``retrieve-workspace``) plus the memory-file export. State persists across +invocations through a SQLite database (``--db``, default ``./data/memu.sqlite3``), +so the CLI composes like the library: memorize in one call, retrieve in the next. + +Configuration mirrors the library defaults; every flag also reads a ``MEMU_*`` +environment variable so CI/agents can configure once and pass only the command. + +Usage: + memu memorize notes/meeting.md + memu memorize-workspace ./workspace + memu retrieve "What are this user's launch preferences?" + memu retrieve-workspace "deploy checklist" + memu export +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import pathlib +import sys +from collections.abc import Callable, Coroutine +from typing import Any + +from memu.blob.folder import EXT_MODALITY, infer_modality + +MODALITIES = ("conversation", "document", "image", "video", "audio") + + +def _env(name: str, default: str) -> str: + return os.environ.get(name, default) + + +def _add_common_options(parser: argparse.ArgumentParser) -> None: + group = parser.add_argument_group("service options (env var in parens)") + group.add_argument( + "--provider", + default=_env("MEMU_LLM_PROVIDER", "openai"), + help="LLM provider id, e.g. openai, anthropic, deepseek (MEMU_LLM_PROVIDER)", + ) + group.add_argument( + "--model", + default=os.environ.get("MEMU_CHAT_MODEL"), + help="Chat model override; defaults to the provider's default (MEMU_CHAT_MODEL)", + ) + group.add_argument( + "--base-url", + default=os.environ.get("MEMU_BASE_URL"), + help="API base URL override (MEMU_BASE_URL)", + ) + group.add_argument( + "--api-key", + default=os.environ.get("MEMU_API_KEY"), + help="API key value or env-var name; defaults to the provider's env var, e.g. OPENAI_API_KEY (MEMU_API_KEY)", + ) + group.add_argument( + "--db", + default=_env("MEMU_DB", "./data/memu.sqlite3"), + help="SQLite file path, postgres:// DSN, or ':memory:' (MEMU_DB)", + ) + group.add_argument( + "--resources-dir", + default=_env("MEMU_RESOURCES_DIR", "./data/resources"), + help="Directory where ingested source files are copied (MEMU_RESOURCES_DIR)", + ) + group.add_argument( + "--memory-dir", + default=_env("MEMU_MEMORY_DIR", "./data/memory"), + help="Directory for the INDEX.md/MEMORY.md/SKILL.md markdown tree (MEMU_MEMORY_DIR)", + ) + group.add_argument( + "--synthesize", + action="store_true", + default=_env("MEMU_SYNTHESIZE", "") == "1", + help="LLM-synthesize MEMORY.md/SKILL.md overviews instead of deterministic indexes (MEMU_SYNTHESIZE=1)", + ) + parser.add_argument("--json", action="store_true", help="Print the raw JSON response") + + +def _database_config(db: str) -> dict[str, Any]: + if db in (":memory:", "inmemory"): + return {"metadata_store": {"provider": "inmemory"}} + if db.startswith(("postgres://", "postgresql://")): + return {"metadata_store": {"provider": "postgres", "dsn": db}} + path = pathlib.Path(db).expanduser() + path.parent.mkdir(parents=True, exist_ok=True) + return {"metadata_store": {"provider": "sqlite", "dsn": f"sqlite:///{path}"}} + + +def _build_service(args: argparse.Namespace, *, memory_files: bool) -> Any: + # Imported lazily so `memu --help` stays fast and dependency errors surface + # only when a command actually runs. + from memu.app import MemoryService + + llm: dict[str, Any] = {"provider": args.provider} + if args.model: + llm["chat_model"] = args.model + if args.base_url: + llm["base_url"] = args.base_url + if args.api_key: + llm["api_key"] = args.api_key + return MemoryService( + llm_profiles={"default": llm}, + database_config=_database_config(args.db), + blob_config={"resources_dir": args.resources_dir}, + memory_files_config={ + "enabled": memory_files, + "output_dir": args.memory_dir, + "synthesize": args.synthesize, + }, + ) + + +def _print_json(payload: Any) -> None: + print(json.dumps(payload, indent=2, ensure_ascii=False, default=str)) + + +async def _cmd_memorize(args: argparse.Namespace) -> int: + path = pathlib.Path(args.path).expanduser() + if not path.exists(): + print(f"error: no such file: {path}", file=sys.stderr) + return 2 + modality = args.modality or infer_modality(path) + if modality is None: + supported = ", ".join(sorted(EXT_MODALITY)) + print( + f"error: cannot infer modality for '{path.suffix}'; pass --modality (supported extensions: {supported})", + file=sys.stderr, + ) + return 2 + service = _build_service(args, memory_files=False) + result = await service.memorize(resource_url=str(path), modality=modality) + if args.json: + _print_json(result) + return 0 + entries = result.get("items", []) + files = result.get("categories", []) + print(f"memorized {path} ({modality}): {len(entries)} entries across {len(files)} files") + for f in files: + print(f" - {f.get('name')}") + return 0 + + +async def _cmd_memorize_workspace(args: argparse.Namespace) -> int: + folder = pathlib.Path(args.folder).expanduser() + if not folder.is_dir(): + print(f"error: no such folder: {folder}", file=sys.stderr) + return 2 + service = _build_service(args, memory_files=not args.no_export) + result = await service.memorize_workspace(folder=str(folder)) + if args.json: + _print_json(result) + return 0 + added, modified, deleted = result.get("added", []), result.get("modified", []), result.get("deleted", []) + print(f"synced {folder}: +{len(added)} added, ~{len(modified)} modified, -{len(deleted)} deleted") + for label, names in (("+", added), ("~", modified), ("-", deleted)): + for name in names: + print(f" {label} {name}") + if not args.no_export: + print(f"memory files written to {args.memory_dir}") + return 0 + + +async def _cmd_retrieve(args: argparse.Namespace) -> int: + service = _build_service(args, memory_files=False) + result = await service.retrieve(queries=[{"role": "user", "content": {"text": args.query}}]) + _print_json(result) + return 0 + + +async def _cmd_retrieve_workspace(args: argparse.Namespace) -> int: + service = _build_service(args, memory_files=False) + result = await service.retrieve_workspace(args.query) + _print_json(result) + return 0 + + +async def _cmd_export(args: argparse.Namespace) -> int: + service = _build_service(args, memory_files=True) + result = await service.export_memory_files() + if args.json: + _print_json(result) + else: + print(f"memory files written to {args.memory_dir}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="memu", + description="memU — personal memory as files. Memorize sources, retrieve context.", + ) + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("memorize", help="Memorize a single file (conversation, document, image, audio, video)") + p.add_argument("path", help="Path to the source file") + p.add_argument("--modality", choices=MODALITIES, help="Override the extension-inferred modality") + _add_common_options(p) + p.set_defaults(handler=_cmd_memorize) + + p = sub.add_parser( + "memorize-workspace", + aliases=["sync"], + help="Diff-sync a folder into memory (chat/ -> memory, agent/ -> skills, other -> index)", + ) + p.add_argument("folder", help="Workspace folder to scan and sync") + p.add_argument("--no-export", action="store_true", help="Skip rebuilding the markdown memory tree") + _add_common_options(p) + p.set_defaults(handler=_cmd_memorize_workspace) + + p = sub.add_parser("retrieve", help="LLM-routed retrieval over memorized entries (heavy, high quality)") + p.add_argument("query", help="Natural-language query") + _add_common_options(p) + p.set_defaults(handler=_cmd_retrieve) + + p = sub.add_parser( + "retrieve-workspace", + aliases=["search"], + help="Single-shot embedding retrieval over segments/files/resources (LLM-free, fast)", + ) + p.add_argument("query", help="Natural-language query") + _add_common_options(p) + p.set_defaults(handler=_cmd_retrieve_workspace) + + p = sub.add_parser("export", help="Rebuild the INDEX.md/MEMORY.md/SKILL.md markdown tree from the store") + _add_common_options(p) + p.set_defaults(handler=_cmd_export) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + handler: Callable[[argparse.Namespace], Coroutine[Any, Any, int]] = args.handler + try: + return asyncio.run(handler(args)) + except KeyboardInterrupt: + return 130 + except Exception as exc: + if os.environ.get("MEMU_DEBUG") == "1": + raise + print(f"error: {exc} (set MEMU_DEBUG=1 for a traceback)", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/memu/database/__init__.py b/src/memu/database/__init__.py new file mode 100644 index 0000000..84960b0 --- /dev/null +++ b/src/memu/database/__init__.py @@ -0,0 +1,41 @@ +"""Storage backends for MemU.""" + +from memu.database.factory import build_database +from memu.database.interfaces import ( + Database, + RecallEntryRecord, + RecallFileEntryRecord, + RecallFileRecord, + RecallFileResourceRecord, + RecallFileSegmentRecord, + ResourceRecord, +) +from memu.database.repositories import ( + RecallEntryRepo, + RecallFileEntryRepo, + RecallFileRepo, + RecallFileResourceRepo, + RecallFileSegmentRepo, + ResourceRepo, +) + +__all__ = [ + "Database", + "RecallEntryRecord", + "RecallEntryRepo", + "RecallFileEntryRecord", + "RecallFileEntryRepo", + "RecallFileRecord", + "RecallFileRepo", + "RecallFileResourceRecord", + "RecallFileResourceRepo", + "RecallFileSegmentRecord", + "RecallFileSegmentRepo", + "ResourceRecord", + "ResourceRepo", + "build_database", + "inmemory", + "postgres", + "schema", + "sqlite", +] diff --git a/src/memu/database/factory.py b/src/memu/database/factory.py new file mode 100644 index 0000000..dd1fcb5 --- /dev/null +++ b/src/memu/database/factory.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from pydantic import BaseModel + +from memu.app.settings import DatabaseConfig +from memu.database.inmemory import build_inmemory_database +from memu.database.interfaces import Database + +if TYPE_CHECKING: + pass + + +def build_database( + *, + config: DatabaseConfig, + user_model: type[BaseModel], +) -> Database: + """ + Initialize a database backend for the configured provider. + + Supported providers: + - "inmemory": In-memory storage (default, no persistence) + - "postgres": PostgreSQL with optional pgvector support + - "sqlite": SQLite file-based storage (lightweight, portable) + """ + provider = config.metadata_store.provider + if provider == "inmemory": + return build_inmemory_database(config=config, user_model=user_model) + elif provider == "postgres": + # Lazy import to avoid requiring pgvector when not using postgres + from memu.database.postgres import build_postgres_database + + return build_postgres_database(config=config, user_model=user_model) + elif provider == "sqlite": + # Lazy import to avoid loading SQLite dependencies when not needed + from memu.database.sqlite import build_sqlite_database + + return build_sqlite_database(config=config, user_model=user_model) + else: + msg = f"Unsupported metadata_store provider: {provider}" + raise ValueError(msg) diff --git a/src/memu/database/inmemory/__init__.py b/src/memu/database/inmemory/__init__.py new file mode 100644 index 0000000..0de91a8 --- /dev/null +++ b/src/memu/database/inmemory/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pydantic import BaseModel + +from memu.app.settings import DatabaseConfig +from memu.database.inmemory.models import build_inmemory_models +from memu.database.inmemory.repo import InMemoryStore + + +def build_inmemory_database( + *, + config: DatabaseConfig, + user_model: type[BaseModel], +) -> InMemoryStore: + ( + resource_model, + recall_file_model, + recall_entry_model, + recall_file_entry_model, + recall_file_resource_model, + recall_file_segment_model, + ) = build_inmemory_models(user_model) + return InMemoryStore( + scope_model=user_model, + resource_model=resource_model, + recall_entry_model=recall_entry_model, + recall_file_model=recall_file_model, + recall_file_entry_model=recall_file_entry_model, + recall_file_resource_model=recall_file_resource_model, + recall_file_segment_model=recall_file_segment_model, + ) + + +__all__ = ["build_inmemory_database"] diff --git a/src/memu/database/inmemory/models.py b/src/memu/database/inmemory/models.py new file mode 100644 index 0000000..0ff2412 --- /dev/null +++ b/src/memu/database/inmemory/models.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from pydantic import BaseModel + +from memu.database.models import ( + RecallEntry, + RecallFile, + RecallFileEntry, + RecallFileResource, + RecallFileSegment, + Resource, + merge_scope_model, +) + + +class InMemoryResource(Resource): + """Concrete in-memory resource model.""" + + +class InMemoryRecallEntry(RecallEntry): + """Concrete in-memory memory item model.""" + + +class InMemoryRecallFile(RecallFile): + """Concrete in-memory memory category model.""" + + +class InMemoryFileEntry(RecallFileEntry): + """Concrete in-memory relation model.""" + + +class InMemoryFileResource(RecallFileResource): + """Concrete in-memory resource-category relation model.""" + + +class InMemoryFileSegment(RecallFileSegment): + """Concrete in-memory file-segment model.""" + + +def build_inmemory_models( + user_model: type[BaseModel], +) -> tuple[ + type[InMemoryResource], + type[InMemoryRecallFile], + type[InMemoryRecallEntry], + type[InMemoryFileEntry], + type[InMemoryFileResource], + type[InMemoryFileSegment], +]: + """ + Build scoped in-memory models that inherit from both the base interface and the user scope model. + """ + resource_model = merge_scope_model(user_model, InMemoryResource, name_suffix="Resource") + recall_file_model = merge_scope_model(user_model, InMemoryRecallFile, name_suffix="RecallFile") + recall_entry_model = merge_scope_model(user_model, InMemoryRecallEntry, name_suffix="RecallEntry") + recall_file_entry_model = merge_scope_model(user_model, InMemoryFileEntry, name_suffix="RecallFileEntry") + recall_file_resource_model = merge_scope_model(user_model, InMemoryFileResource, name_suffix="RecallFileResource") + recall_file_segment_model = merge_scope_model(user_model, InMemoryFileSegment, name_suffix="RecallFileSegment") + return ( + resource_model, + recall_file_model, + recall_entry_model, + recall_file_entry_model, + recall_file_resource_model, + recall_file_segment_model, + ) + + +__all__ = [ + "InMemoryFileEntry", + "InMemoryFileResource", + "InMemoryFileSegment", + "InMemoryRecallEntry", + "InMemoryRecallFile", + "InMemoryResource", + "build_inmemory_models", +] diff --git a/src/memu/database/inmemory/repo.py b/src/memu/database/inmemory/repo.py new file mode 100644 index 0000000..0da8228 --- /dev/null +++ b/src/memu/database/inmemory/repo.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel + +from memu.database.inmemory.models import build_inmemory_models +from memu.database.inmemory.repositories import ( + InMemoryFileEntryRepository, + InMemoryFileResourceRepository, + InMemoryFileSegmentRepository, + InMemoryRecallEntryRepository, + InMemoryRecallFileRepository, + InMemoryResourceRepository, +) +from memu.database.inmemory.state import InMemoryState +from memu.database.interfaces import Database +from memu.database.models import ( + RecallEntry, + RecallFile, + RecallFileEntry, + RecallFileResource, + RecallFileSegment, + Resource, +) +from memu.database.repositories import RecallFileRepo, ResourceRepo + + +class InMemoryStore(Database): + def __init__( + self, + *, + scope_model: type[BaseModel] | None = None, + resource_model: type[Any] | None = None, + recall_entry_model: type[Any] | None = None, + recall_file_model: type[Any] | None = None, + recall_file_entry_model: type[Any] | None = None, + recall_file_resource_model: type[Any] | None = None, + recall_file_segment_model: type[Any] | None = None, + state: InMemoryState | None = None, + ) -> None: + self.scope_model = scope_model or BaseModel + ( + default_resource_model, + default_recall_file_model, + default_recall_entry_model, + default_recall_file_entry_model, + default_recall_file_resource_model, + default_recall_file_segment_model, + ) = build_inmemory_models(self.scope_model) + + self.state = state or InMemoryState() + self.resources: dict[str, Resource] = self.state.resources + self.items: dict[str, RecallEntry] = self.state.items + self.categories: dict[str, RecallFile] = self.state.categories + self.relations: list[RecallFileEntry] = self.state.relations + self.resource_relations: list[RecallFileResource] = self.state.resource_relations + self.segments: list[RecallFileSegment] = self.state.segments + + resource_model = resource_model or default_resource_model or Resource + recall_entry_model = recall_entry_model or default_recall_entry_model or RecallEntry + recall_file_model = recall_file_model or default_recall_file_model or RecallFile + recall_file_entry_model = recall_file_entry_model or default_recall_file_entry_model or RecallFileEntry + recall_file_resource_model = ( + recall_file_resource_model or default_recall_file_resource_model or RecallFileResource + ) + recall_file_segment_model = recall_file_segment_model or default_recall_file_segment_model or RecallFileSegment + + self.resource_repo: ResourceRepo = InMemoryResourceRepository(state=self.state, resource_model=resource_model) + self.recall_file_repo: RecallFileRepo = InMemoryRecallFileRepository( + state=self.state, recall_file_model=recall_file_model + ) + self.recall_entry_repo = InMemoryRecallEntryRepository(state=self.state, recall_entry_model=recall_entry_model) + self.recall_file_entry_repo = InMemoryFileEntryRepository( + state=self.state, recall_file_entry_model=recall_file_entry_model + ) + self.recall_file_resource_repo = InMemoryFileResourceRepository( + state=self.state, recall_file_resource_model=recall_file_resource_model + ) + self.recall_file_segment_repo = InMemoryFileSegmentRepository( + state=self.state, recall_file_segment_model=recall_file_segment_model + ) + + def close(self) -> None: + return None diff --git a/src/memu/database/inmemory/repositories/__init__.py b/src/memu/database/inmemory/repositories/__init__.py new file mode 100644 index 0000000..fa6481b --- /dev/null +++ b/src/memu/database/inmemory/repositories/__init__.py @@ -0,0 +1,33 @@ +from memu.database.inmemory.repositories.recall_entry_repo import InMemoryRecallEntryRepository, RecallEntryRepo +from memu.database.inmemory.repositories.recall_file_entry_repo import ( + InMemoryFileEntryRepository, + RecallFileEntryRepo, +) +from memu.database.inmemory.repositories.recall_file_repo import ( + InMemoryRecallFileRepository, + RecallFileRepo, +) +from memu.database.inmemory.repositories.recall_file_resource_repo import ( + InMemoryFileResourceRepository, + RecallFileResourceRepo, +) +from memu.database.inmemory.repositories.recall_file_segment_repo import ( + InMemoryFileSegmentRepository, + RecallFileSegmentRepo, +) +from memu.database.inmemory.repositories.resource_repo import InMemoryResourceRepository, ResourceRepo + +__all__ = [ + "InMemoryFileEntryRepository", + "InMemoryFileResourceRepository", + "InMemoryFileSegmentRepository", + "InMemoryRecallEntryRepository", + "InMemoryRecallFileRepository", + "InMemoryResourceRepository", + "RecallEntryRepo", + "RecallFileEntryRepo", + "RecallFileRepo", + "RecallFileResourceRepo", + "RecallFileSegmentRepo", + "ResourceRepo", +] diff --git a/src/memu/database/inmemory/repositories/filter.py b/src/memu/database/inmemory/repositories/filter.py new file mode 100644 index 0000000..ad245cc --- /dev/null +++ b/src/memu/database/inmemory/repositories/filter.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + + +def matches_where(obj: Any, where: Mapping[str, Any] | None) -> bool: + """Basic field/`__in` matcher for in-memory repos.""" + if not where: + return True + for raw_key, expected in where.items(): + if expected is None: + continue + field, op = [*raw_key.split("__", 1), None][:2] + actual = getattr(obj, str(field), None) + if op == "in": + if isinstance(expected, str): + if actual != expected: + return False + else: + try: + if actual not in expected: + return False + except TypeError: + return False + else: + if actual != expected: + return False + return True + + +__all__ = ["matches_where"] diff --git a/src/memu/database/inmemory/repositories/recall_entry_repo.py b/src/memu/database/inmemory/repositories/recall_entry_repo.py new file mode 100644 index 0000000..e875b34 --- /dev/null +++ b/src/memu/database/inmemory/repositories/recall_entry_repo.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import uuid +from collections.abc import Mapping +from typing import Any, override + +import pendulum + +from memu.database.inmemory.repositories.filter import matches_where +from memu.database.inmemory.state import InMemoryState +from memu.database.models import EntryType, RecallEntry, compute_content_hash +from memu.database.repositories.recall_entry import RecallEntryRepo +from memu.vector import cosine_topk, cosine_topk_salience + + +class InMemoryRecallEntryRepository(RecallEntryRepo): + def __init__(self, *, state: InMemoryState, recall_entry_model: type[RecallEntry]) -> None: + self._state = state + self.recall_entry_model = recall_entry_model + self.items: dict[str, RecallEntry] = self._state.items + + def list_items(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallEntry]: + if not where: + return dict(self.items) + return {mid: item for mid, item in self.items.items() if matches_where(item, where)} + + def list_items_by_ref_ids( + self, ref_ids: list[str], where: Mapping[str, Any] | None = None + ) -> dict[str, RecallEntry]: + """List items by their ref_id in the extra column. + + Args: + ref_ids: List of ref_ids to query. + where: Additional filter conditions. + + Returns: + Dict mapping item_id -> RecallEntry for items whose extra.ref_id is in ref_ids. + """ + if not ref_ids: + return {} + ref_id_set = set(ref_ids) + result: dict[str, RecallEntry] = {} + for mid, item in self.items.items(): + # Check where filter first + if where and not matches_where(item, where): + continue + # Check if ref_id is in the requested set + item_ref_id = (item.extra or {}).get("ref_id") + if item_ref_id and item_ref_id in ref_id_set: + result[mid] = item + return result + + def clear_items(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallEntry]: + if not where: + matches = self.items.copy() + self.items.clear() + return matches + matches = {mid: item for mid, item in self.items.items() if matches_where(item, where)} + for mid in matches: + self.items.pop(mid, None) + return matches + + def _find_by_hash(self, content_hash: str, user_data: dict[str, Any]) -> RecallEntry | None: + """ + Find existing item by content hash within the same user scope. + + This enables deduplication: if the same content exists for the same user, + we reinforce it instead of creating a duplicate. + """ + for item in self.items.values(): + # Read content_hash from extra dict + item_hash = (item.extra or {}).get("content_hash") + if item_hash != content_hash: + continue + # Check scope match (user_id, agent_id, etc.) + if matches_where(item, user_data): + return item + return None + + def create_item( + self, + *, + resource_id: str, + memory_type: EntryType, + summary: str, + embedding: list[float], + user_data: dict[str, Any], + reinforce: bool = False, + tool_record: dict[str, Any] | None = None, + ) -> RecallEntry: + if reinforce and memory_type != "tool": + return self.create_item_reinforce( + resource_id=resource_id, + memory_type=memory_type, + summary=summary, + embedding=embedding, + user_data=user_data, + ) + + # Build extra dict with tool_record fields at top level + extra: dict[str, Any] = {} + if tool_record: + if tool_record.get("when_to_use") is not None: + extra["when_to_use"] = tool_record["when_to_use"] + if tool_record.get("metadata") is not None: + extra["metadata"] = tool_record["metadata"] + if tool_record.get("tool_calls") is not None: + extra["tool_calls"] = tool_record["tool_calls"] + + mid = str(uuid.uuid4()) + it = self.recall_entry_model( + id=mid, + resource_id=resource_id, + memory_type=memory_type, + summary=summary, + embedding=embedding, + extra=extra if extra else {}, + **user_data, + ) + self.items[mid] = it + return it + + def create_item_reinforce( + self, + *, + resource_id: str, + memory_type: EntryType, + summary: str, + embedding: list[float], + user_data: dict[str, Any], + reinforce: bool = False, + ) -> RecallEntry: + content_hash = compute_content_hash(summary, memory_type) + + # Check for existing item with same hash in same scope (deduplication) + existing = self._find_by_hash(content_hash, user_data) + if existing: + # Reinforce existing memory instead of creating duplicate + current_extra = existing.extra or {} + current_count = current_extra.get("reinforcement_count", 1) + existing.extra = { + **current_extra, + "reinforcement_count": current_count + 1, + "last_reinforced_at": pendulum.now("UTC").isoformat(), + } + existing.updated_at = pendulum.now("UTC") + return existing + + # Create new item with salience tracking in extra + mid = str(uuid.uuid4()) + now = pendulum.now("UTC") + item_extra = user_data.pop("extra", {}) if "extra" in user_data else {} + item_extra.update({ + "content_hash": content_hash, + "reinforcement_count": 1, + "last_reinforced_at": now.isoformat(), + }) + it = self.recall_entry_model( + id=mid, + resource_id=resource_id, + memory_type=memory_type, + summary=summary, + embedding=embedding, + extra=item_extra, + **user_data, + ) + self.items[mid] = it + return it + + def vector_search_items( + self, + query_vec: list[float], + top_k: int, + where: Mapping[str, Any] | None = None, + *, + ranking: str = "similarity", + recency_decay_days: float = 30.0, + ) -> list[tuple[str, float]]: + pool = self.list_items(where) + + if ranking == "salience": + # Salience-aware ranking: similarity x reinforcement x recency + # Read values from extra dict + corpus = [ + ( + i.id, + i.embedding, + (i.extra or {}).get("reinforcement_count", 1), + self._parse_datetime((i.extra or {}).get("last_reinforced_at")), + ) + for i in pool.values() + ] + return cosine_topk_salience(query_vec, corpus, k=top_k, recency_decay_days=recency_decay_days) + + # Default: pure cosine similarity (backward compatible) + hits = cosine_topk(query_vec, [(i.id, i.embedding) for i in pool.values()], k=top_k) + return hits + + def load_existing(self) -> None: + return None + + def get_item(self, item_id: str) -> RecallEntry | None: + return self.items.get(item_id) + + @staticmethod + def _parse_datetime(dt_str: str | None) -> pendulum.DateTime | None: + """Parse ISO datetime string from extra dict.""" + if dt_str is None: + return None + try: + parsed = pendulum.parse(dt_str) + except (ValueError, TypeError): + return None + else: + if isinstance(parsed, pendulum.DateTime): + return parsed + return None + + @override + def delete_item(self, item_id: str) -> None: + if item_id in self.items: + del self.items[item_id] + + @override + def update_item( + self, + *, + item_id: str, + memory_type: EntryType | None = None, + summary: str | None = None, + embedding: list[float] | None = None, + extra: dict[str, Any] | None = None, + tool_record: dict[str, Any] | None = None, + ) -> RecallEntry: + item = self.items.get(item_id) + if item is None: + msg = f"Item with id {item_id} not found" + raise KeyError(msg) + + if memory_type is not None: + item.memory_type = memory_type + if summary is not None: + item.summary = summary + if embedding is not None: + item.embedding = embedding + + # Merge extra and tool_record into existing extra dict + current_extra = item.extra or {} + if extra is not None: + current_extra = {**current_extra, **extra} + if tool_record is not None: + # Merge tool_record fields at top level + for key in ("when_to_use", "metadata", "tool_calls"): + if tool_record.get(key) is not None: + current_extra[key] = tool_record[key] + if extra is not None or tool_record is not None: + item.extra = current_extra + + self.items[item_id] = item + return item + + +__all__ = ["InMemoryRecallEntryRepository"] diff --git a/src/memu/database/inmemory/repositories/recall_file_entry_repo.py b/src/memu/database/inmemory/repositories/recall_file_entry_repo.py new file mode 100644 index 0000000..d96422f --- /dev/null +++ b/src/memu/database/inmemory/repositories/recall_file_entry_repo.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import uuid +from collections.abc import Mapping +from typing import Any, override + +from memu.database.inmemory.repositories.filter import matches_where +from memu.database.inmemory.state import InMemoryState +from memu.database.models import RecallFileEntry +from memu.database.repositories.recall_file_entry import RecallFileEntryRepo + + +class InMemoryFileEntryRepository(RecallFileEntryRepo): + def __init__(self, *, state: InMemoryState, recall_file_entry_model: type[RecallFileEntry]) -> None: + self._state = state + self.recall_file_entry_model = recall_file_entry_model + self.relations: list[RecallFileEntry] = self._state.relations + + def list_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileEntry]: + if not where: + return list(self.relations) + return [rel for rel in self.relations if matches_where(rel, where)] + + def link_item_category(self, item_id: str, cat_id: str, user_data: dict[str, Any]) -> RecallFileEntry: + _ = item_id # enforced by caller via existing state + for rel in self.relations: + if rel.item_id == item_id and rel.category_id == cat_id: + return rel + rel = self.recall_file_entry_model(id=str(uuid.uuid4()), item_id=item_id, category_id=cat_id, **user_data) + self.relations.append(rel) + return rel + + def load_existing(self) -> None: + return None + + @override + def get_item_categories(self, item_id: str) -> list[RecallFileEntry]: + return [rel for rel in self.relations if rel.item_id == item_id] + + @override + def unlink_item_category(self, item_id: str, cat_id: str) -> None: + # Mutate the shared state list in place so the DatabaseState reference and + # this repo's view never diverge (rebinding self.relations would orphan the + # shared state.relations list). + self.relations[:] = [ + rel for rel in self.relations if not (rel.item_id == item_id and rel.category_id == cat_id) + ] + + def unlink_item(self, item_id: str) -> list[RecallFileEntry]: + removed = [rel for rel in self.relations if rel.item_id == item_id] + self.relations[:] = [rel for rel in self.relations if rel.item_id != item_id] + return removed + + def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileEntry]: + if not where: + removed = list(self.relations) + self.relations.clear() + return removed + removed = [rel for rel in self.relations if matches_where(rel, where)] + removed_ids = {rel.id for rel in removed} + self.relations[:] = [rel for rel in self.relations if rel.id not in removed_ids] + return removed + + +__all__ = ["InMemoryFileEntryRepository"] diff --git a/src/memu/database/inmemory/repositories/recall_file_repo.py b/src/memu/database/inmemory/repositories/recall_file_repo.py new file mode 100644 index 0000000..a9bb78a --- /dev/null +++ b/src/memu/database/inmemory/repositories/recall_file_repo.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import uuid +from collections.abc import Mapping +from typing import Any + +import pendulum + +from memu.database.inmemory.repositories.filter import matches_where +from memu.database.inmemory.state import InMemoryState +from memu.database.models import RecallFile +from memu.database.repositories.recall_file import RecallFileRepo as RecallFileRepoProtocol + + +class InMemoryRecallFileRepository(RecallFileRepoProtocol): + def __init__(self, *, state: InMemoryState, recall_file_model: type[RecallFile]) -> None: + self._state = state + self.recall_file_model = recall_file_model + self.categories: dict[str, RecallFile] = self._state.categories + + def list_categories(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallFile]: + if not where: + return dict(self.categories) + return {cid: cat for cid, cat in self.categories.items() if matches_where(cat, where)} + + def clear_categories(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallFile]: + if not where: + matches = self.categories.copy() + self.categories.clear() + return matches + matches = {cid: cat for cid, cat in self.categories.items() if matches_where(cat, where)} + for cid in matches: + self.categories.pop(cid, None) + return matches + + def get_or_create_category( + self, + *, + name: str, + description: str, + embedding: list[float], + user_data: dict[str, Any], + track: str = "memory", + ) -> RecallFile: + for c in self.categories.values(): + if c.name == name and c.track == track and all(getattr(c, k) == v for k, v in user_data.items()): + now = pendulum.now("UTC") + if c.embedding is None: + c.embedding = embedding + c.updated_at = now + if not c.description: + c.description = description + c.updated_at = now + return c + cid = str(uuid.uuid4()) + cat = self.recall_file_model( + id=cid, name=name, description=description, embedding=embedding, track=track, **user_data + ) + self.categories[cid] = cat + return cat + + def update_category( + self, + *, + category_id: str, + name: str | None = None, + description: str | None = None, + embedding: list[float] | None = None, + content: str | None = None, + ) -> RecallFile: + cat = self.categories.get(category_id) + if cat is None: + msg = f"Category with id {category_id} not found" + raise KeyError(msg) + + if name is not None: + cat.name = name + if description is not None: + cat.description = description + if embedding is not None: + cat.embedding = embedding + if content is not None: + cat.content = content + + cat.updated_at = pendulum.now("UTC") + return cat + + def load_existing(self) -> None: + return None + + +RecallFileRepo = InMemoryRecallFileRepository + +__all__ = ["InMemoryRecallFileRepository", "RecallFileRepo"] diff --git a/src/memu/database/inmemory/repositories/recall_file_resource_repo.py b/src/memu/database/inmemory/repositories/recall_file_resource_repo.py new file mode 100644 index 0000000..909c685 --- /dev/null +++ b/src/memu/database/inmemory/repositories/recall_file_resource_repo.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import uuid +from collections.abc import Mapping +from typing import Any, override + +from memu.database.inmemory.repositories.filter import matches_where +from memu.database.inmemory.state import InMemoryState +from memu.database.models import RecallFileResource +from memu.database.repositories.recall_file_resource import RecallFileResourceRepo + + +class InMemoryFileResourceRepository(RecallFileResourceRepo): + def __init__(self, *, state: InMemoryState, recall_file_resource_model: type[RecallFileResource]) -> None: + self._state = state + self.recall_file_resource_model = recall_file_resource_model + self.relations: list[RecallFileResource] = self._state.resource_relations + + def list_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileResource]: + if not where: + return list(self.relations) + return [rel for rel in self.relations if matches_where(rel, where)] + + def link_resource_category(self, resource_id: str, cat_id: str, user_data: dict[str, Any]) -> RecallFileResource: + _ = resource_id # enforced by caller via existing state + for rel in self.relations: + if rel.resource_id == resource_id and rel.file_id == cat_id: + return rel + rel = self.recall_file_resource_model( + id=str(uuid.uuid4()), resource_id=resource_id, file_id=cat_id, **user_data + ) + self.relations.append(rel) + return rel + + def load_existing(self) -> None: + return None + + @override + def get_resource_categories(self, resource_id: str) -> list[RecallFileResource]: + return [rel for rel in self.relations if rel.resource_id == resource_id] + + @override + def unlink_resource_category(self, resource_id: str, cat_id: str) -> None: + # Mutate the shared state list in place so the DatabaseState reference and + # this repo's view never diverge (rebinding self.relations would orphan the + # shared state.resource_relations list). + self.relations[:] = [ + rel for rel in self.relations if not (rel.resource_id == resource_id and rel.file_id == cat_id) + ] + + def unlink_resource(self, resource_id: str) -> list[RecallFileResource]: + removed = [rel for rel in self.relations if rel.resource_id == resource_id] + self.relations[:] = [rel for rel in self.relations if rel.resource_id != resource_id] + return removed + + def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileResource]: + if not where: + removed = list(self.relations) + self.relations.clear() + return removed + removed = [rel for rel in self.relations if matches_where(rel, where)] + removed_ids = {rel.id for rel in removed} + self.relations[:] = [rel for rel in self.relations if rel.id not in removed_ids] + return removed + + +__all__ = ["InMemoryFileResourceRepository"] diff --git a/src/memu/database/inmemory/repositories/recall_file_segment_repo.py b/src/memu/database/inmemory/repositories/recall_file_segment_repo.py new file mode 100644 index 0000000..2eb238f --- /dev/null +++ b/src/memu/database/inmemory/repositories/recall_file_segment_repo.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import uuid +from collections.abc import Mapping +from typing import Any + +from memu.database.inmemory.repositories.filter import matches_where +from memu.database.inmemory.state import InMemoryState +from memu.database.models import RecallFileSegment +from memu.database.repositories.recall_file_segment import RecallFileSegmentRepo + + +class InMemoryFileSegmentRepository(RecallFileSegmentRepo): + def __init__(self, *, state: InMemoryState, recall_file_segment_model: type[RecallFileSegment]) -> None: + self._state = state + self.recall_file_segment_model = recall_file_segment_model + self.segments: list[RecallFileSegment] = self._state.segments + + def list_segments(self, where: Mapping[str, Any] | None = None) -> list[RecallFileSegment]: + if not where: + return list(self.segments) + return [seg for seg in self.segments if matches_where(seg, where)] + + def list_segments_for_file(self, recall_file_id: str) -> list[RecallFileSegment]: + return [seg for seg in self.segments if seg.recall_file_id == recall_file_id] + + def create_segment( + self, + *, + recall_file_id: str, + text: str, + embedding: list[float] | None, + user_data: dict[str, Any], + track: str = "memory", + ) -> RecallFileSegment: + seg = self.recall_file_segment_model( + id=str(uuid.uuid4()), + recall_file_id=recall_file_id, + track=track, + text=text, + embedding=embedding, + **user_data, + ) + self.segments.append(seg) + return seg + + def delete_segment(self, segment_id: str) -> None: + # Mutate the shared state list in place so the DatabaseState reference and this + # repo's view never diverge. + self.segments[:] = [seg for seg in self.segments if seg.id != segment_id] + + def delete_segments_for_file(self, recall_file_id: str) -> list[RecallFileSegment]: + removed = [seg for seg in self.segments if seg.recall_file_id == recall_file_id] + self.segments[:] = [seg for seg in self.segments if seg.recall_file_id != recall_file_id] + return removed + + def clear_segments(self, where: Mapping[str, Any] | None = None) -> list[RecallFileSegment]: + if not where: + removed = list(self.segments) + self.segments.clear() + return removed + removed = [seg for seg in self.segments if matches_where(seg, where)] + removed_ids = {seg.id for seg in removed} + self.segments[:] = [seg for seg in self.segments if seg.id not in removed_ids] + return removed + + def load_existing(self) -> None: + return None + + +__all__ = ["InMemoryFileSegmentRepository"] diff --git a/src/memu/database/inmemory/repositories/resource_repo.py b/src/memu/database/inmemory/repositories/resource_repo.py new file mode 100644 index 0000000..6116ec2 --- /dev/null +++ b/src/memu/database/inmemory/repositories/resource_repo.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import uuid +from collections.abc import Mapping +from typing import Any + +from memu.database.inmemory.repositories.filter import matches_where +from memu.database.inmemory.state import InMemoryState +from memu.database.models import Resource +from memu.database.repositories.resource import ResourceRepo as ResourceRepoProtocol +from memu.vector import cosine_topk + + +class InMemoryResourceRepository(ResourceRepoProtocol): + def __init__(self, *, state: InMemoryState, resource_model: type[Resource]) -> None: + self._state = state + self.resource_model = resource_model + self.resources: dict[str, Resource] = self._state.resources + + def list_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, Resource]: + if not where: + return dict(self.resources) + return {rid: res for rid, res in self.resources.items() if matches_where(res, where)} + + def clear_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, Resource]: + if not where: + matches = self.resources.copy() + self.resources.clear() + return matches + matches = {rid: res for rid, res in self.resources.items() if matches_where(res, where)} + for rid in matches: + self.resources.pop(rid, None) + return matches + + def delete_resource(self, resource_id: str) -> None: + self.resources.pop(resource_id, None) + + def create_resource( + self, + *, + url: str, + modality: str, + local_path: str, + caption: str | None, + embedding: list[float] | None, + user_data: dict[str, Any], + track: str | None = None, + ) -> Resource: + rid = str(uuid.uuid4()) + res = self.resource_model( + id=rid, + url=url, + modality=modality, + local_path=local_path, + caption=caption, + embedding=embedding, + track=track, + **user_data, + ) + self.resources[rid] = res + return res + + def vector_search_resources( + self, + query_vec: list[float], + top_k: int, + where: Mapping[str, Any] | None = None, + ) -> list[tuple[str, float]]: + pool = self.list_resources(where) + corpus = [(rid, res.embedding) for rid, res in pool.items() if res.embedding] + return cosine_topk(query_vec, corpus, k=top_k) + + def load_existing(self) -> None: + return None + + +ResourceRepo = InMemoryResourceRepository + +__all__ = ["InMemoryResourceRepository", "ResourceRepo"] diff --git a/src/memu/database/inmemory/state.py b/src/memu/database/inmemory/state.py new file mode 100644 index 0000000..b9dfcf0 --- /dev/null +++ b/src/memu/database/inmemory/state.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from memu.database.state import DatabaseState + +InMemoryState = DatabaseState + +__all__ = ["DatabaseState", "InMemoryState"] diff --git a/src/memu/database/inmemory/vector.py b/src/memu/database/inmemory/vector.py new file mode 100644 index 0000000..441ddb3 --- /dev/null +++ b/src/memu/database/inmemory/vector.py @@ -0,0 +1,13 @@ +"""Backward-compatible re-export of the storage-neutral vector helpers. + +The actual cosine/salience math now lives in :mod:`memu.vector` so that neither +the app retrieval layer nor any database backend has to depend on the in-memory +backend's internals. This shim is kept so existing imports +(``memu.database.inmemory.vector``) keep working. +""" + +from __future__ import annotations + +from memu.vector import _cosine, cosine_topk, cosine_topk_salience, salience_score + +__all__ = ["_cosine", "cosine_topk", "cosine_topk_salience", "salience_score"] diff --git a/src/memu/database/interfaces.py b/src/memu/database/interfaces.py new file mode 100644 index 0000000..1ebb531 --- /dev/null +++ b/src/memu/database/interfaces.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from memu.database.models import RecallEntry as RecallEntryRecord +from memu.database.models import RecallFile as RecallFileRecord +from memu.database.models import RecallFileEntry as RecallFileEntryRecord +from memu.database.models import RecallFileResource as RecallFileResourceRecord +from memu.database.models import RecallFileSegment as RecallFileSegmentRecord +from memu.database.models import Resource as ResourceRecord +from memu.database.repositories import ( + RecallEntryRepo, + RecallFileEntryRepo, + RecallFileRepo, + RecallFileResourceRepo, + RecallFileSegmentRepo, + ResourceRepo, +) + + +@runtime_checkable +class Database(Protocol): + """Backend-agnostic database contract.""" + + resource_repo: ResourceRepo + recall_file_repo: RecallFileRepo + recall_entry_repo: RecallEntryRepo + recall_file_entry_repo: RecallFileEntryRepo + recall_file_resource_repo: RecallFileResourceRepo + recall_file_segment_repo: RecallFileSegmentRepo + + resources: dict[str, ResourceRecord] + items: dict[str, RecallEntryRecord] + categories: dict[str, RecallFileRecord] + relations: list[RecallFileEntryRecord] + resource_relations: list[RecallFileResourceRecord] + segments: list[RecallFileSegmentRecord] + + def close(self) -> None: ... + + +__all__ = [ + "Database", + "RecallEntryRecord", + "RecallFileEntryRecord", + "RecallFileRecord", + "RecallFileResourceRecord", + "RecallFileSegmentRecord", + "ResourceRecord", +] diff --git a/src/memu/database/models.py b/src/memu/database/models.py new file mode 100644 index 0000000..94f7970 --- /dev/null +++ b/src/memu/database/models.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import hashlib +import json +import uuid +from datetime import datetime +from typing import Any, Literal + +import pendulum +from pydantic import BaseModel, ConfigDict, Field + +EntryType = Literal["profile", "event", "knowledge", "behavior", "skill", "tool"] + + +def compute_content_hash(summary: str, memory_type: str) -> str: + """ + Generate unique hash for memory deduplication. + + Operates on post-summary content. Normalizes whitespace to handle + minor formatting differences like "I love coffee" vs "I love coffee". + + Args: + summary: The memory summary text + memory_type: The type of memory (profile, event, etc.) + + Returns: + A 16-character hex hash string + """ + # Normalize: lowercase, strip, collapse whitespace + normalized = " ".join(summary.lower().split()) + content = f"{memory_type}:{normalized}" + return hashlib.sha256(content.encode()).hexdigest()[:16] + + +class BaseRecord(BaseModel): + """Backend-agnostic record interface.""" + + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + created_at: datetime = Field(default_factory=lambda: pendulum.now("UTC")) + updated_at: datetime = Field(default_factory=lambda: pendulum.now("UTC")) + + +class ToolCallResult(BaseModel): + """Represents the result of a tool invocation for Tool Memory.""" + + tool_name: str = Field(..., description="Name of the tool that was called") + input: dict[str, Any] | str = Field(default="", description="Tool input parameters") + output: str = Field(default="", description="Tool output result") + success: bool = Field(default=True, description="Whether the tool invocation succeeded") + time_cost: float = Field(default=0.0, description="Time consumed by the tool invocation in seconds") + token_cost: int = Field(default=-1, description="Token consumption of the tool (-1 if unknown)") + score: float = Field(default=0.0, description="Quality score from 0.0 to 1.0") + call_hash: str = Field(default="", description="Hash of input+output for deduplication") + created_at: datetime = Field(default_factory=lambda: pendulum.now("UTC")) + + def generate_hash(self) -> str: + """Generate MD5 hash from tool input and output for deduplication.""" + input_str = json.dumps(self.input, sort_keys=True) if isinstance(self.input, dict) else str(self.input) + combined = f"{self.tool_name}|{input_str}|{self.output}" + return hashlib.md5(combined.encode("utf-8"), usedforsecurity=False).hexdigest() + + def ensure_hash(self) -> None: + """Ensure call_hash is set, generate if empty.""" + if not self.call_hash: + self.call_hash = self.generate_hash() + + +class Resource(BaseRecord): + url: str + modality: str + local_path: str + caption: str | None = None + embedding: list[float] | None = None + # Which workspace track this resource came from: "chat", "skill", or + # "workspace" (set by ``memorize_workspace``). None for legacy ``memorize``. + track: str | None = None + + +class RecallEntry(BaseRecord): + resource_id: str | None + memory_type: str + summary: str + embedding: list[float] | None = None + happened_at: datetime | None = None + extra: dict[str, Any] = {} + # extra may contain: + # # reinforcement tracking fields + # - content_hash: str + # - reinforcement_count: int + # - last_reinforced_at: str (isoformat) + # # Reference tracking field + # - ref_id: str + # # Tool memory fields + # - when_to_use: str - Hint for when this memory should be retrieved + # - metadata: dict - Type-specific metadata (e.g., tool_name, avg_success_rate) + # - tool_calls: list[dict] - Tool call history for tool memories (serialized ToolCallResult) + + +class RecallFile(BaseRecord): + name: str + # Which track this file belongs to: "memory" (a memory category) or "skill" + # (a synthesized skill). Defaults to "memory" so existing rows backfill correctly. + track: str = "memory" + description: str + embedding: list[float] | None = None + content: str | None = None + + +class RecallFileEntry(BaseRecord): + item_id: str + category_id: str + + +class RecallFileResource(BaseRecord): + resource_id: str + file_id: str + + +class RecallFileSegment(BaseRecord): + """A searchable slice (L2 item) of a ``RecallFile`` (ADR 0007). + + Each file has 1..n segments; ``text`` is the embed/search unit and ``embedding`` + its vector. Retrieval ranks segments and rolls the top hits up to their file via + ``recall_file_id``. Segments carry no ordinal: how a file is sliced is track-specific + and not necessarily sequential, so position would not be informative. + + ``track`` mirrors the owning file's track ("memory"/"skill"), denormalized here so + retrieval can filter segments by track with a plain column predicate instead of a + join. It is immutable for a segment's lifetime (segments are drop-and-recreated when + a file is re-sliced), so it never drifts from the file. + """ + + recall_file_id: str + track: str = "memory" + text: str + embedding: list[float] | None = None + + +def merge_scope_model[TBaseRecord: BaseRecord]( + user_model: type[BaseModel], core_model: type[TBaseRecord], *, name_suffix: str +) -> type[TBaseRecord]: + """Create a scoped model inheriting both the user scope model and the core model.""" + overlap = set(user_model.model_fields) & set(core_model.model_fields) + if overlap: + msg = f"Scope fields conflict with core model fields: {sorted(overlap)}" + raise TypeError(msg) + + return type( + f"{user_model.__name__}{core_model.__name__}{name_suffix}", + (user_model, core_model), + {"model_config": ConfigDict(extra="allow")}, + ) + + +def build_scoped_models( + user_model: type[BaseModel], +) -> tuple[ + type[Resource], + type[RecallFile], + type[RecallEntry], + type[RecallFileEntry], + type[RecallFileResource], + type[RecallFileSegment], +]: + """ + Build scoped interface models (Pydantic) that inherit from the base record models and user scope. + """ + resource_model = merge_scope_model(user_model, Resource, name_suffix="Resource") + recall_file_model = merge_scope_model(user_model, RecallFile, name_suffix="RecallFile") + recall_entry_model = merge_scope_model(user_model, RecallEntry, name_suffix="RecallEntry") + recall_file_entry_model = merge_scope_model(user_model, RecallFileEntry, name_suffix="RecallFileEntry") + recall_file_resource_model = merge_scope_model(user_model, RecallFileResource, name_suffix="RecallFileResource") + recall_file_segment_model = merge_scope_model(user_model, RecallFileSegment, name_suffix="RecallFileSegment") + return ( + resource_model, + recall_file_model, + recall_entry_model, + recall_file_entry_model, + recall_file_resource_model, + recall_file_segment_model, + ) + + +__all__ = [ + "BaseRecord", + "EntryType", + "RecallEntry", + "RecallFile", + "RecallFileEntry", + "RecallFileResource", + "RecallFileSegment", + "Resource", + "ToolCallResult", + "build_scoped_models", + "compute_content_hash", + "merge_scope_model", +] diff --git a/src/memu/database/postgres/__init__.py b/src/memu/database/postgres/__init__.py new file mode 100644 index 0000000..c3e3381 --- /dev/null +++ b/src/memu/database/postgres/__init__.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from pydantic import BaseModel + +from memu.app.settings import DatabaseConfig +from memu.database.postgres.postgres import PostgresStore +from memu.database.postgres.schema import SQLAModels, get_sqlalchemy_models + + +def build_postgres_database( + *, + config: DatabaseConfig, + user_model: type[BaseModel], +) -> PostgresStore: + dsn = config.metadata_store.dsn + if not dsn: + msg = "Postgres metadata_store requires a DSN" + raise ValueError(msg) + + vector_provider = config.vector_index.provider if config.vector_index else None + sqla_models: SQLAModels = get_sqlalchemy_models(scope_model=user_model) + + return PostgresStore( + dsn=dsn, + ddl_mode=config.metadata_store.ddl_mode, + vector_provider=vector_provider, + scope_model=user_model, + resource_model=sqla_models.Resource, + recall_file_model=sqla_models.RecallFile, + recall_entry_model=sqla_models.RecallEntry, + recall_file_entry_model=sqla_models.RecallFileEntry, + sqla_models=sqla_models, + ) + + +__all__ = ["build_postgres_database"] diff --git a/src/memu/database/postgres/migration.py b/src/memu/database/postgres/migration.py new file mode 100644 index 0000000..e8a5f33 --- /dev/null +++ b/src/memu/database/postgres/migration.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import logging +import re +from pathlib import Path +from typing import Any, Literal + +from sqlalchemy import create_engine, inspect + +try: # Optional dependency for Postgres backend + from alembic import command + from alembic.config import Config as AlembicConfig +except ImportError as exc: # pragma: no cover - optional dependency + msg = "alembic is required for Postgres migrations" + raise ImportError(msg) from exc + +logger = logging.getLogger(__name__) + +DDLMode = Literal["create", "validate"] +_UNESCAPED_CONFIGPARSER_PERCENT = re.compile(r"(? str: + return _UNESCAPED_CONFIGPARSER_PERCENT.sub("%%", value) + + +def make_alembic_config(*, dsn: str, scope_model: type[Any]) -> AlembicConfig: + cfg = AlembicConfig() + cfg.set_main_option("script_location", str(Path(__file__).with_name("migrations"))) + cfg.set_main_option("sqlalchemy.url", _escape_for_config_parser(dsn)) + cfg.attributes["scope_model"] = scope_model + return cfg + + +def run_migrations(*, dsn: str, scope_model: type[Any], ddl_mode: DDLMode = "create") -> None: + """ + Run database migrations based on the ddl_mode setting. + + Args: + dsn: Database connection string + scope_model: User scope model for scoped tables + ddl_mode: "create" to apply migrations up to head, "validate" to only check schema + + Alembic is the source of truth for schema: "create" runs ``upgrade head`` + rather than ``metadata.create_all`` so that a fresh database is built from + the committed revisions. The pgvector extension is enabled by the initial + revision, so no separate bootstrap step is required here. + """ + from memu.database.postgres.schema import get_metadata + + if ddl_mode == "create": + cfg = make_alembic_config(dsn=dsn, scope_model=scope_model) + command.upgrade(cfg, "head") + logger.info("Database migrated to head") + elif ddl_mode == "validate": + metadata = get_metadata(scope_model) + engine = create_engine(dsn) + try: + inspector = inspect(engine) + existing_tables = set(inspector.get_table_names()) + expected_tables = set(metadata.tables.keys()) + missing_tables = expected_tables - existing_tables + + if missing_tables: + msg = f"Database schema validation failed. Missing tables: {sorted(missing_tables)}" + raise RuntimeError(msg) + logger.info("Database schema validated successfully") + finally: + engine.dispose() + + +__all__ = ["DDLMode", "make_alembic_config", "run_migrations"] diff --git a/src/memu/database/postgres/migrations/__init__.py b/src/memu/database/postgres/migrations/__init__.py new file mode 100644 index 0000000..c82052e --- /dev/null +++ b/src/memu/database/postgres/migrations/__init__.py @@ -0,0 +1 @@ +# Alembic migrations package for memu Postgres storage. diff --git a/src/memu/database/postgres/migrations/env.py b/src/memu/database/postgres/migrations/env.py new file mode 100644 index 0000000..621dcd6 --- /dev/null +++ b/src/memu/database/postgres/migrations/env.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from logging.config import fileConfig +from typing import Any, Literal + +from alembic import context +from alembic.autogenerate.api import AutogenContext +from alembic.runtime.environment import NameFilterParentNames, NameFilterType +from sqlalchemy import MetaData, engine_from_config, pool + +from memu.database.postgres.schema import get_metadata + +config = context.config + +if config.config_file_name is not None: # pragma: no cover - alembic bootstrap + fileConfig(config.config_file_name) + + +def get_target_metadata() -> MetaData | None: + scope_model = config.attributes.get("scope_model") + return get_metadata(scope_model) + + +target_metadata: MetaData | None = get_target_metadata() + + +def include_name(name: str | None, type_: NameFilterType, parent_names: NameFilterParentNames) -> bool: + """Only manage tables declared in our metadata. + + Keeps autogenerate from emitting drops for unrelated tables when the + reflection target happens to share a schema with other applications. + """ + if type_ == "table" and target_metadata is not None: + return name in target_metadata.tables + return True + + +def render_item(type_: str, obj: Any, autogen_context: AutogenContext) -> str | Literal[False]: + """Keep generated revisions self-contained (no app-model imports).""" + if type_ == "type": + module = obj.__class__.__module__ + if module.startswith("pgvector."): + autogen_context.imports.add("import pgvector.sqlalchemy") + return f"pgvector.sqlalchemy.{obj!r}" + # TZDateTime is just a timezone-aware DateTime; render it as such so + # the migration does not have to import memu app modules. + if obj.__class__.__name__ == "TZDateTime": + return "sa.DateTime(timezone=True)" + # SQLModel's AutoString (used for scope str columns) is a plain VARCHAR; + # render it as sa.String() for parity with the other string columns and + # to avoid an extra sqlmodel import in the migration. + if module.startswith("sqlmodel.") and obj.__class__.__name__ == "AutoString": + return "sa.String()" + return False + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + include_name=include_name, + render_item=render_item, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + configuration = {"sqlalchemy.url": config.get_main_option("sqlalchemy.url")} + connectable = engine_from_config( + configuration, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + include_name=include_name, + render_item=render_item, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/src/memu/database/postgres/migrations/script.py.mako b/src/memu/database/postgres/migrations/script.py.mako new file mode 100644 index 0000000..8fb2b5f --- /dev/null +++ b/src/memu/database/postgres/migrations/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: str | Sequence[str] | None = ${repr(down_revision)} +branch_labels: str | Sequence[str] | None = ${repr(branch_labels)} +depends_on: str | Sequence[str] | None = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/src/memu/database/postgres/migrations/versions/20260703_0001_initial_schema.py b/src/memu/database/postgres/migrations/versions/20260703_0001_initial_schema.py new file mode 100644 index 0000000..74d968c --- /dev/null +++ b/src/memu/database/postgres/migrations/versions/20260703_0001_initial_schema.py @@ -0,0 +1,161 @@ +"""initial schema + +Revision ID: 20260703_0001 +Revises: +Create Date: 2026-07-03 19:47:40.690785 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import pgvector.sqlalchemy +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "20260703_0001" +down_revision: str | Sequence[str] | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # pgvector must exist before any VECTOR column is created. + op.execute("CREATE EXTENSION IF NOT EXISTS vector") + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "recall_files", + sa.Column("id", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("track", sa.String(), server_default="memory", nullable=False), + sa.Column("description", sa.Text(), nullable=False), + sa.Column("embedding", pgvector.sqlalchemy.VECTOR(), nullable=True), + sa.Column("content", sa.Text(), nullable=True), + sa.Column("user_id", sa.String(), nullable=True), + sa.Column("agent_id", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_recall_files__scope", "recall_files", ["user_id", "agent_id"], unique=False) + op.create_index(op.f("ix_recall_files_id"), "recall_files", ["id"], unique=False) + op.create_index(op.f("ix_recall_files_name"), "recall_files", ["name"], unique=False) + op.create_table( + "resources", + sa.Column("id", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("url", sa.String(), nullable=False), + sa.Column("modality", sa.String(), nullable=False), + sa.Column("local_path", sa.String(), nullable=False), + sa.Column("caption", sa.Text(), nullable=True), + sa.Column("embedding", pgvector.sqlalchemy.VECTOR(), nullable=True), + sa.Column("track", sa.String(), nullable=True), + sa.Column("user_id", sa.String(), nullable=True), + sa.Column("agent_id", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_resources__scope", "resources", ["user_id", "agent_id"], unique=False) + op.create_index(op.f("ix_resources_id"), "resources", ["id"], unique=False) + op.create_table( + "recall_entries", + sa.Column("id", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("resource_id", sa.String(), nullable=True), + sa.Column("memory_type", sa.String(), nullable=False), + sa.Column("summary", sa.Text(), nullable=False), + sa.Column("embedding", pgvector.sqlalchemy.VECTOR(), nullable=True), + sa.Column("happened_at", sa.DateTime(), nullable=True), + sa.Column("extra", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("user_id", sa.String(), nullable=True), + sa.Column("agent_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint(["resource_id"], ["resources.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_recall_entries__scope", "recall_entries", ["user_id", "agent_id"], unique=False) + op.create_index(op.f("ix_recall_entries_id"), "recall_entries", ["id"], unique=False) + op.create_table( + "recall_file_resources", + sa.Column("id", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("resource_id", sa.String(), nullable=False), + sa.Column("file_id", sa.String(), nullable=False), + sa.Column("user_id", sa.String(), nullable=True), + sa.Column("agent_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint(["file_id"], ["recall_files.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["resource_id"], ["resources.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "idx_recall_file_resources_unique", "recall_file_resources", ["resource_id", "file_id"], unique=True + ) + op.create_index("ix_recall_file_resources__scope", "recall_file_resources", ["user_id", "agent_id"], unique=False) + op.create_index(op.f("ix_recall_file_resources_id"), "recall_file_resources", ["id"], unique=False) + op.create_table( + "recall_file_segments", + sa.Column("id", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("recall_file_id", sa.String(), nullable=False), + sa.Column("track", sa.String(), server_default="memory", nullable=False), + sa.Column("text", sa.Text(), nullable=False), + sa.Column("embedding", pgvector.sqlalchemy.VECTOR(), nullable=True), + sa.Column("user_id", sa.String(), nullable=True), + sa.Column("agent_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint(["recall_file_id"], ["recall_files.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_recall_file_segments__scope", "recall_file_segments", ["user_id", "agent_id"], unique=False) + op.create_index(op.f("ix_recall_file_segments_id"), "recall_file_segments", ["id"], unique=False) + op.create_index( + op.f("ix_recall_file_segments_recall_file_id"), "recall_file_segments", ["recall_file_id"], unique=False + ) + op.create_table( + "recall_file_entries", + sa.Column("id", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("item_id", sa.String(), nullable=False), + sa.Column("category_id", sa.String(), nullable=False), + sa.Column("user_id", sa.String(), nullable=True), + sa.Column("agent_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint(["category_id"], ["recall_files.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["item_id"], ["recall_entries.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("idx_recall_file_entries_unique", "recall_file_entries", ["item_id", "category_id"], unique=True) + op.create_index("ix_recall_file_entries__scope", "recall_file_entries", ["user_id", "agent_id"], unique=False) + op.create_index(op.f("ix_recall_file_entries_id"), "recall_file_entries", ["id"], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_recall_file_entries_id"), table_name="recall_file_entries") + op.drop_index("ix_recall_file_entries__scope", table_name="recall_file_entries") + op.drop_index("idx_recall_file_entries_unique", table_name="recall_file_entries") + op.drop_table("recall_file_entries") + op.drop_index(op.f("ix_recall_file_segments_recall_file_id"), table_name="recall_file_segments") + op.drop_index(op.f("ix_recall_file_segments_id"), table_name="recall_file_segments") + op.drop_index("ix_recall_file_segments__scope", table_name="recall_file_segments") + op.drop_table("recall_file_segments") + op.drop_index(op.f("ix_recall_file_resources_id"), table_name="recall_file_resources") + op.drop_index("ix_recall_file_resources__scope", table_name="recall_file_resources") + op.drop_index("idx_recall_file_resources_unique", table_name="recall_file_resources") + op.drop_table("recall_file_resources") + op.drop_index(op.f("ix_recall_entries_id"), table_name="recall_entries") + op.drop_index("ix_recall_entries__scope", table_name="recall_entries") + op.drop_table("recall_entries") + op.drop_index(op.f("ix_resources_id"), table_name="resources") + op.drop_index("ix_resources__scope", table_name="resources") + op.drop_table("resources") + op.drop_index(op.f("ix_recall_files_name"), table_name="recall_files") + op.drop_index(op.f("ix_recall_files_id"), table_name="recall_files") + op.drop_index("ix_recall_files__scope", table_name="recall_files") + op.drop_table("recall_files") + # ### end Alembic commands ### diff --git a/src/memu/database/postgres/models.py b/src/memu/database/postgres/models.py new file mode 100644 index 0000000..57f9b96 --- /dev/null +++ b/src/memu/database/postgres/models.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +import pendulum + +try: + from pgvector.sqlalchemy import VECTOR as Vector +except ImportError as exc: + msg = "pgvector is required for Postgres vector support" + raise ImportError(msg) from exc + +from pydantic import BaseModel +from sqlalchemy import ForeignKey, MetaData, String, Text +from sqlalchemy.dialects.postgresql import JSONB +from sqlmodel import Column, DateTime, Field, Index, SQLModel, func + +from memu.database.models import ( + EntryType, + RecallEntry, + RecallFile, + RecallFileEntry, + RecallFileResource, + RecallFileSegment, + Resource, +) + + +class TZDateTime(DateTime): + def __init__(self, timezone: bool = True, **kw: Any) -> None: + super().__init__(timezone=timezone, **kw) + + +class BaseModelMixin(SQLModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + primary_key=True, + index=True, + sa_type=String, + ) + created_at: datetime = Field( + default_factory=lambda: pendulum.now("UTC"), + sa_type=TZDateTime, + sa_column_kwargs={"server_default": func.now()}, + ) + updated_at: datetime = Field( + default_factory=lambda: pendulum.now("UTC"), + sa_type=TZDateTime, + ) + + +class ResourceModel(BaseModelMixin, Resource): + url: str = Field(sa_column=Column(String, nullable=False)) + modality: str = Field(sa_column=Column(String, nullable=False)) + local_path: str = Field(sa_column=Column(String, nullable=False)) + caption: str | None = Field(default=None, sa_column=Column(Text, nullable=True)) + embedding: list[float] | None = Field(default=None, sa_column=Column(Vector(), nullable=True)) + track: str | None = Field(default=None, sa_column=Column(String, nullable=True)) + + +class RecallEntryModel(BaseModelMixin, RecallEntry): + resource_id: str | None = Field(sa_column=Column(ForeignKey("resources.id", ondelete="CASCADE"), nullable=True)) + memory_type: EntryType = Field(sa_column=Column(String, nullable=False)) + summary: str = Field(sa_column=Column(Text, nullable=False)) + embedding: list[float] | None = Field(default=None, sa_column=Column(Vector(), nullable=True)) + happened_at: datetime | None = Field(default=None, sa_column=Column(DateTime, nullable=True)) + extra: dict[str, Any] = Field(default={}, sa_column=Column(JSONB, nullable=True)) + + +class RecallFileModel(BaseModelMixin, RecallFile): + name: str = Field(sa_column=Column(String, nullable=False, index=True)) + description: str = Field(sa_column=Column(Text, nullable=False)) + embedding: list[float] | None = Field(default=None, sa_column=Column(Vector(), nullable=True)) + content: str | None = Field(default=None, sa_column=Column(Text, nullable=True)) + track: str = Field(default="memory", sa_column=Column(String, nullable=False, server_default="memory")) + + +class RecallFileEntryModel(BaseModelMixin, RecallFileEntry): + item_id: str = Field(sa_column=Column(ForeignKey("recall_entries.id", ondelete="CASCADE"), nullable=False)) + category_id: str = Field(sa_column=Column(ForeignKey("recall_files.id", ondelete="CASCADE"), nullable=False)) + + __table_args__ = (Index("idx_recall_file_entries_unique", "item_id", "category_id", unique=True),) + + +class RecallFileResourceModel(BaseModelMixin, RecallFileResource): + resource_id: str = Field(sa_column=Column(ForeignKey("resources.id", ondelete="CASCADE"), nullable=False)) + file_id: str = Field(sa_column=Column(ForeignKey("recall_files.id", ondelete="CASCADE"), nullable=False)) + + __table_args__ = (Index("idx_recall_file_resources_unique", "resource_id", "file_id", unique=True),) + + +class RecallFileSegmentModel(BaseModelMixin, RecallFileSegment): + recall_file_id: str = Field( + sa_column=Column(ForeignKey("recall_files.id", ondelete="CASCADE"), nullable=False, index=True) + ) + track: str = Field(default="memory", sa_column=Column(String, nullable=False, server_default="memory")) + text: str = Field(sa_column=Column(Text, nullable=False)) + embedding: list[float] | None = Field(default=None, sa_column=Column(Vector(), nullable=True)) + + +def _normalize_table_args(table_args: Any) -> tuple[list[Any], dict[str, Any]]: + if table_args is None: + return [], {} + if isinstance(table_args, dict): + return [], dict(table_args) + if not isinstance(table_args, tuple): + return [table_args], {} + + args = list(table_args) + kwargs: dict[str, Any] = {} + if args and isinstance(args[-1], dict): + kwargs = dict(args.pop()) + return args, kwargs + + +def _merge_models( + user_model: type[BaseModel], + core_model: type[SQLModel], + *, + name_suffix: str, + base_attrs: dict[str, Any], +) -> type[SQLModel]: + overlap = set(user_model.model_fields) & set(core_model.model_fields) + if overlap: + msg = f"Scope fields conflict with core model fields: {sorted(overlap)}" + raise TypeError(msg) + + return type( + f"{user_model.__name__}{core_model.__name__}{name_suffix}", + (user_model, core_model), + base_attrs, + ) + + +def build_table_model( + user_model: type[BaseModel], + core_model: type[SQLModel], + *, + tablename: str, + metadata: MetaData | None = None, + extra_table_args: tuple[Any, ...] | None = None, + unique_with_scope: list[str] | None = None, +) -> type[SQLModel]: + overlap = set(user_model.model_fields) & set(core_model.model_fields) + if overlap: + msg = f"Scope fields conflict with core model fields: {sorted(overlap)}" + raise TypeError(msg) + + scope_fields = list(user_model.model_fields.keys()) + base_table_args, table_kwargs = _normalize_table_args(getattr(core_model, "__table_args__", None)) + table_args = list(base_table_args) + if extra_table_args: + table_args.extend(extra_table_args) + if scope_fields: + table_args.append(Index(f"ix_{tablename}__scope", *scope_fields)) + if unique_with_scope: + unique_cols = [*unique_with_scope, *scope_fields] + table_args.append(Index(f"ix_{tablename}__unique_scoped", *unique_cols, unique=True)) + + base_attrs: dict[str, Any] = {"__module__": core_model.__module__, "__tablename__": tablename} + if metadata is not None: + base_attrs["metadata"] = metadata + if table_args or table_kwargs: + if table_kwargs: + base_attrs["__table_args__"] = (*table_args, table_kwargs) + else: + base_attrs["__table_args__"] = tuple(table_args) + + base = _merge_models(user_model, core_model, name_suffix="Base", base_attrs=base_attrs) + + # Use type() instead of create_model to properly preserve SQLModel table behavior + table_attrs: dict[str, Any] = {"__module__": core_model.__module__} + return type( + f"{user_model.__name__}{core_model.__name__}Table", + (base,), + table_attrs, + table=True, + ) + + +def build_scoped_models( + user_model: type[BaseModel], +) -> tuple[type[SQLModel], type[SQLModel], type[SQLModel], type[SQLModel], type[SQLModel], type[SQLModel]]: + """ + Build scoped SQLModel tables for each entity (resource, category, item, relation, segment). + """ + resource_model = build_table_model(user_model, ResourceModel, tablename="resources") + recall_file_model = build_table_model( + user_model, RecallFileModel, tablename="recall_files", unique_with_scope=["name"] + ) + recall_entry_model = build_table_model(user_model, RecallEntryModel, tablename="recall_entries") + recall_file_entry_model = build_table_model(user_model, RecallFileEntryModel, tablename="recall_file_entries") + recall_file_resource_model = build_table_model( + user_model, RecallFileResourceModel, tablename="recall_file_resources" + ) + recall_file_segment_model = build_table_model(user_model, RecallFileSegmentModel, tablename="recall_file_segments") + return ( + resource_model, + recall_file_model, + recall_entry_model, + recall_file_entry_model, + recall_file_resource_model, + recall_file_segment_model, + ) + + +__all__ = [ + "BaseModelMixin", + "RecallEntryModel", + "RecallFileEntryModel", + "RecallFileModel", + "RecallFileResourceModel", + "RecallFileSegmentModel", + "ResourceModel", + "build_scoped_models", + "build_table_model", +] diff --git a/src/memu/database/postgres/postgres.py b/src/memu/database/postgres/postgres.py new file mode 100644 index 0000000..3f785c0 --- /dev/null +++ b/src/memu/database/postgres/postgres.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import logging +from typing import Any + +from pydantic import BaseModel + +from memu.database.interfaces import Database +from memu.database.models import ( + RecallEntry, + RecallFile, + RecallFileEntry, + RecallFileResource, + RecallFileSegment, + Resource, +) +from memu.database.postgres.migration import DDLMode, run_migrations +from memu.database.postgres.repositories.recall_entry_repo import PostgresRecallEntryRepo +from memu.database.postgres.repositories.recall_file_entry_repo import PostgresRecallFileEntryRepo +from memu.database.postgres.repositories.recall_file_repo import PostgresRecallFileRepo +from memu.database.postgres.repositories.recall_file_resource_repo import PostgresRecallFileResourceRepo +from memu.database.postgres.repositories.recall_file_segment_repo import PostgresRecallFileSegmentRepo +from memu.database.postgres.repositories.resource_repo import PostgresResourceRepo +from memu.database.postgres.schema import SQLAModels, get_sqlalchemy_models, require_sqlalchemy +from memu.database.postgres.session import SessionManager +from memu.database.repositories import ( + RecallEntryRepo, + RecallFileEntryRepo, + RecallFileRepo, + RecallFileResourceRepo, + RecallFileSegmentRepo, + ResourceRepo, +) +from memu.database.state import DatabaseState + +logger = logging.getLogger(__name__) + + +class PostgresStore(Database): + resource_repo: ResourceRepo + recall_file_repo: RecallFileRepo + recall_entry_repo: RecallEntryRepo + recall_file_entry_repo: RecallFileEntryRepo + recall_file_resource_repo: RecallFileResourceRepo + recall_file_segment_repo: RecallFileSegmentRepo + resources: dict[str, Resource] + items: dict[str, RecallEntry] + categories: dict[str, RecallFile] + relations: list[RecallFileEntry] + resource_relations: list[RecallFileResource] + segments: list[RecallFileSegment] + + def __init__( + self, + *, + dsn: str, + ddl_mode: DDLMode = "create", + vector_provider: str | None = None, + scope_model: type[BaseModel] | None = None, + base_model: type[BaseModel] | None = None, + resource_model: type[Any] | None = None, + recall_file_model: type[Any] | None = None, + recall_entry_model: type[Any] | None = None, + recall_file_entry_model: type[Any] | None = None, + recall_file_resource_model: type[Any] | None = None, + recall_file_segment_model: type[Any] | None = None, + sqla_models: SQLAModels | None = None, + ) -> None: + require_sqlalchemy() + self.dsn = dsn + self.ddl_mode = ddl_mode + self.vector_provider = vector_provider + self._use_vector_type = vector_provider == "pgvector" + self._scope_model: type[BaseModel] = scope_model or base_model or BaseModel + self._scope_fields = list(getattr(self._scope_model, "model_fields", {}).keys()) + self._state = DatabaseState() + self._sessions = SessionManager(dsn=self.dsn) + self._sqla_models: SQLAModels = sqla_models or get_sqlalchemy_models(scope_model=self._scope_model) + run_migrations(dsn=self.dsn, scope_model=self._scope_model, ddl_mode=self.ddl_mode) + + resource_model = resource_model or self._sqla_models.Resource + recall_file_model = recall_file_model or self._sqla_models.RecallFile + recall_entry_model = recall_entry_model or self._sqla_models.RecallEntry + recall_file_entry_model = recall_file_entry_model or self._sqla_models.RecallFileEntry + recall_file_resource_model = recall_file_resource_model or self._sqla_models.RecallFileResource + recall_file_segment_model = recall_file_segment_model or self._sqla_models.RecallFileSegment + + self.resource_repo = PostgresResourceRepo( + state=self._state, + resource_model=resource_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + self.recall_file_repo = PostgresRecallFileRepo( + state=self._state, + recall_file_model=recall_file_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + self.recall_entry_repo = PostgresRecallEntryRepo( + state=self._state, + recall_entry_model=recall_entry_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + use_vector=self._use_vector_type, + ) + self.recall_file_entry_repo = PostgresRecallFileEntryRepo( + state=self._state, + recall_file_entry_model=recall_file_entry_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + self.recall_file_resource_repo = PostgresRecallFileResourceRepo( + state=self._state, + recall_file_resource_model=recall_file_resource_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + self.recall_file_segment_repo = PostgresRecallFileSegmentRepo( + state=self._state, + recall_file_segment_model=recall_file_segment_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + + self.resources = self._state.resources + self.items = self._state.items + self.categories = self._state.categories + self.relations = self._state.relations + self.resource_relations = self._state.resource_relations + self.segments = self._state.segments + + # self._load_existing() + + def close(self) -> None: + self._sessions.close() + + def _load_existing(self) -> None: + self.resource_repo.load_existing() + self.recall_file_repo.load_existing() + self.recall_entry_repo.load_existing() + self.recall_file_entry_repo.load_existing() + self.recall_file_resource_repo.load_existing() + self.recall_file_segment_repo.load_existing() diff --git a/src/memu/database/postgres/repositories/__init__.py b/src/memu/database/postgres/repositories/__init__.py new file mode 100644 index 0000000..1d2aeb5 --- /dev/null +++ b/src/memu/database/postgres/repositories/__init__.py @@ -0,0 +1,15 @@ +from memu.database.postgres.repositories.recall_entry_repo import PostgresRecallEntryRepo +from memu.database.postgres.repositories.recall_file_entry_repo import PostgresRecallFileEntryRepo +from memu.database.postgres.repositories.recall_file_repo import PostgresRecallFileRepo +from memu.database.postgres.repositories.recall_file_resource_repo import PostgresRecallFileResourceRepo +from memu.database.postgres.repositories.recall_file_segment_repo import PostgresRecallFileSegmentRepo +from memu.database.postgres.repositories.resource_repo import PostgresResourceRepo + +__all__ = [ + "PostgresRecallEntryRepo", + "PostgresRecallFileEntryRepo", + "PostgresRecallFileRepo", + "PostgresRecallFileResourceRepo", + "PostgresRecallFileSegmentRepo", + "PostgresResourceRepo", +] diff --git a/src/memu/database/postgres/repositories/base.py b/src/memu/database/postgres/repositories/base.py new file mode 100644 index 0000000..0823dbf --- /dev/null +++ b/src/memu/database/postgres/repositories/base.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +import pendulum + +from memu.database.postgres.session import SessionManager +from memu.database.state import DatabaseState + +logger = logging.getLogger(__name__) + + +class PostgresRepoBase: + def __init__( + self, + *, + state: DatabaseState, + sqla_models: Any, + sessions: SessionManager, + scope_fields: list[str], + use_vector: bool = True, + ) -> None: + self._state = state + self._sqla_models = sqla_models + self._sessions = sessions + self._scope_fields = scope_fields + self._use_vector = use_vector + + def _scope_kwargs_from(self, obj: Any) -> dict[str, Any]: + return {field: getattr(obj, field, None) for field in self._scope_fields} + + def _normalize_embedding(self, embedding: Any) -> list[float] | None: + if embedding is None: + return None + if hasattr(embedding, "to_list"): + try: + return [float(x) for x in embedding.to_list()] + except Exception: + logger.debug("Could not convert pgvector value %s", embedding) + return None + if isinstance(embedding, str): + stripped = embedding.strip("[]") + if not stripped: + return [] + return [float(x) for x in stripped.split(",")] + try: + return [float(x) for x in embedding] + except Exception: + logger.debug("Could not normalize embedding %s", embedding) + return None + + def _prepare_embedding(self, embedding: list[float] | None) -> Any: + if embedding is None: + return None + return embedding + + def _merge_and_commit(self, obj: Any) -> None: + with self._sessions.session() as session: + session.merge(obj) + session.commit() + + def _now(self) -> pendulum.DateTime: + return pendulum.now("UTC") + + def _build_filters(self, model: Any, where: Mapping[str, Any] | None) -> list[Any]: + if not where: + return [] + filters: list[Any] = [] + for raw_key, expected in where.items(): + if expected is None: + continue + field, op = [*raw_key.split("__", 1), None][:2] + column = getattr(model, str(field), None) + if column is None: + msg = f"Unknown filter field '{field}' for model '{model.__name__}'" + raise ValueError(msg) + if op == "in": + if isinstance(expected, str): + filters.append(column == expected) + else: + filters.append(column.in_(expected)) + else: + filters.append(column == expected) + return filters + + @staticmethod + def _matches_where(obj: Any, where: Mapping[str, Any] | None) -> bool: + if not where: + return True + for raw_key, expected in where.items(): + if expected is None: + continue + field, op = [*raw_key.split("__", 1), None][:2] + actual = getattr(obj, str(field), None) + if op == "in": + if isinstance(expected, str): + if actual != expected: + return False + else: + try: + if actual not in expected: + return False + except TypeError: + return False + else: + if actual != expected: + return False + return True + + +__all__ = ["PostgresRepoBase"] diff --git a/src/memu/database/postgres/repositories/recall_entry_repo.py b/src/memu/database/postgres/repositories/recall_entry_repo.py new file mode 100644 index 0000000..bca8bbc --- /dev/null +++ b/src/memu/database/postgres/repositories/recall_entry_repo.py @@ -0,0 +1,401 @@ +from __future__ import annotations + +import math +from collections.abc import Mapping +from datetime import datetime +from typing import Any + +from memu.database.models import EntryType, RecallEntry, compute_content_hash +from memu.database.postgres.repositories.base import PostgresRepoBase +from memu.database.postgres.session import SessionManager +from memu.database.state import DatabaseState + + +class PostgresRecallEntryRepo(PostgresRepoBase): + def __init__( + self, + *, + state: DatabaseState, + recall_entry_model: type[RecallEntry], + sqla_models: Any, + sessions: SessionManager, + scope_fields: list[str], + use_vector: bool, + ) -> None: + super().__init__( + state=state, sqla_models=sqla_models, sessions=sessions, scope_fields=scope_fields, use_vector=use_vector + ) + self._recall_entry_model = recall_entry_model + self.items: dict[str, RecallEntry] = self._state.items + + def get_item(self, memory_id: str) -> RecallEntry | None: + from sqlmodel import select + + with self._sessions.session() as session: + row = session.scalar( + select(self._sqla_models.RecallEntry).where(self._sqla_models.RecallEntry.id == memory_id) + ) + if row: + row.embedding = self._normalize_embedding(row.embedding) + return self._cache_item(row) + return None + + def list_items(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallEntry]: + from sqlmodel import select + + filters = self._build_filters(self._sqla_models.RecallEntry, where) + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallEntry).where(*filters)).all() + result: dict[str, RecallEntry] = {} + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + item = self._cache_item(row) + result[item.id] = item + return result + + def list_items_by_ref_ids( + self, ref_ids: list[str], where: Mapping[str, Any] | None = None + ) -> dict[str, RecallEntry]: + """List items by their ref_id in the extra column. + + Args: + ref_ids: List of ref_ids to query. + where: Additional filter conditions. + + Returns: + Dict mapping item_id -> RecallEntry for items whose extra->>'ref_id' is in ref_ids. + """ + if not ref_ids: + return {} + + from sqlmodel import select + + filters = self._build_filters(self._sqla_models.RecallEntry, where) + # Add filter for extra->>'ref_id' IN ref_ids (only rows with ref_id key) + ref_id_col = self._sqla_models.RecallEntry.extra["ref_id"].astext + filters.append(ref_id_col.isnot(None)) + filters.append(ref_id_col.in_(ref_ids)) + + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallEntry).where(*filters)).all() + result: dict[str, RecallEntry] = {} + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + item = self._cache_item(row) + result[item.id] = item + return result + + def clear_items(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallEntry]: + from sqlmodel import delete, select + + filters = self._build_filters(self._sqla_models.RecallEntry, where) + with self._sessions.session() as session: + # First get the objects to delete + rows = session.scalars(select(self._sqla_models.RecallEntry).where(*filters)).all() + deleted: dict[str, RecallEntry] = {} + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + deleted[row.id] = row + + if not deleted: + return {} + + # Delete from database + session.exec(delete(self._sqla_models.RecallEntry).where(*filters)) + session.commit() + + # Clean up cache + for item_id in deleted: + self.items.pop(item_id, None) + + return deleted + + def create_item( + self, + *, + resource_id: str | None = None, + memory_type: EntryType, + summary: str, + embedding: list[float], + user_data: dict[str, Any], + reinforce: bool = False, + tool_record: dict[str, Any] | None = None, + ) -> RecallEntry: + if reinforce and memory_type != "tool": + return self.create_item_reinforce( + resource_id=resource_id, + memory_type=memory_type, + summary=summary, + embedding=embedding, + user_data=user_data, + ) + + # Build extra dict with tool_record fields at top level + extra: dict[str, Any] = {} + if tool_record: + if tool_record.get("when_to_use") is not None: + extra["when_to_use"] = tool_record["when_to_use"] + if tool_record.get("metadata") is not None: + extra["metadata"] = tool_record["metadata"] + if tool_record.get("tool_calls") is not None: + extra["tool_calls"] = tool_record["tool_calls"] + + item = self._recall_entry_model( + resource_id=resource_id, + memory_type=memory_type, + summary=summary, + embedding=self._prepare_embedding(embedding), + extra=extra if extra else {}, + **user_data, + created_at=self._now(), + updated_at=self._now(), + ) + + with self._sessions.session() as session: + session.add(item) + session.commit() + session.refresh(item) + + self.items[item.id] = item + return item + + def create_item_reinforce( + self, + *, + resource_id: str | None = None, + memory_type: EntryType, + summary: str, + embedding: list[float], + user_data: dict[str, Any], + ) -> RecallEntry: + from sqlmodel import select + + content_hash = compute_content_hash(summary, memory_type) + + with self._sessions.session() as session: + # Check for existing item with same hash in same scope (deduplication) + # Use extra->>'content_hash' for query performance + content_hash_col = self._sqla_models.RecallEntry.extra["content_hash"].astext + filters = [content_hash_col == content_hash] + filters.extend(self._build_filters(self._sqla_models.RecallEntry, user_data)) + + existing = session.scalar(select(self._sqla_models.RecallEntry).where(*filters)) + + if existing: + # Reinforce existing memory instead of creating duplicate + current_extra = existing.extra or {} + current_count = current_extra.get("reinforcement_count", 1) + existing.extra = { + **current_extra, + "reinforcement_count": current_count + 1, + "last_reinforced_at": self._now().isoformat(), + } + existing.updated_at = self._now() + session.add(existing) + session.commit() + session.refresh(existing) + existing.embedding = self._normalize_embedding(existing.embedding) + return self._cache_item(existing) + + # Create new item with salience tracking in extra + now = self._now() + + item = self._recall_entry_model( + resource_id=resource_id, + memory_type=memory_type, + summary=summary, + embedding=self._prepare_embedding(embedding), + **user_data, + created_at=now, + updated_at=now, + extra={ + "content_hash": content_hash, + "reinforcement_count": 1, + "last_reinforced_at": now.isoformat(), + }, + ) + + session.add(item) + session.commit() + session.refresh(item) + + self.items[item.id] = item + return item + + def update_item( + self, + *, + item_id: str, + memory_type: EntryType | None = None, + summary: str | None = None, + embedding: list[float] | None = None, + extra: dict[str, Any] | None = None, + tool_record: dict[str, Any] | None = None, + ) -> RecallEntry: + from sqlmodel import select + + now = self._now() + with self._sessions.session() as session: + item = session.scalar( + select(self._sqla_models.RecallEntry).where(self._sqla_models.RecallEntry.id == item_id) + ) + if item is None: + msg = f"Item with id {item_id} not found" + raise KeyError(msg) + + if memory_type is not None: + item.memory_type = memory_type + if summary is not None: + item.summary = summary + if embedding is not None: + item.embedding = self._prepare_embedding(embedding) + + # Merge extra and tool_record into existing extra dict + current_extra = item.extra or {} + if extra is not None: + current_extra = {**current_extra, **extra} + if tool_record is not None: + # Merge tool_record fields at top level + for key in ("when_to_use", "metadata", "tool_calls"): + if tool_record.get(key) is not None: + current_extra[key] = tool_record[key] + if extra is not None or tool_record is not None: + item.extra = current_extra + + item.updated_at = now + session.add(item) + session.commit() + session.refresh(item) + item.embedding = self._normalize_embedding(item.embedding) + + return self._cache_item(item) + + def delete_item(self, item_id: str) -> None: + from sqlmodel import delete + + with self._sessions.session() as session: + session.exec(delete(self._sqla_models.RecallEntry).where(self._sqla_models.RecallEntry.id == item_id)) + session.commit() + + def vector_search_items( + self, + query_vec: list[float], + top_k: int, + where: Mapping[str, Any] | None = None, + *, + ranking: str = "similarity", + recency_decay_days: float = 30.0, + ) -> list[tuple[str, float]]: + if not self._use_vector or ranking == "salience": + # For salience ranking or when pgvector is not available, use local search + return self._vector_search_local( + query_vec, top_k, where=where, ranking=ranking, recency_decay_days=recency_decay_days + ) + + from sqlmodel import select + + distance = self._sqla_models.RecallEntry.embedding.cosine_distance(query_vec) + filters = [self._sqla_models.RecallEntry.embedding.isnot(None)] + filters.extend(self._build_filters(self._sqla_models.RecallEntry, where)) + stmt = ( + select(self._sqla_models.RecallEntry.id, (1 - distance).label("score")) + .where(*filters) + .order_by(distance) + .limit(top_k) + ) + with self._sessions.session() as session: + rows = session.execute(stmt).all() + return [(rid, float(score)) for rid, score in rows] + + def load_existing(self) -> None: + from sqlmodel import select + + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallEntry)).all() + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + self._cache_item(row) + + def _vector_search_local( + self, + query_vec: list[float], + top_k: int, + where: Mapping[str, Any] | None = None, + *, + ranking: str = "similarity", + recency_decay_days: float = 30.0, + ) -> list[tuple[str, float]]: + scored: list[tuple[str, float]] = [] + for item in self.items.values(): + if item.embedding is None: + continue + if not self._matches_where(item, where): + continue + + similarity = self._cosine(query_vec, item.embedding) + + if ranking == "salience": + # Salience-aware scoring - read from extra dict + extra = item.extra or {} + reinforcement_count = extra.get("reinforcement_count", 1) + last_reinforced_at = self._parse_datetime(extra.get("last_reinforced_at")) + score = self._salience_score( + similarity, + reinforcement_count, + last_reinforced_at, + recency_decay_days, + ) + else: + score = similarity + + scored.append((item.id, score)) + + scored.sort(key=lambda x: x[1], reverse=True) + return scored[:top_k] + + @staticmethod + def _salience_score( + similarity: float, + reinforcement_count: int, + last_reinforced_at: datetime | None, + recency_decay_days: float, + ) -> float: + """Compute salience score: similarity * reinforcement * recency.""" + reinforcement_factor = math.log(reinforcement_count + 1) + + if last_reinforced_at is None: + recency_factor = 0.5 + else: + now = datetime.now(last_reinforced_at.tzinfo) if last_reinforced_at.tzinfo else datetime.utcnow() + days_ago = (now - last_reinforced_at).total_seconds() / 86400 + recency_factor = math.exp(-0.693 * days_ago / recency_decay_days) + + return similarity * reinforcement_factor * recency_factor + + def _cache_item(self, item: RecallEntry) -> RecallEntry: + self.items[item.id] = item + return item + + @staticmethod + def _parse_datetime(dt_str: str | None) -> datetime | None: + """Parse ISO datetime string from extra dict.""" + if dt_str is None: + return None + try: + import pendulum + + parsed = pendulum.parse(dt_str) + except (ValueError, TypeError): + return None + else: + if isinstance(parsed, datetime): + return parsed + return None + + @staticmethod + def _cosine(a: list[float], b: list[float]) -> float: + denom = (sum(x * x for x in a) ** 0.5) * (sum(y * y for y in b) ** 0.5) + 1e-9 + return float(sum(x * y for x, y in zip(a, b, strict=True)) / denom) + + +__all__ = ["PostgresRecallEntryRepo"] diff --git a/src/memu/database/postgres/repositories/recall_file_entry_repo.py b/src/memu/database/postgres/repositories/recall_file_entry_repo.py new file mode 100644 index 0000000..9198d0e --- /dev/null +++ b/src/memu/database/postgres/repositories/recall_file_entry_repo.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from memu.database.models import RecallFileEntry +from memu.database.postgres.repositories.base import PostgresRepoBase +from memu.database.postgres.session import SessionManager +from memu.database.repositories.recall_file_entry import RecallFileEntryRepo +from memu.database.state import DatabaseState + + +class PostgresRecallFileEntryRepo(PostgresRepoBase, RecallFileEntryRepo): + def __init__( + self, + *, + state: DatabaseState, + recall_file_entry_model: type[RecallFileEntry], + sqla_models: Any, + sessions: SessionManager, + scope_fields: list[str], + ) -> None: + super().__init__(state=state, sqla_models=sqla_models, sessions=sessions, scope_fields=scope_fields) + self._recall_file_entry_model = recall_file_entry_model + self.relations: list[RecallFileEntry] = self._state.relations + + def list_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileEntry]: + from sqlmodel import select + + filters = self._build_filters(self._sqla_models.RecallFileEntry, where) + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFileEntry).where(*filters)).all() + return [self._cache_relation(row) for row in rows] + + def link_item_category(self, item_id: str, cat_id: str, user_data: dict[str, Any]) -> RecallFileEntry: + from sqlmodel import select + + # Avoid duplicate inserts using local cache + for rel in self.relations: + if rel.item_id == item_id and rel.category_id == cat_id: + return rel + + now = self._now() + new_rel = self._recall_file_entry_model( + item_id=item_id, + category_id=cat_id, + **user_data, + created_at=now, + updated_at=now, + ) + + with self._sessions.session() as session: + existing = session.scalar( + select(self._sqla_models.RecallFileEntry).where( + self._sqla_models.RecallFileEntry.item_id == item_id, + self._sqla_models.RecallFileEntry.category_id == cat_id, + ) + ) + if existing: + return self._cache_relation(existing) + + session.add(new_rel) + session.commit() + session.refresh(new_rel) + + return self._cache_relation(new_rel) + + def unlink_item_category(self, item_id: str, cat_id: str) -> None: + from sqlmodel import delete + + with self._sessions.session() as session: + session.exec( + delete(self._sqla_models.RecallFileEntry).where( + self._sqla_models.RecallFileEntry.item_id == item_id, + self._sqla_models.RecallFileEntry.category_id == cat_id, + ) + ) + session.commit() + self.relations[:] = [r for r in self.relations if not (r.item_id == item_id and r.category_id == cat_id)] + + def _row_to_record(self, row: Any) -> RecallFileEntry: + return RecallFileEntry( + id=row.id, + item_id=row.item_id, + category_id=row.category_id, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + + def unlink_item(self, item_id: str) -> list[RecallFileEntry]: + from sqlmodel import delete, select + + with self._sessions.session() as session: + rows = session.scalars( + select(self._sqla_models.RecallFileEntry).where(self._sqla_models.RecallFileEntry.item_id == item_id) + ).all() + removed = [self._row_to_record(row) for row in rows] + if removed: + session.exec( + delete(self._sqla_models.RecallFileEntry).where( + self._sqla_models.RecallFileEntry.item_id == item_id + ) + ) + session.commit() + self.relations[:] = [r for r in self.relations if r.item_id != item_id] + return removed + + def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileEntry]: + from sqlmodel import delete, select + + filters = self._build_filters(self._sqla_models.RecallFileEntry, where) + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFileEntry).where(*filters)).all() + removed = [self._row_to_record(row) for row in rows] + if removed: + session.exec(delete(self._sqla_models.RecallFileEntry).where(*filters)) + session.commit() + removed_ids = {rel.id for rel in removed} + self.relations[:] = [r for r in self.relations if r.id not in removed_ids] + return removed + + def get_item_categories(self, item_id: str) -> list[RecallFileEntry]: + from sqlmodel import select + + with self._sessions.session() as session: + rows = session.scalars( + select(self._sqla_models.RecallFileEntry).where(self._sqla_models.RecallFileEntry.item_id == item_id) + ).all() + return [self._cache_relation(row) for row in rows] + + def load_existing(self) -> None: + from sqlmodel import select + + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFileEntry)).all() + for row in rows: + self._cache_relation(row) + + def _cache_relation(self, rel: RecallFileEntry) -> RecallFileEntry: + self.relations.append(rel) + return rel + + +__all__ = ["PostgresRecallFileEntryRepo"] diff --git a/src/memu/database/postgres/repositories/recall_file_repo.py b/src/memu/database/postgres/repositories/recall_file_repo.py new file mode 100644 index 0000000..79cd57e --- /dev/null +++ b/src/memu/database/postgres/repositories/recall_file_repo.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from memu.database.models import RecallFile +from memu.database.postgres.repositories.base import PostgresRepoBase +from memu.database.postgres.session import SessionManager +from memu.database.repositories.recall_file import RecallFileRepo +from memu.database.state import DatabaseState + + +class PostgresRecallFileRepo(PostgresRepoBase, RecallFileRepo): + def __init__( + self, + *, + state: DatabaseState, + recall_file_model: type[RecallFile], + sqla_models: Any, + sessions: SessionManager, + scope_fields: list[str], + ) -> None: + super().__init__(state=state, sqla_models=sqla_models, sessions=sessions, scope_fields=scope_fields) + self._recall_file_model = recall_file_model + self.categories: dict[str, RecallFile] = self._state.categories + + def list_categories(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallFile]: + from sqlmodel import select + + filters = self._build_filters(self._sqla_models.RecallFile, where) + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFile).where(*filters)).all() + result: dict[str, RecallFile] = {} + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + cat = self._cache_category(row) + result[cat.id] = cat + return result + + def clear_categories(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallFile]: + from sqlmodel import delete, select + + filters = self._build_filters(self._sqla_models.RecallFile, where) + with self._sessions.session() as session: + # First get the objects to delete + rows = session.scalars(select(self._sqla_models.RecallFile).where(*filters)).all() + deleted: dict[str, RecallFile] = {} + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + deleted[row.id] = row + + if not deleted: + return {} + + # Delete from database + session.exec(delete(self._sqla_models.RecallFile).where(*filters)) + session.commit() + + # Clean up cache + for cat_id in deleted: + self.categories.pop(cat_id, None) + + return deleted + + def get_or_create_category( + self, + *, + name: str, + description: str, + embedding: list[float], + user_data: dict[str, Any], + track: str = "memory", + ) -> RecallFile: + from sqlmodel import select + + now = self._now() + with self._sessions.session() as session: + filters = [ + self._sqla_models.RecallFile.name == name, + self._sqla_models.RecallFile.track == track, + ] + for key, value in user_data.items(): + filters.append(getattr(self._sqla_models.RecallFile, key) == value) + existing = session.scalar(select(self._sqla_models.RecallFile).where(*filters)) + + if existing: + updated = False + if getattr(existing, "embedding", None) is None: + existing.embedding = self._prepare_embedding(embedding) + updated = True + if getattr(existing, "description", None) is None: + existing.description = description + updated = True + if updated: + existing.updated_at = now + session.add(existing) + session.commit() + session.refresh(existing) + return self._cache_category(existing) + + cat = self._recall_file_model( + name=name, + description=description, + embedding=self._prepare_embedding(embedding), + track=track, + created_at=now, + updated_at=now, + **user_data, + ) + session.add(cat) + session.commit() + session.refresh(cat) + + return self._cache_category(cat) + + def update_category( + self, + *, + category_id: str, + name: str | None = None, + description: str | None = None, + embedding: list[float] | None = None, + content: str | None = None, + ) -> RecallFile: + from sqlmodel import select + + now = self._now() + with self._sessions.session() as session: + cat = session.scalar( + select(self._sqla_models.RecallFile).where(self._sqla_models.RecallFile.id == category_id) + ) + if cat is None: + msg = f"Category with id {category_id} not found" + raise KeyError(msg) + + if name is not None: + cat.name = name + if description is not None: + cat.description = description + if embedding is not None: + cat.embedding = self._prepare_embedding(embedding) + if content is not None: + cat.content = content + + cat.updated_at = now + session.add(cat) + session.commit() + session.refresh(cat) + cat.embedding = self._normalize_embedding(cat.embedding) + + return self._cache_category(cat) + + def load_existing(self) -> None: + from sqlmodel import select + + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFile)).all() + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + self._cache_category(row) + + def _cache_category(self, cat: RecallFile) -> RecallFile: + self.categories[cat.id] = cat + return cat + + +__all__ = ["PostgresRecallFileRepo"] diff --git a/src/memu/database/postgres/repositories/recall_file_resource_repo.py b/src/memu/database/postgres/repositories/recall_file_resource_repo.py new file mode 100644 index 0000000..5a4ba41 --- /dev/null +++ b/src/memu/database/postgres/repositories/recall_file_resource_repo.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from memu.database.models import RecallFileResource +from memu.database.postgres.repositories.base import PostgresRepoBase +from memu.database.postgres.session import SessionManager +from memu.database.repositories.recall_file_resource import RecallFileResourceRepo +from memu.database.state import DatabaseState + + +class PostgresRecallFileResourceRepo(PostgresRepoBase, RecallFileResourceRepo): + def __init__( + self, + *, + state: DatabaseState, + recall_file_resource_model: type[RecallFileResource], + sqla_models: Any, + sessions: SessionManager, + scope_fields: list[str], + ) -> None: + super().__init__(state=state, sqla_models=sqla_models, sessions=sessions, scope_fields=scope_fields) + self._recall_file_resource_model = recall_file_resource_model + self.relations: list[RecallFileResource] = self._state.resource_relations + + def list_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileResource]: + from sqlmodel import select + + filters = self._build_filters(self._sqla_models.RecallFileResource, where) + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFileResource).where(*filters)).all() + return [self._cache_relation(row) for row in rows] + + def link_resource_category(self, resource_id: str, cat_id: str, user_data: dict[str, Any]) -> RecallFileResource: + from sqlmodel import select + + # Avoid duplicate inserts using local cache + for rel in self.relations: + if rel.resource_id == resource_id and rel.file_id == cat_id: + return rel + + now = self._now() + new_rel = self._recall_file_resource_model( + resource_id=resource_id, + file_id=cat_id, + **user_data, + created_at=now, + updated_at=now, + ) + + with self._sessions.session() as session: + existing = session.scalar( + select(self._sqla_models.RecallFileResource).where( + self._sqla_models.RecallFileResource.resource_id == resource_id, + self._sqla_models.RecallFileResource.file_id == cat_id, + ) + ) + if existing: + return self._cache_relation(existing) + + session.add(new_rel) + session.commit() + session.refresh(new_rel) + + return self._cache_relation(new_rel) + + def unlink_resource_category(self, resource_id: str, cat_id: str) -> None: + from sqlmodel import delete + + with self._sessions.session() as session: + session.exec( + delete(self._sqla_models.RecallFileResource).where( + self._sqla_models.RecallFileResource.resource_id == resource_id, + self._sqla_models.RecallFileResource.file_id == cat_id, + ) + ) + session.commit() + self.relations[:] = [r for r in self.relations if not (r.resource_id == resource_id and r.file_id == cat_id)] + + def _row_to_record(self, row: Any) -> RecallFileResource: + return RecallFileResource( + id=row.id, + resource_id=row.resource_id, + file_id=row.file_id, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + + def unlink_resource(self, resource_id: str) -> list[RecallFileResource]: + from sqlmodel import delete, select + + with self._sessions.session() as session: + rows = session.scalars( + select(self._sqla_models.RecallFileResource).where( + self._sqla_models.RecallFileResource.resource_id == resource_id + ) + ).all() + removed = [self._row_to_record(row) for row in rows] + if removed: + session.exec( + delete(self._sqla_models.RecallFileResource).where( + self._sqla_models.RecallFileResource.resource_id == resource_id + ) + ) + session.commit() + self.relations[:] = [r for r in self.relations if r.resource_id != resource_id] + return removed + + def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileResource]: + from sqlmodel import delete, select + + filters = self._build_filters(self._sqla_models.RecallFileResource, where) + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFileResource).where(*filters)).all() + removed = [self._row_to_record(row) for row in rows] + if removed: + session.exec(delete(self._sqla_models.RecallFileResource).where(*filters)) + session.commit() + removed_ids = {rel.id for rel in removed} + self.relations[:] = [r for r in self.relations if r.id not in removed_ids] + return removed + + def get_resource_categories(self, resource_id: str) -> list[RecallFileResource]: + from sqlmodel import select + + with self._sessions.session() as session: + rows = session.scalars( + select(self._sqla_models.RecallFileResource).where( + self._sqla_models.RecallFileResource.resource_id == resource_id + ) + ).all() + return [self._cache_relation(row) for row in rows] + + def load_existing(self) -> None: + from sqlmodel import select + + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFileResource)).all() + for row in rows: + self._cache_relation(row) + + def _cache_relation(self, rel: RecallFileResource) -> RecallFileResource: + self.relations.append(rel) + return rel + + +__all__ = ["PostgresRecallFileResourceRepo"] diff --git a/src/memu/database/postgres/repositories/recall_file_segment_repo.py b/src/memu/database/postgres/repositories/recall_file_segment_repo.py new file mode 100644 index 0000000..3ef234c --- /dev/null +++ b/src/memu/database/postgres/repositories/recall_file_segment_repo.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from memu.database.models import RecallFileSegment +from memu.database.postgres.repositories.base import PostgresRepoBase +from memu.database.postgres.session import SessionManager +from memu.database.repositories.recall_file_segment import RecallFileSegmentRepo +from memu.database.state import DatabaseState + + +class PostgresRecallFileSegmentRepo(PostgresRepoBase, RecallFileSegmentRepo): + def __init__( + self, + *, + state: DatabaseState, + recall_file_segment_model: type[RecallFileSegment], + sqla_models: Any, + sessions: SessionManager, + scope_fields: list[str], + ) -> None: + super().__init__(state=state, sqla_models=sqla_models, sessions=sessions, scope_fields=scope_fields) + self._recall_file_segment_model = recall_file_segment_model + self.segments: list[RecallFileSegment] = self._state.segments + + def _row_to_record(self, row: Any) -> RecallFileSegment: + return RecallFileSegment( + id=row.id, + recall_file_id=row.recall_file_id, + track=row.track, + text=row.text, + embedding=self._normalize_embedding(row.embedding), + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + + def _cache_segment(self, row: Any) -> RecallFileSegment: + seg = self._row_to_record(row) + self.segments.append(seg) + return seg + + def list_segments(self, where: Mapping[str, Any] | None = None) -> list[RecallFileSegment]: + from sqlmodel import select + + filters = self._build_filters(self._sqla_models.RecallFileSegment, where) + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFileSegment).where(*filters)).all() + return [self._cache_segment(row) for row in rows] + + def list_segments_for_file(self, recall_file_id: str) -> list[RecallFileSegment]: + return self.list_segments({"recall_file_id": recall_file_id}) + + def create_segment( + self, + *, + recall_file_id: str, + text: str, + embedding: list[float] | None, + user_data: dict[str, Any], + track: str = "memory", + ) -> RecallFileSegment: + now = self._now() + row = self._recall_file_segment_model( + recall_file_id=recall_file_id, + track=track, + text=text, + embedding=self._prepare_embedding(embedding), + created_at=now, + updated_at=now, + **user_data, + ) + with self._sessions.session() as session: + session.add(row) + session.commit() + session.refresh(row) + return self._cache_segment(row) + + def delete_segment(self, segment_id: str) -> None: + from sqlmodel import delete + + with self._sessions.session() as session: + session.exec( + delete(self._sqla_models.RecallFileSegment).where(self._sqla_models.RecallFileSegment.id == segment_id) + ) + session.commit() + self.segments[:] = [seg for seg in self.segments if seg.id != segment_id] + + def delete_segments_for_file(self, recall_file_id: str) -> list[RecallFileSegment]: + from sqlmodel import delete, select + + with self._sessions.session() as session: + rows = session.scalars( + select(self._sqla_models.RecallFileSegment).where( + self._sqla_models.RecallFileSegment.recall_file_id == recall_file_id + ) + ).all() + removed = [self._row_to_record(row) for row in rows] + if removed: + session.exec( + delete(self._sqla_models.RecallFileSegment).where( + self._sqla_models.RecallFileSegment.recall_file_id == recall_file_id + ) + ) + session.commit() + self.segments[:] = [seg for seg in self.segments if seg.recall_file_id != recall_file_id] + return removed + + def clear_segments(self, where: Mapping[str, Any] | None = None) -> list[RecallFileSegment]: + from sqlmodel import delete, select + + filters = self._build_filters(self._sqla_models.RecallFileSegment, where) + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFileSegment).where(*filters)).all() + removed = [self._row_to_record(row) for row in rows] + if removed: + session.exec(delete(self._sqla_models.RecallFileSegment).where(*filters)) + session.commit() + removed_ids = {seg.id for seg in removed} + self.segments[:] = [seg for seg in self.segments if seg.id not in removed_ids] + return removed + + def load_existing(self) -> None: + from sqlmodel import select + + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.RecallFileSegment)).all() + for row in rows: + self._cache_segment(row) + + +__all__ = ["PostgresRecallFileSegmentRepo"] diff --git a/src/memu/database/postgres/repositories/resource_repo.py b/src/memu/database/postgres/repositories/resource_repo.py new file mode 100644 index 0000000..e71e6da --- /dev/null +++ b/src/memu/database/postgres/repositories/resource_repo.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from memu.database.models import Resource +from memu.database.postgres.repositories.base import PostgresRepoBase +from memu.database.postgres.session import SessionManager +from memu.database.repositories.resource import ResourceRepo +from memu.database.state import DatabaseState +from memu.vector import cosine_topk + + +class PostgresResourceRepo(PostgresRepoBase, ResourceRepo): + def __init__( + self, + *, + state: DatabaseState, + resource_model: type[Resource], + sqla_models: Any, + sessions: SessionManager, + scope_fields: list[str], + ) -> None: + super().__init__(state=state, sqla_models=sqla_models, sessions=sessions, scope_fields=scope_fields) + self._resource_model = resource_model + self.resources: dict[str, Resource] = self._state.resources + + def list_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, Resource]: + from sqlmodel import select + + filters = self._build_filters(self._sqla_models.Resource, where) + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.Resource).where(*filters)).all() + result: dict[str, Resource] = {} + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + res = self._cache_resource(row) + result[res.id] = res + return result + + def clear_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, Resource]: + from sqlmodel import delete, select + + filters = self._build_filters(self._sqla_models.Resource, where) + with self._sessions.session() as session: + # First get the objects to delete + rows = session.scalars(select(self._sqla_models.Resource).where(*filters)).all() + deleted: dict[str, Resource] = {} + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + deleted[row.id] = row + + if not deleted: + return {} + + # Delete from database + session.exec(delete(self._sqla_models.Resource).where(*filters)) + session.commit() + + # Clean up cache + for res_id in deleted: + self.resources.pop(res_id, None) + + return deleted + + def delete_resource(self, resource_id: str) -> None: + """Delete a single resource by id (used for cascade sync).""" + from sqlmodel import delete + + with self._sessions.session() as session: + session.exec(delete(self._sqla_models.Resource).where(self._sqla_models.Resource.id == resource_id)) + session.commit() + self.resources.pop(resource_id, None) + + def create_resource( + self, + *, + url: str, + modality: str, + local_path: str, + caption: str | None, + embedding: list[float] | None, + user_data: dict[str, Any], + track: str | None = None, + ) -> Resource: + res = self._resource_model( + url=url, + modality=modality, + local_path=local_path, + caption=caption, + embedding=self._prepare_embedding(embedding), + track=track, + **user_data, + created_at=self._now(), + updated_at=self._now(), + ) + + with self._sessions.session() as session: + session.add(res) + session.commit() + session.refresh(res) + + return self._cache_resource(res) + + def vector_search_resources( + self, + query_vec: list[float], + top_k: int, + where: Mapping[str, Any] | None = None, + ) -> list[tuple[str, float]]: + """Rank resources by cosine similarity over stored embeddings. + + Resource captions are not indexed in pgvector, so this scores the loaded + resources in Python, matching the previous app-layer behavior. + """ + pool = self.list_resources(where) + corpus = [(rid, res.embedding) for rid, res in pool.items() if res.embedding] + return cosine_topk(query_vec, corpus, k=top_k) + + def load_existing(self) -> None: + from sqlmodel import select + + with self._sessions.session() as session: + rows = session.scalars(select(self._sqla_models.Resource)).all() + for row in rows: + row.embedding = self._normalize_embedding(row.embedding) + self._cache_resource(row) + + def _cache_resource(self, res: Resource) -> Resource: + self.resources[res.id] = res + return res + + +__all__ = ["PostgresResourceRepo"] diff --git a/src/memu/database/postgres/schema.py b/src/memu/database/postgres/schema.py new file mode 100644 index 0000000..70d2cee --- /dev/null +++ b/src/memu/database/postgres/schema.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from pydantic import BaseModel + +try: + from sqlmodel import SQLModel +except ImportError as exc: + msg = "sqlmodel is required for Postgres storage support" + raise ImportError(msg) from exc + +try: + from sqlalchemy import MetaData +except ImportError as exc: + msg = "sqlalchemy is required for Postgres storage support" + raise ImportError(msg) from exc + +try: + from pgvector.sqlalchemy import VECTOR as Vector +except ImportError as exc: + msg = "pgvector is required for Postgres vector support" + raise ImportError(msg) from exc + +from memu.app.settings import DefaultUserModel +from memu.database.postgres.models import ( + RecallEntryModel, + RecallFileEntryModel, + RecallFileModel, + RecallFileResourceModel, + RecallFileSegmentModel, + ResourceModel, + build_table_model, +) + +# Default user scope for the committed Alembic baseline. ``DefaultUserModel`` +# (in ``memu.app.settings``) is the single source of truth for the built-in +# scope columns (``user_id`` / ``agent_id``); this alias keeps the schema and +# the migration generated against it in sync with the app default. +DefaultScope = DefaultUserModel + + +@dataclass +class SQLAModels: + Base: type[Any] + Resource: type[Any] + RecallFile: type[Any] + RecallEntry: type[Any] + RecallFileEntry: type[Any] + RecallFileResource: type[Any] + RecallFileSegment: type[Any] + + +_MODEL_CACHE: dict[type[Any], SQLAModels] = {} + + +def require_sqlalchemy() -> None: + return None + + +def get_sqlalchemy_models(*, scope_model: type[BaseModel] | None = None) -> SQLAModels: + """ + Build (and cache) SQLModel ORM models for Postgres storage. + """ + require_sqlalchemy() + scope = scope_model or DefaultScope + cache_key = scope + cached = _MODEL_CACHE.get(cache_key) + if cached: + return cached + + metadata_obj = MetaData() + + resource_model = build_table_model( + scope, + ResourceModel, + tablename="resources", + metadata=metadata_obj, + ) + recall_file_model = build_table_model( + scope, + RecallFileModel, + tablename="recall_files", + metadata=metadata_obj, + ) + recall_entry_model = build_table_model( + scope, + RecallEntryModel, + tablename="recall_entries", + metadata=metadata_obj, + ) + recall_file_entry_model = build_table_model( + scope, + RecallFileEntryModel, + tablename="recall_file_entries", + metadata=metadata_obj, + ) + recall_file_resource_model = build_table_model( + scope, + RecallFileResourceModel, + tablename="recall_file_resources", + metadata=metadata_obj, + ) + recall_file_segment_model = build_table_model( + scope, + RecallFileSegmentModel, + tablename="recall_file_segments", + metadata=metadata_obj, + ) + + class Base(SQLModel): + __abstract__ = True + metadata = metadata_obj + + models = SQLAModels( + Base=Base, + Resource=resource_model, + RecallFile=recall_file_model, + RecallEntry=recall_entry_model, + RecallFileEntry=recall_file_entry_model, + RecallFileResource=recall_file_resource_model, + RecallFileSegment=recall_file_segment_model, + ) + _MODEL_CACHE[cache_key] = models + return models + + +def get_metadata(scope_model: type[BaseModel] | None = None) -> MetaData: + from typing import cast + + return cast(MetaData, get_sqlalchemy_models(scope_model=scope_model).Base.metadata) + + +__all__ = ["SQLAModels", "Vector", "get_metadata", "get_sqlalchemy_models", "require_sqlalchemy"] diff --git a/src/memu/database/postgres/session.py b/src/memu/database/postgres/session.py new file mode 100644 index 0000000..8bdac42 --- /dev/null +++ b/src/memu/database/postgres/session.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import logging +from typing import Any + +try: # Optional dependency for Postgres backend + from sqlmodel import Session, create_engine +except ImportError as exc: # pragma: no cover - optional dependency + msg = "sqlmodel is required for Postgres storage support" + raise ImportError(msg) from exc + +logger = logging.getLogger(__name__) + + +class SessionManager: + """Handle engine lifecycle and session creation for Postgres store.""" + + def __init__(self, *, dsn: str, engine_kwargs: dict[str, Any] | None = None) -> None: + kw = {"pool_pre_ping": True} + if engine_kwargs: + kw.update(engine_kwargs) + self._engine = create_engine(dsn, **kw) + + def session(self) -> Session: + return Session(self._engine, expire_on_commit=False) + + def close(self) -> None: + try: + self._engine.dispose() + except Exception: + logger.exception("Failed to close Postgres engine") + + +__all__ = ["SessionManager"] diff --git a/src/memu/database/repositories/__init__.py b/src/memu/database/repositories/__init__.py new file mode 100644 index 0000000..e6016d6 --- /dev/null +++ b/src/memu/database/repositories/__init__.py @@ -0,0 +1,15 @@ +from memu.database.repositories.recall_entry import RecallEntryRepo +from memu.database.repositories.recall_file import RecallFileRepo +from memu.database.repositories.recall_file_entry import RecallFileEntryRepo +from memu.database.repositories.recall_file_resource import RecallFileResourceRepo +from memu.database.repositories.recall_file_segment import RecallFileSegmentRepo +from memu.database.repositories.resource import ResourceRepo + +__all__ = [ + "RecallEntryRepo", + "RecallFileEntryRepo", + "RecallFileRepo", + "RecallFileResourceRepo", + "RecallFileSegmentRepo", + "ResourceRepo", +] diff --git a/src/memu/database/repositories/recall_entry.py b/src/memu/database/repositories/recall_entry.py new file mode 100644 index 0000000..e91d052 --- /dev/null +++ b/src/memu/database/repositories/recall_entry.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Literal, Protocol, runtime_checkable + +from memu.database.models import EntryType, RecallEntry + + +@runtime_checkable +class RecallEntryRepo(Protocol): + """Repository contract for memory items.""" + + items: dict[str, RecallEntry] + + def get_item(self, item_id: str) -> RecallEntry | None: ... + + def list_items(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallEntry]: ... + + def clear_items(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallEntry]: ... + + def create_item( + self, + *, + resource_id: str, + memory_type: EntryType, + summary: str, + embedding: list[float], + user_data: dict[str, Any], + reinforce: bool = False, + tool_record: dict[str, Any] | None = None, + ) -> RecallEntry: ... + + def update_item( + self, + *, + item_id: str, + memory_type: EntryType | None = None, + summary: str | None = None, + embedding: list[float] | None = None, + extra: dict[str, Any] | None = None, + tool_record: dict[str, Any] | None = None, + ) -> RecallEntry: ... + + def delete_item(self, item_id: str) -> None: ... + + def list_items_by_ref_ids( + self, ref_ids: list[str], where: Mapping[str, Any] | None = None + ) -> dict[str, RecallEntry]: ... + + def vector_search_items( + self, + query_vec: list[float], + top_k: int, + where: Mapping[str, Any] | None = None, + *, + ranking: Literal["similarity", "salience"] = "similarity", + recency_decay_days: float = 30.0, + ) -> list[tuple[str, float]]: ... + + def load_existing(self) -> None: ... diff --git a/src/memu/database/repositories/recall_file.py b/src/memu/database/repositories/recall_file.py new file mode 100644 index 0000000..5752e1d --- /dev/null +++ b/src/memu/database/repositories/recall_file.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol, runtime_checkable + +from memu.database.models import RecallFile + + +@runtime_checkable +class RecallFileRepo(Protocol): + """Repository contract for memory categories.""" + + categories: dict[str, RecallFile] + + def list_categories(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallFile]: ... + + def clear_categories(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallFile]: ... + + def get_or_create_category( + self, + *, + name: str, + description: str, + embedding: list[float], + user_data: dict[str, Any], + track: str = "memory", + ) -> RecallFile: ... + + def update_category( + self, + *, + category_id: str, + name: str | None = None, + description: str | None = None, + embedding: list[float] | None = None, + content: str | None = None, + ) -> RecallFile: ... + + def load_existing(self) -> None: ... diff --git a/src/memu/database/repositories/recall_file_entry.py b/src/memu/database/repositories/recall_file_entry.py new file mode 100644 index 0000000..dc1d8a4 --- /dev/null +++ b/src/memu/database/repositories/recall_file_entry.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol, runtime_checkable + +from memu.database.models import RecallFileEntry + + +@runtime_checkable +class RecallFileEntryRepo(Protocol): + """Repository contract for item/category relations.""" + + relations: list[RecallFileEntry] + + def list_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileEntry]: ... + + def link_item_category(self, item_id: str, cat_id: str, user_data: dict[str, Any]) -> RecallFileEntry: ... + + def unlink_item_category(self, item_id: str, cat_id: str) -> None: ... + + def unlink_item(self, item_id: str) -> list[RecallFileEntry]: + """Remove all relations for a given item. Returns the removed relations.""" + ... + + def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileEntry]: + """Remove all relations matching the scope. Returns the removed relations.""" + ... + + def get_item_categories(self, item_id: str) -> list[RecallFileEntry]: ... + + def load_existing(self) -> None: ... diff --git a/src/memu/database/repositories/recall_file_resource.py b/src/memu/database/repositories/recall_file_resource.py new file mode 100644 index 0000000..ae667e6 --- /dev/null +++ b/src/memu/database/repositories/recall_file_resource.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol, runtime_checkable + +from memu.database.models import RecallFileResource + + +@runtime_checkable +class RecallFileResourceRepo(Protocol): + """Repository contract for resource/category relations.""" + + relations: list[RecallFileResource] + + def list_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileResource]: ... + + def link_resource_category( + self, resource_id: str, cat_id: str, user_data: dict[str, Any] + ) -> RecallFileResource: ... + + def unlink_resource_category(self, resource_id: str, cat_id: str) -> None: ... + + def unlink_resource(self, resource_id: str) -> list[RecallFileResource]: + """Remove all relations for a given resource. Returns the removed relations.""" + ... + + def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileResource]: + """Remove all relations matching the scope. Returns the removed relations.""" + ... + + def get_resource_categories(self, resource_id: str) -> list[RecallFileResource]: ... + + def load_existing(self) -> None: ... diff --git a/src/memu/database/repositories/recall_file_segment.py b/src/memu/database/repositories/recall_file_segment.py new file mode 100644 index 0000000..b11a13c --- /dev/null +++ b/src/memu/database/repositories/recall_file_segment.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol, runtime_checkable + +from memu.database.models import RecallFileSegment + + +@runtime_checkable +class RecallFileSegmentRepo(Protocol): + """Repository contract for file segments (searchable L2 slices of a ``RecallFile``).""" + + segments: list[RecallFileSegment] + + def list_segments(self, where: Mapping[str, Any] | None = None) -> list[RecallFileSegment]: ... + + def list_segments_for_file(self, recall_file_id: str) -> list[RecallFileSegment]: + """Return all segments belonging to a given file.""" + ... + + def create_segment( + self, + *, + recall_file_id: str, + text: str, + embedding: list[float] | None, + user_data: dict[str, Any], + track: str = "memory", + ) -> RecallFileSegment: ... + + def delete_segment(self, segment_id: str) -> None: ... + + def delete_segments_for_file(self, recall_file_id: str) -> list[RecallFileSegment]: + """Remove all segments for a given file. Returns the removed segments.""" + ... + + def clear_segments(self, where: Mapping[str, Any] | None = None) -> list[RecallFileSegment]: + """Remove all segments matching the scope. Returns the removed segments.""" + ... + + def load_existing(self) -> None: ... diff --git a/src/memu/database/repositories/resource.py b/src/memu/database/repositories/resource.py new file mode 100644 index 0000000..a88ec01 --- /dev/null +++ b/src/memu/database/repositories/resource.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Protocol, runtime_checkable + +from memu.database.models import Resource + + +@runtime_checkable +class ResourceRepo(Protocol): + """Repository contract for resource records.""" + + resources: dict[str, Resource] + + def list_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, Resource]: ... + + def clear_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, Resource]: ... + + def delete_resource(self, resource_id: str) -> None: ... + + def create_resource( + self, + *, + url: str, + modality: str, + local_path: str, + caption: str | None, + embedding: list[float] | None, + user_data: dict[str, Any], + track: str | None = None, + ) -> Resource: ... + + def vector_search_resources( + self, + query_vec: list[float], + top_k: int, + where: Mapping[str, Any] | None = None, + ) -> list[tuple[str, float]]: + """Rank resources by cosine similarity of their stored embeddings. + + Returns a list of ``(resource_id, score)`` tuples ordered by descending + similarity. Resources without an embedding are skipped. + """ + ... + + def load_existing(self) -> None: ... diff --git a/src/memu/database/sqlite/__init__.py b/src/memu/database/sqlite/__init__.py new file mode 100644 index 0000000..c1a909d --- /dev/null +++ b/src/memu/database/sqlite/__init__.py @@ -0,0 +1,36 @@ +"""SQLite database backend for MemU.""" + +from __future__ import annotations + +from pydantic import BaseModel + +from memu.app.settings import DatabaseConfig +from memu.database.sqlite.sqlite import SQLiteStore + + +def build_sqlite_database( + *, + config: DatabaseConfig, + user_model: type[BaseModel], +) -> SQLiteStore: + """Build a SQLite database store instance. + + Args: + config: Database configuration containing metadata_store settings. + user_model: Pydantic model for user scope fields. + + Returns: + Configured SQLiteStore instance. + """ + dsn = config.metadata_store.dsn + if not dsn: + # Default to a local file if no DSN provided + dsn = "sqlite:///memu.db" + + return SQLiteStore( + dsn=dsn, + scope_model=user_model, + ) + + +__all__ = ["SQLiteStore", "build_sqlite_database"] diff --git a/src/memu/database/sqlite/models.py b/src/memu/database/sqlite/models.py new file mode 100644 index 0000000..6832d61 --- /dev/null +++ b/src/memu/database/sqlite/models.py @@ -0,0 +1,211 @@ +"""SQLite-specific models for MemU database storage.""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +import pendulum +from pydantic import BaseModel +from sqlalchemy import JSON, MetaData, String, Text +from sqlmodel import Column, DateTime, Field, Index, SQLModel, func + +from memu.database.models import ( + EntryType, + RecallEntry, + RecallFile, + RecallFileEntry, + RecallFileResource, + RecallFileSegment, + Resource, +) + + +class TZDateTime(DateTime): + """DateTime type with timezone support.""" + + def __init__(self, timezone: bool = True, **kw: Any) -> None: + super().__init__(timezone=timezone, **kw) + + +class SQLiteBaseModelMixin(SQLModel): + """Base mixin for SQLite models with common fields.""" + + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + primary_key=True, + index=True, + sa_type=String, + ) + created_at: datetime = Field( + default_factory=lambda: pendulum.now("UTC"), + sa_type=TZDateTime, + sa_column_kwargs={"server_default": func.now()}, + ) + updated_at: datetime = Field( + default_factory=lambda: pendulum.now("UTC"), + sa_type=TZDateTime, + ) + + +class SQLiteResourceModel(SQLiteBaseModelMixin, Resource): + """SQLite resource model.""" + + url: str = Field(sa_column=Column(String, nullable=False)) + modality: str = Field(sa_column=Column(String, nullable=False)) + local_path: str = Field(sa_column=Column(String, nullable=False)) + caption: str | None = Field(default=None, sa_column=Column(Text, nullable=True)) + # Override inherited embedding field: SQLite has no native vector type, so store the + # vector in a JSON column (a bare ``list`` annotation is not mappable by SQLModel). + embedding: list[float] | None = Field(default=None, sa_column=Column(JSON, nullable=True)) + track: str | None = Field(default=None, sa_column=Column(String, nullable=True)) + + +class SQLiteRecallEntryModel(SQLiteBaseModelMixin, RecallEntry): + """SQLite memory item model.""" + + resource_id: str | None = Field(sa_column=Column(String, nullable=True)) + memory_type: EntryType = Field(sa_column=Column(String, nullable=False)) + summary: str = Field(sa_column=Column(Text, nullable=False)) + # Override inherited embedding field: SQLite has no native vector type, so store the + # vector in a JSON column (a bare ``list`` annotation is not mappable by SQLModel). + embedding: list[float] | None = Field(default=None, sa_column=Column(JSON, nullable=True)) + happened_at: datetime | None = Field(default=None, sa_column=Column(DateTime, nullable=True)) + extra: dict[str, Any] = Field(default_factory=dict, sa_column=Column(JSON, nullable=True)) + + +class SQLiteRecallFileModel(SQLiteBaseModelMixin, RecallFile): + """SQLite memory category model.""" + + name: str = Field(sa_column=Column(String, nullable=False, index=True)) + description: str = Field(sa_column=Column(Text, nullable=False)) + # Override inherited embedding field: SQLite has no native vector type, so store the + # vector in a JSON column (a bare ``list`` annotation is not mappable by SQLModel). + embedding: list[float] | None = Field(default=None, sa_column=Column(JSON, nullable=True)) + content: str | None = Field(default=None, sa_column=Column(Text, nullable=True)) + track: str = Field(default="memory", sa_column=Column(String, nullable=False, server_default="memory")) + + +class SQLiteRecallFileEntryModel(SQLiteBaseModelMixin, RecallFileEntry): + """SQLite category-item relation model.""" + + item_id: str = Field(sa_column=Column(String, nullable=False)) + category_id: str = Field(sa_column=Column(String, nullable=False)) + + __table_args__ = (Index("idx_sqlite_recall_file_entries_unique", "item_id", "category_id", unique=True),) + + +class SQLiteRecallFileResourceModel(SQLiteBaseModelMixin, RecallFileResource): + """SQLite category-resource relation model.""" + + resource_id: str = Field(sa_column=Column(String, nullable=False)) + file_id: str = Field(sa_column=Column(String, nullable=False)) + + __table_args__ = (Index("idx_sqlite_recall_file_resources_unique", "resource_id", "file_id", unique=True),) + + +class SQLiteRecallFileSegmentModel(SQLiteBaseModelMixin, RecallFileSegment): + """SQLite file-segment model.""" + + recall_file_id: str = Field(sa_column=Column(String, nullable=False, index=True)) + track: str = Field(default="memory", sa_column=Column(String, nullable=False, server_default="memory")) + text: str = Field(sa_column=Column(Text, nullable=False)) + # Override inherited embedding field: SQLite has no native vector type, so store the + # vector in a JSON column (a bare ``list`` annotation is not mappable by SQLModel). + embedding: list[float] | None = Field(default=None, sa_column=Column(JSON, nullable=True)) + + +def _normalize_table_args(table_args: Any) -> tuple[list[Any], dict[str, Any]]: + """Normalize SQLAlchemy table args to a consistent format.""" + if table_args is None: + return [], {} + if isinstance(table_args, dict): + return [], dict(table_args) + if not isinstance(table_args, tuple): + return [table_args], {} + + args = list(table_args) + kwargs: dict[str, Any] = {} + if args and isinstance(args[-1], dict): + kwargs = dict(args.pop()) + return args, kwargs + + +def _merge_models( + user_model: type[BaseModel], + core_model: type[SQLModel], + *, + name_suffix: str, + base_attrs: dict[str, Any], +) -> type[SQLModel]: + """Merge user scope model with core SQLModel.""" + overlap = set(user_model.model_fields) & set(core_model.model_fields) + if overlap: + msg = f"Scope fields conflict with core model fields: {sorted(overlap)}" + raise TypeError(msg) + + return type( + f"{user_model.__name__}{core_model.__name__}{name_suffix}", + (user_model, core_model), + base_attrs, + ) + + +def build_sqlite_table_model( + user_model: type[BaseModel], + core_model: type[SQLModel], + *, + tablename: str, + metadata: MetaData | None = None, + extra_table_args: tuple[Any, ...] | None = None, + unique_with_scope: list[str] | None = None, +) -> type[SQLModel]: + """Build a scoped SQLite table model.""" + overlap = set(user_model.model_fields) & set(core_model.model_fields) + if overlap: + msg = f"Scope fields conflict with core model fields: {sorted(overlap)}" + raise TypeError(msg) + + scope_fields = list(user_model.model_fields.keys()) + base_table_args, table_kwargs = _normalize_table_args(getattr(core_model, "__table_args__", None)) + table_args = list(base_table_args) + if extra_table_args: + table_args.extend(extra_table_args) + if scope_fields: + table_args.append(Index(f"ix_{tablename}__scope", *scope_fields)) + if unique_with_scope: + unique_cols = [*unique_with_scope, *scope_fields] + table_args.append(Index(f"ix_{tablename}__unique_scoped", *unique_cols, unique=True)) + + base_attrs: dict[str, Any] = {"__module__": core_model.__module__, "__tablename__": tablename} + if metadata is not None: + base_attrs["metadata"] = metadata + if table_args or table_kwargs: + if table_kwargs: + base_attrs["__table_args__"] = (*table_args, table_kwargs) + else: + base_attrs["__table_args__"] = tuple(table_args) + + base = _merge_models(user_model, core_model, name_suffix="SQLiteBase", base_attrs=base_attrs) + + # Use type() instead of create_model to properly preserve SQLModel table behavior + table_attrs: dict[str, Any] = {"__module__": core_model.__module__} + return type( + f"{user_model.__name__}{core_model.__name__}SQLiteTable", + (base,), + table_attrs, + table=True, + ) + + +__all__ = [ + "SQLiteBaseModelMixin", + "SQLiteRecallEntryModel", + "SQLiteRecallFileEntryModel", + "SQLiteRecallFileModel", + "SQLiteRecallFileResourceModel", + "SQLiteRecallFileSegmentModel", + "SQLiteResourceModel", + "build_sqlite_table_model", +] diff --git a/src/memu/database/sqlite/repositories/__init__.py b/src/memu/database/sqlite/repositories/__init__.py new file mode 100644 index 0000000..31f980d --- /dev/null +++ b/src/memu/database/sqlite/repositories/__init__.py @@ -0,0 +1,19 @@ +"""SQLite repository implementations for MemU.""" + +from memu.database.sqlite.repositories.base import SQLiteRepoBase +from memu.database.sqlite.repositories.recall_entry_repo import SQLiteRecallEntryRepo +from memu.database.sqlite.repositories.recall_file_entry_repo import SQLiteRecallFileEntryRepo +from memu.database.sqlite.repositories.recall_file_repo import SQLiteRecallFileRepo +from memu.database.sqlite.repositories.recall_file_resource_repo import SQLiteRecallFileResourceRepo +from memu.database.sqlite.repositories.recall_file_segment_repo import SQLiteRecallFileSegmentRepo +from memu.database.sqlite.repositories.resource_repo import SQLiteResourceRepo + +__all__ = [ + "SQLiteRecallEntryRepo", + "SQLiteRecallFileEntryRepo", + "SQLiteRecallFileRepo", + "SQLiteRecallFileResourceRepo", + "SQLiteRecallFileSegmentRepo", + "SQLiteRepoBase", + "SQLiteResourceRepo", +] diff --git a/src/memu/database/sqlite/repositories/base.py b/src/memu/database/sqlite/repositories/base.py new file mode 100644 index 0000000..502d8ed --- /dev/null +++ b/src/memu/database/sqlite/repositories/base.py @@ -0,0 +1,128 @@ +"""Base repository class for SQLite backend.""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Mapping +from typing import Any + +import pendulum + +from memu.database.sqlite.session import SQLiteSessionManager +from memu.database.state import DatabaseState + +logger = logging.getLogger(__name__) + + +class SQLiteRepoBase: + """Base class for SQLite repository implementations.""" + + def __init__( + self, + *, + state: DatabaseState, + sqla_models: Any, + sessions: SQLiteSessionManager, + scope_fields: list[str], + ) -> None: + """Initialize base repository. + + Args: + state: Shared database state for caching. + sqla_models: SQLAlchemy model definitions. + sessions: Session manager for database connections. + scope_fields: List of user scope field names. + """ + self._state = state + self._sqla_models = sqla_models + self._sessions = sessions + self._scope_fields = scope_fields + + def _scope_kwargs_from(self, obj: Any) -> dict[str, Any]: + """Extract scope fields from an object.""" + return {field: getattr(obj, field, None) for field in self._scope_fields} + + def _normalize_embedding(self, embedding: Any) -> list[float] | None: + """Normalize an embedding read from the JSON column to list[float].""" + if embedding is None: + return None + # Legacy/defensive: tolerate a JSON-encoded string if one slipped in. + if isinstance(embedding, str): + try: + return [float(x) for x in json.loads(embedding)] + except (json.JSONDecodeError, TypeError): + logger.debug("Could not parse embedding JSON: %s", embedding) + return None + # Handle list format (the JSON column deserializes to a Python list). + try: + return [float(x) for x in embedding] + except (ValueError, TypeError, OverflowError): + logger.debug("Could not normalize embedding %s", embedding) + return None + + def _prepare_embedding(self, embedding: list[float] | None) -> list[float] | None: + """Return the embedding for storage in the JSON column (no string encoding).""" + if embedding is None: + return None + return list(embedding) + + def _merge_and_commit(self, obj: Any) -> None: + """Merge object into session and commit.""" + with self._sessions.session() as session: + session.merge(obj) + session.commit() + + def _now(self) -> pendulum.DateTime: + """Get current UTC time.""" + return pendulum.now("UTC") + + def _build_filters(self, model: Any, where: Mapping[str, Any] | None) -> list[Any]: + """Build SQLAlchemy filter expressions from where clause.""" + if not where: + return [] + filters: list[Any] = [] + for raw_key, expected in where.items(): + if expected is None: + continue + field, op = [*raw_key.split("__", 1), None][:2] + column = getattr(model, str(field), None) + if column is None: + msg = f"Unknown filter field '{field}' for model '{model.__name__}'" + raise ValueError(msg) + if op == "in": + if isinstance(expected, str): + filters.append(column == expected) + else: + filters.append(column.in_(expected)) + else: + filters.append(column == expected) + return filters + + @staticmethod + def _matches_where(obj: Any, where: Mapping[str, Any] | None) -> bool: + """Check if object matches where clause (for in-memory filtering).""" + if not where: + return True + for raw_key, expected in where.items(): + if expected is None: + continue + field, op = [*raw_key.split("__", 1), None][:2] + actual = getattr(obj, str(field), None) + if op == "in": + if isinstance(expected, str): + if actual != expected: + return False + else: + try: + if actual not in expected: + return False + except TypeError: + return False + else: + if actual != expected: + return False + return True + + +__all__ = ["SQLiteRepoBase"] diff --git a/src/memu/database/sqlite/repositories/recall_entry_repo.py b/src/memu/database/sqlite/repositories/recall_entry_repo.py new file mode 100644 index 0000000..363519d --- /dev/null +++ b/src/memu/database/sqlite/repositories/recall_entry_repo.py @@ -0,0 +1,544 @@ +"""SQLite memory item repository implementation.""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +import pendulum +from sqlmodel import delete, select + +from memu.database.models import EntryType, RecallEntry, compute_content_hash +from memu.database.repositories.recall_entry import RecallEntryRepo +from memu.database.sqlite.repositories.base import SQLiteRepoBase +from memu.database.sqlite.schema import SQLiteSQLAModels +from memu.database.sqlite.session import SQLiteSessionManager +from memu.database.state import DatabaseState +from memu.vector import cosine_topk, cosine_topk_salience + +logger = logging.getLogger(__name__) + + +class SQLiteRecallEntryRepo(SQLiteRepoBase, RecallEntryRepo): + """SQLite implementation of memory item repository.""" + + def __init__( + self, + *, + state: DatabaseState, + recall_entry_model: type[Any], + sqla_models: SQLiteSQLAModels, + sessions: SQLiteSessionManager, + scope_fields: list[str], + ) -> None: + """Initialize memory item repository. + + Args: + state: Shared database state for caching. + recall_entry_model: SQLModel class for memory items. + sqla_models: SQLAlchemy model container. + sessions: Session manager for database connections. + scope_fields: List of user scope field names. + """ + super().__init__( + state=state, + sqla_models=sqla_models, + sessions=sessions, + scope_fields=scope_fields, + ) + self._recall_entry_model = recall_entry_model + self.items = self._state.items + + def get_item(self, item_id: str) -> RecallEntry | None: + """Get a memory item by ID. + + Args: + item_id: The item ID to look up. + + Returns: + RecallEntry if found, None otherwise. + """ + # Check cache first + if item_id in self.items: + return self.items[item_id] + + with self._sessions.session() as session: + stmt = select(self._recall_entry_model).where(self._recall_entry_model.id == item_id) + row = session.exec(stmt).first() + + if row is None: + return None + + item = RecallEntry( + id=row.id, + resource_id=row.resource_id, + memory_type=row.memory_type, + summary=row.summary, + embedding=self._normalize_embedding(row.embedding), + created_at=row.created_at, + updated_at=row.updated_at, + extra=row.extra, + **self._scope_kwargs_from(row), + ) + self.items[row.id] = item + return item + + def list_items(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallEntry]: + """List memory items matching the where clause. + + Args: + where: Optional filter conditions. + + Returns: + Dictionary of item ID to RecallEntry mapping. + """ + with self._sessions.session() as session: + stmt = select(self._recall_entry_model) + filters = self._build_filters(self._recall_entry_model, where) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + result: dict[str, RecallEntry] = {} + for row in rows: + item = RecallEntry( + id=row.id, + resource_id=row.resource_id, + memory_type=row.memory_type, + summary=row.summary, + embedding=self._normalize_embedding(row.embedding), + created_at=row.created_at, + updated_at=row.updated_at, + extra=row.extra, + **self._scope_kwargs_from(row), + ) + result[row.id] = item + self.items[row.id] = item + + return result + + def list_items_by_ref_ids( + self, ref_ids: list[str], where: Mapping[str, Any] | None = None + ) -> dict[str, RecallEntry]: + """List items by their ref_id in the extra column. + + Args: + ref_ids: List of ref_ids to query. + where: Additional filter conditions. + + Returns: + Dict mapping item_id -> RecallEntry for items whose extra.ref_id is in ref_ids. + """ + if not ref_ids: + return {} + + from sqlalchemy import func + + with self._sessions.session() as session: + stmt = select(self._recall_entry_model) + filters = self._build_filters(self._recall_entry_model, where) + # Add filter for json_extract(extra, '$.ref_id') IN ref_ids (only rows with ref_id key) + ref_id_col = func.json_extract(self._recall_entry_model.extra, "$.ref_id") + filters.append(ref_id_col.isnot(None)) + filters.append(ref_id_col.in_(ref_ids)) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + result: dict[str, RecallEntry] = {} + for row in rows: + item = RecallEntry( + id=row.id, + resource_id=row.resource_id, + memory_type=row.memory_type, + summary=row.summary, + embedding=self._normalize_embedding(row.embedding), + created_at=row.created_at, + updated_at=row.updated_at, + extra=row.extra, + **self._scope_kwargs_from(row), + ) + result[row.id] = item + self.items[row.id] = item + + return result + + def clear_items(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallEntry]: + """Clear items matching the where clause. + + Args: + where: Optional filter conditions. + + Returns: + Dictionary of deleted item ID to RecallEntry mapping. + """ + filters = self._build_filters(self._recall_entry_model, where) + with self._sessions.session() as session: + # First get the objects to delete + stmt = select(self._recall_entry_model) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + deleted: dict[str, RecallEntry] = {} + for row in rows: + item = RecallEntry( + id=row.id, + resource_id=row.resource_id, + memory_type=row.memory_type, + summary=row.summary, + embedding=self._normalize_embedding(row.embedding), + created_at=row.created_at, + updated_at=row.updated_at, + extra=row.extra, + **self._scope_kwargs_from(row), + ) + deleted[row.id] = item + + if not deleted: + return {} + + # Delete from database + del_stmt = delete(self._recall_entry_model) + if filters: + del_stmt = del_stmt.where(*filters) + session.exec(del_stmt) + session.commit() + + # Clean up cache + for item_id in deleted: + self.items.pop(item_id, None) + + return deleted + + def create_item( + self, + *, + resource_id: str, + memory_type: EntryType, + summary: str, + embedding: list[float], + user_data: dict[str, Any], + reinforce: bool = False, + tool_record: dict[str, Any] | None = None, + ) -> RecallEntry: + """Create a new memory item. + + Args: + resource_id: Associated resource ID. + memory_type: Type of memory. + summary: Memory summary text. + embedding: Embedding vector. + user_data: User scope data. + reinforce: If True, reinforce existing item instead of creating duplicate. + tool_record: Tool-related fields (when_to_use, metadata, tool_calls) to store in extra. + + Returns: + Created RecallEntry object. + """ + if reinforce and memory_type != "tool": + return self.create_item_reinforce( + resource_id=resource_id, + memory_type=memory_type, + summary=summary, + embedding=embedding, + user_data=user_data, + ) + + # Build extra dict with tool_record fields at top level + extra: dict[str, Any] = {} + if tool_record: + if tool_record.get("when_to_use") is not None: + extra["when_to_use"] = tool_record["when_to_use"] + if tool_record.get("metadata") is not None: + extra["metadata"] = tool_record["metadata"] + if tool_record.get("tool_calls") is not None: + extra["tool_calls"] = tool_record["tool_calls"] + + now = self._now() + row = self._recall_entry_model( + resource_id=resource_id, + memory_type=memory_type, + summary=summary, + embedding=self._prepare_embedding(embedding), + extra=extra if extra else {}, + created_at=now, + updated_at=now, + **user_data, + ) + with self._sessions.session() as session: + session.add(row) + session.commit() + session.refresh(row) + + item = RecallEntry( + id=row.id, + resource_id=row.resource_id, + memory_type=row.memory_type, + summary=row.summary, + embedding=embedding, + extra=row.extra, + created_at=row.created_at, + updated_at=row.updated_at, + **user_data, + ) + self.items[row.id] = item + return item + + def create_item_reinforce( + self, + *, + resource_id: str, + memory_type: EntryType, + summary: str, + embedding: list[float], + user_data: dict[str, Any], + ) -> RecallEntry: + """Create or reinforce a memory item with deduplication. + + If an item with the same content hash exists in the same scope, + reinforce it instead of creating a duplicate. + + Args: + resource_id: Associated resource ID. + memory_type: Type of memory. + summary: Memory summary text. + embedding: Embedding vector. + user_data: User scope data. + + Returns: + Created or reinforced RecallEntry object. + """ + from sqlalchemy import func + + content_hash = compute_content_hash(summary, memory_type) + + with self._sessions.session() as session: + # Check for existing item with same hash in same scope (deduplication) + # Use json_extract(extra, '$.content_hash') for query + content_hash_col = func.json_extract(self._recall_entry_model.extra, "$.content_hash") + filters = [content_hash_col == content_hash] + filters.extend(self._build_filters(self._recall_entry_model, user_data)) + + existing = session.exec(select(self._recall_entry_model).where(*filters)).first() + + if existing: + # Reinforce existing memory instead of creating duplicate + current_extra = existing.extra or {} + current_count = current_extra.get("reinforcement_count", 1) + existing.extra = { + **current_extra, + "reinforcement_count": current_count + 1, + "last_reinforced_at": self._now().isoformat(), + } + existing.updated_at = self._now() + session.add(existing) + session.commit() + session.refresh(existing) + + item = RecallEntry( + id=existing.id, + resource_id=existing.resource_id, + memory_type=existing.memory_type, + summary=existing.summary, + embedding=self._normalize_embedding(existing.embedding), + created_at=existing.created_at, + updated_at=existing.updated_at, + extra=existing.extra, + **self._scope_kwargs_from(existing), + ) + self.items[existing.id] = item + return item + + # Create new item with salience tracking in extra + now = self._now() + item_extra = user_data.pop("extra", {}) if "extra" in user_data else {} + item_extra.update({ + "content_hash": content_hash, + "reinforcement_count": 1, + "last_reinforced_at": now.isoformat(), + }) + + row = self._recall_entry_model( + resource_id=resource_id, + memory_type=memory_type, + summary=summary, + embedding=self._prepare_embedding(embedding), + extra=item_extra, + created_at=now, + updated_at=now, + **user_data, + ) + + session.add(row) + session.commit() + session.refresh(row) + + item = RecallEntry( + id=row.id, + resource_id=row.resource_id, + memory_type=row.memory_type, + summary=row.summary, + embedding=embedding, + created_at=row.created_at, + updated_at=row.updated_at, + extra=row.extra, + **self._scope_kwargs_from(row), + ) + self.items[row.id] = item + return item + + def update_item( + self, + *, + item_id: str, + memory_type: EntryType | None = None, + summary: str | None = None, + embedding: list[float] | None = None, + extra: dict[str, Any] | None = None, + tool_record: dict[str, Any] | None = None, + ) -> RecallEntry: + """Update an existing memory item. + + Args: + item_id: ID of item to update. + memory_type: New memory type (optional). + summary: New summary text (optional). + embedding: New embedding vector (optional). + extra: Extra data to merge into existing extra dict (optional). + tool_record: Tool-related fields (when_to_use, metadata, tool_calls) to merge into extra. + + Returns: + Updated RecallEntry object. + + Raises: + KeyError: If item not found. + """ + with self._sessions.session() as session: + stmt = select(self._recall_entry_model).where(self._recall_entry_model.id == item_id) + row = session.exec(stmt).first() + + if row is None: + msg = f"Item with id {item_id} not found" + raise KeyError(msg) + + if memory_type is not None: + row.memory_type = memory_type + if summary is not None: + row.summary = summary + if embedding is not None: + row.embedding = self._prepare_embedding(embedding) + + # Merge extra and tool_record into existing extra dict + current_extra = row.extra or {} + if extra is not None: + current_extra = {**current_extra, **extra} + if tool_record is not None: + # Merge tool_record fields at top level + for key in ("when_to_use", "metadata", "tool_calls"): + if tool_record.get(key) is not None: + current_extra[key] = tool_record[key] + if extra is not None or tool_record is not None: + row.extra = current_extra + + row.updated_at = self._now() + + session.add(row) + session.commit() + session.refresh(row) + + item = RecallEntry( + id=row.id, + resource_id=row.resource_id, + memory_type=row.memory_type, + summary=row.summary, + embedding=self._normalize_embedding(row.embedding), + extra=row.extra, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + self.items[row.id] = item + return item + + def delete_item(self, item_id: str) -> None: + """Delete a memory item. + + Args: + item_id: ID of item to delete. + """ + with self._sessions.session() as session: + stmt = select(self._recall_entry_model).where(self._recall_entry_model.id == item_id) + row = session.exec(stmt).first() + if row: + session.delete(row) + session.commit() + + if item_id in self.items: + del self.items[item_id] + + def vector_search_items( + self, + query_vec: list[float], + top_k: int, + where: Mapping[str, Any] | None = None, + *, + ranking: str = "similarity", + recency_decay_days: float = 30.0, + ) -> list[tuple[str, float]]: + """Perform vector similarity search on memory items. + + Uses brute-force cosine similarity since SQLite doesn't have native vector support. + + Args: + query_vec: Query embedding vector. + top_k: Maximum number of results to return. + where: Optional filter conditions. + ranking: Ranking strategy - "similarity" (default) or "salience". + recency_decay_days: Half-life for recency decay in salience ranking. + + Returns: + List of (item_id, similarity_score) tuples. + """ + # Load items from database with filters + pool = self.list_items(where) + + if ranking == "salience": + # Salience-aware ranking: similarity x reinforcement x recency + # Read values from extra dict + corpus = [ + ( + i.id, + i.embedding, + (i.extra or {}).get("reinforcement_count", 1), + self._parse_datetime((i.extra or {}).get("last_reinforced_at")), + ) + for i in pool.values() + ] + return cosine_topk_salience(query_vec, corpus, k=top_k, recency_decay_days=recency_decay_days) + + # Default: pure cosine similarity (backward compatible) + hits = cosine_topk(query_vec, [(i.id, i.embedding) for i in pool.values()], k=top_k) + return hits + + @staticmethod + def _parse_datetime(dt_str: str | None) -> pendulum.DateTime | None: + """Parse ISO datetime string from extra dict.""" + if dt_str is None: + return None + try: + parsed = pendulum.parse(dt_str) + except (ValueError, TypeError): + return None + else: + if isinstance(parsed, pendulum.DateTime): + return parsed + return None + + def load_existing(self) -> None: + """Load all existing items from database into cache.""" + self.list_items() + + +__all__ = ["SQLiteRecallEntryRepo"] diff --git a/src/memu/database/sqlite/repositories/recall_file_entry_repo.py b/src/memu/database/sqlite/repositories/recall_file_entry_repo.py new file mode 100644 index 0000000..5d45b12 --- /dev/null +++ b/src/memu/database/sqlite/repositories/recall_file_entry_repo.py @@ -0,0 +1,211 @@ +"""SQLite category-item relation repository implementation.""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +from sqlmodel import select + +from memu.database.models import RecallFileEntry +from memu.database.repositories.recall_file_entry import RecallFileEntryRepo +from memu.database.sqlite.repositories.base import SQLiteRepoBase +from memu.database.sqlite.schema import SQLiteSQLAModels +from memu.database.sqlite.session import SQLiteSessionManager +from memu.database.state import DatabaseState + +logger = logging.getLogger(__name__) + + +class SQLiteRecallFileEntryRepo(SQLiteRepoBase, RecallFileEntryRepo): + """SQLite implementation of category-item relation repository.""" + + def __init__( + self, + *, + state: DatabaseState, + recall_file_entry_model: type[Any], + sqla_models: SQLiteSQLAModels, + sessions: SQLiteSessionManager, + scope_fields: list[str], + ) -> None: + """Initialize category-item repository. + + Args: + state: Shared database state for caching. + recall_file_entry_model: SQLModel class for category-item relations. + sqla_models: SQLAlchemy model container. + sessions: Session manager for database connections. + scope_fields: List of user scope field names. + """ + super().__init__( + state=state, + sqla_models=sqla_models, + sessions=sessions, + scope_fields=scope_fields, + ) + self._recall_file_entry_model = recall_file_entry_model + self.relations = self._state.relations + + def list_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileEntry]: + """List category-item relations matching the where clause. + + Args: + where: Optional filter conditions. + + Returns: + List of RecallFileEntry relations. + """ + with self._sessions.session() as session: + stmt = select(self._recall_file_entry_model) + filters = self._build_filters(self._recall_file_entry_model, where) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + result: list[RecallFileEntry] = [] + for row in rows: + rel = RecallFileEntry( + id=row.id, + item_id=row.item_id, + category_id=row.category_id, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + result.append(rel) + # Update cache + if not any(r.id == rel.id for r in self.relations): + self.relations.append(rel) + + return result + + def link_item_category(self, item_id: str, category_id: str, user_data: dict[str, Any]) -> RecallFileEntry: + """Create a link between an item and a category. + + Args: + item_id: Memory item ID. + category_id: Category ID. + user_data: User scope data. + + Returns: + Created RecallFileEntry relation. + """ + # Check if relation already exists + where: dict[str, Any] = { + "item_id": item_id, + "category_id": category_id, + **user_data, + } + with self._sessions.session() as session: + stmt = select(self._recall_file_entry_model) + filters = self._build_filters(self._recall_file_entry_model, where) + if filters: + stmt = stmt.where(*filters) + existing = session.exec(stmt).first() + + if existing: + rel = RecallFileEntry( + id=existing.id, + item_id=existing.item_id, + category_id=existing.category_id, + created_at=existing.created_at, + updated_at=existing.updated_at, + **self._scope_kwargs_from(existing), + ) + return rel + + # Create new relation + now = self._now() + row = self._recall_file_entry_model( + item_id=item_id, + category_id=category_id, + created_at=now, + updated_at=now, + **user_data, + ) + session.add(row) + session.commit() + session.refresh(row) + + rel = RecallFileEntry( + id=row.id, + item_id=row.item_id, + category_id=row.category_id, + created_at=row.created_at, + updated_at=row.updated_at, + **user_data, + ) + self.relations.append(rel) + return rel + + def unlink_item_category(self, item_id: str, category_id: str) -> None: + """Remove a link between an item and a category. + + Args: + item_id: Memory item ID. + category_id: Category ID. + """ + with self._sessions.session() as session: + stmt = select(self._recall_file_entry_model).where( + self._recall_file_entry_model.item_id == item_id, + self._recall_file_entry_model.category_id == category_id, + ) + row = session.exec(stmt).first() + if row: + session.delete(row) + session.commit() + # Remove from cache + self.relations[:] = [ + r for r in self.relations if not (r.item_id == item_id and r.category_id == category_id) + ] + + def unlink_item(self, item_id: str) -> list[RecallFileEntry]: + """Remove all relations for a given item (used on item deletion).""" + from sqlmodel import delete + + removed = self.list_relations({"item_id": item_id}) + if not removed: + return [] + with self._sessions.session() as session: + session.exec(delete(self._recall_file_entry_model).where(self._recall_file_entry_model.item_id == item_id)) + session.commit() + self.relations[:] = [r for r in self.relations if r.item_id != item_id] + return removed + + def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileEntry]: + """Remove all relations matching the scope (used on clear_memory).""" + from sqlmodel import delete + + removed = self.list_relations(where) + if not removed: + return [] + filters = self._build_filters(self._recall_file_entry_model, where) + with self._sessions.session() as session: + del_stmt = delete(self._recall_file_entry_model) + if filters: + del_stmt = del_stmt.where(*filters) + session.exec(del_stmt) + session.commit() + removed_ids = {rel.id for rel in removed} + self.relations[:] = [r for r in self.relations if r.id not in removed_ids] + return removed + + def get_item_categories(self, item_id: str) -> list[RecallFileEntry]: + """Get all category relations for a given item. + + Args: + item_id: Memory item ID. + + Returns: + List of RecallFileEntry relations for the item. + """ + return self.list_relations({"item_id": item_id}) + + def load_existing(self) -> None: + """Load all existing relations from database into cache.""" + self.list_relations() + + +__all__ = ["SQLiteRecallFileEntryRepo"] diff --git a/src/memu/database/sqlite/repositories/recall_file_repo.py b/src/memu/database/sqlite/repositories/recall_file_repo.py new file mode 100644 index 0000000..00ca24d --- /dev/null +++ b/src/memu/database/sqlite/repositories/recall_file_repo.py @@ -0,0 +1,273 @@ +"""SQLite memory category repository implementation.""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +from sqlmodel import delete, select + +from memu.database.models import RecallFile +from memu.database.repositories.recall_file import RecallFileRepo +from memu.database.sqlite.repositories.base import SQLiteRepoBase +from memu.database.sqlite.schema import SQLiteSQLAModels +from memu.database.sqlite.session import SQLiteSessionManager +from memu.database.state import DatabaseState + +logger = logging.getLogger(__name__) + + +class SQLiteRecallFileRepo(SQLiteRepoBase, RecallFileRepo): + """SQLite implementation of memory category repository.""" + + def __init__( + self, + *, + state: DatabaseState, + recall_file_model: type[Any], + sqla_models: SQLiteSQLAModels, + sessions: SQLiteSessionManager, + scope_fields: list[str], + ) -> None: + """Initialize memory category repository. + + Args: + state: Shared database state for caching. + recall_file_model: SQLModel class for memory categories. + sqla_models: SQLAlchemy model container. + sessions: Session manager for database connections. + scope_fields: List of user scope field names. + """ + super().__init__( + state=state, + sqla_models=sqla_models, + sessions=sessions, + scope_fields=scope_fields, + ) + self._recall_file_model = recall_file_model + self.categories = self._state.categories + + def list_categories(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallFile]: + """List categories matching the where clause. + + Args: + where: Optional filter conditions. + + Returns: + Dictionary of category ID to RecallFile mapping. + """ + with self._sessions.session() as session: + stmt = select(self._recall_file_model) + filters = self._build_filters(self._recall_file_model, where) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + result: dict[str, RecallFile] = {} + for row in rows: + cat = RecallFile( + id=row.id, + name=row.name, + description=row.description, + embedding=self._normalize_embedding(row.embedding), + content=row.content, + track=row.track, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + result[row.id] = cat + self.categories[row.id] = cat + + return result + + def clear_categories(self, where: Mapping[str, Any] | None = None) -> dict[str, RecallFile]: + """Clear categories matching the where clause. + + Args: + where: Optional filter conditions. + + Returns: + Dictionary of deleted category ID to RecallFile mapping. + """ + filters = self._build_filters(self._recall_file_model, where) + with self._sessions.session() as session: + # First get the objects to delete + stmt = select(self._recall_file_model) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + deleted: dict[str, RecallFile] = {} + for row in rows: + cat = RecallFile( + id=row.id, + name=row.name, + description=row.description, + embedding=self._normalize_embedding(row.embedding), + content=row.content, + track=row.track, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + deleted[row.id] = cat + + if not deleted: + return {} + + # Delete from database + del_stmt = delete(self._recall_file_model) + if filters: + del_stmt = del_stmt.where(*filters) + session.exec(del_stmt) + session.commit() + + # Clean up cache + for cat_id in deleted: + self.categories.pop(cat_id, None) + + return deleted + + def get_or_create_category( + self, + *, + name: str, + description: str, + embedding: list[float], + user_data: dict[str, Any], + track: str = "memory", + ) -> RecallFile: + """Get existing category by (name, track, scope) or create a new one. + + Args: + name: Category name. + description: Category description. + embedding: Embedding vector. + user_data: User scope data. + track: Which track the file belongs to ("memory" or "skill"). + + Returns: + Existing or newly created RecallFile. + """ + # Check for existing file with same name, track, and scope + where: dict[str, Any] = {"name": name, "track": track, **user_data} + with self._sessions.session() as session: + stmt = select(self._recall_file_model) + filters = self._build_filters(self._recall_file_model, where) + if filters: + stmt = stmt.where(*filters) + existing = session.exec(stmt).first() + + if existing: + cat = RecallFile( + id=existing.id, + name=existing.name, + description=existing.description, + embedding=self._normalize_embedding(existing.embedding), + content=existing.content, + track=existing.track, + created_at=existing.created_at, + updated_at=existing.updated_at, + **self._scope_kwargs_from(existing), + ) + self.categories[existing.id] = cat + return cat + + # Create new file + now = self._now() + row = self._recall_file_model( + name=name, + description=description, + embedding=self._prepare_embedding(embedding), + content=None, + track=track, + created_at=now, + updated_at=now, + **user_data, + ) + session.add(row) + session.commit() + session.refresh(row) + + cat = RecallFile( + id=row.id, + name=row.name, + description=row.description, + embedding=embedding, + content=None, + track=track, + created_at=row.created_at, + updated_at=row.updated_at, + **user_data, + ) + self.categories[row.id] = cat + return cat + + def update_category( + self, + *, + category_id: str, + name: str | None = None, + description: str | None = None, + embedding: list[float] | None = None, + content: str | None = None, + ) -> RecallFile: + """Update an existing category. + + Args: + category_id: ID of category to update. + name: New name (optional). + description: New description (optional). + embedding: New embedding vector (optional). + content: New content text (optional). + + Returns: + Updated RecallFile object. + + Raises: + KeyError: If category not found. + """ + with self._sessions.session() as session: + stmt = select(self._recall_file_model).where(self._recall_file_model.id == category_id) + row = session.exec(stmt).first() + + if row is None: + msg = f"Category with id {category_id} not found" + raise KeyError(msg) + + if name is not None: + row.name = name + if description is not None: + row.description = description + if embedding is not None: + row.embedding = self._prepare_embedding(embedding) + if content is not None: + row.content = content + row.updated_at = self._now() + + session.add(row) + session.commit() + session.refresh(row) + + cat = RecallFile( + id=row.id, + name=row.name, + description=row.description, + embedding=self._normalize_embedding(row.embedding), + content=row.content, + track=row.track, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + self.categories[row.id] = cat + return cat + + def load_existing(self) -> None: + """Load all existing categories from database into cache.""" + self.list_categories() + + +__all__ = ["SQLiteRecallFileRepo"] diff --git a/src/memu/database/sqlite/repositories/recall_file_resource_repo.py b/src/memu/database/sqlite/repositories/recall_file_resource_repo.py new file mode 100644 index 0000000..a82fd8b --- /dev/null +++ b/src/memu/database/sqlite/repositories/recall_file_resource_repo.py @@ -0,0 +1,215 @@ +"""SQLite category-resource relation repository implementation.""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +from sqlmodel import select + +from memu.database.models import RecallFileResource +from memu.database.repositories.recall_file_resource import RecallFileResourceRepo +from memu.database.sqlite.repositories.base import SQLiteRepoBase +from memu.database.sqlite.schema import SQLiteSQLAModels +from memu.database.sqlite.session import SQLiteSessionManager +from memu.database.state import DatabaseState + +logger = logging.getLogger(__name__) + + +class SQLiteRecallFileResourceRepo(SQLiteRepoBase, RecallFileResourceRepo): + """SQLite implementation of category-resource relation repository.""" + + def __init__( + self, + *, + state: DatabaseState, + recall_file_resource_model: type[Any], + sqla_models: SQLiteSQLAModels, + sessions: SQLiteSessionManager, + scope_fields: list[str], + ) -> None: + """Initialize category-resource repository. + + Args: + state: Shared database state for caching. + recall_file_resource_model: SQLModel class for category-resource relations. + sqla_models: SQLAlchemy model container. + sessions: Session manager for database connections. + scope_fields: List of user scope field names. + """ + super().__init__( + state=state, + sqla_models=sqla_models, + sessions=sessions, + scope_fields=scope_fields, + ) + self._recall_file_resource_model = recall_file_resource_model + self.relations = self._state.resource_relations + + def list_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileResource]: + """List category-resource relations matching the where clause. + + Args: + where: Optional filter conditions. + + Returns: + List of RecallFileResource relations. + """ + with self._sessions.session() as session: + stmt = select(self._recall_file_resource_model) + filters = self._build_filters(self._recall_file_resource_model, where) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + result: list[RecallFileResource] = [] + for row in rows: + rel = RecallFileResource( + id=row.id, + resource_id=row.resource_id, + file_id=row.file_id, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + result.append(rel) + # Update cache + if not any(r.id == rel.id for r in self.relations): + self.relations.append(rel) + + return result + + def link_resource_category(self, resource_id: str, file_id: str, user_data: dict[str, Any]) -> RecallFileResource: + """Create a link between a resource and a category. + + Args: + resource_id: Resource ID. + file_id: File ID. + user_data: User scope data. + + Returns: + Created RecallFileResource relation. + """ + # Check if relation already exists + where: dict[str, Any] = { + "resource_id": resource_id, + "file_id": file_id, + **user_data, + } + with self._sessions.session() as session: + stmt = select(self._recall_file_resource_model) + filters = self._build_filters(self._recall_file_resource_model, where) + if filters: + stmt = stmt.where(*filters) + existing = session.exec(stmt).first() + + if existing: + rel = RecallFileResource( + id=existing.id, + resource_id=existing.resource_id, + file_id=existing.file_id, + created_at=existing.created_at, + updated_at=existing.updated_at, + **self._scope_kwargs_from(existing), + ) + return rel + + # Create new relation + now = self._now() + row = self._recall_file_resource_model( + resource_id=resource_id, + file_id=file_id, + created_at=now, + updated_at=now, + **user_data, + ) + session.add(row) + session.commit() + session.refresh(row) + + rel = RecallFileResource( + id=row.id, + resource_id=row.resource_id, + file_id=row.file_id, + created_at=row.created_at, + updated_at=row.updated_at, + **user_data, + ) + self.relations.append(rel) + return rel + + def unlink_resource_category(self, resource_id: str, file_id: str) -> None: + """Remove a link between a resource and a category. + + Args: + resource_id: Resource ID. + file_id: File ID. + """ + with self._sessions.session() as session: + stmt = select(self._recall_file_resource_model).where( + self._recall_file_resource_model.resource_id == resource_id, + self._recall_file_resource_model.file_id == file_id, + ) + row = session.exec(stmt).first() + if row: + session.delete(row) + session.commit() + # Remove from cache + self.relations[:] = [ + r for r in self.relations if not (r.resource_id == resource_id and r.file_id == file_id) + ] + + def unlink_resource(self, resource_id: str) -> list[RecallFileResource]: + """Remove all relations for a given resource (used on resource deletion).""" + from sqlmodel import delete + + removed = self.list_relations({"resource_id": resource_id}) + if not removed: + return [] + with self._sessions.session() as session: + session.exec( + delete(self._recall_file_resource_model).where( + self._recall_file_resource_model.resource_id == resource_id + ) + ) + session.commit() + self.relations[:] = [r for r in self.relations if r.resource_id != resource_id] + return removed + + def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[RecallFileResource]: + """Remove all relations matching the scope (used on clear_memory).""" + from sqlmodel import delete + + removed = self.list_relations(where) + if not removed: + return [] + filters = self._build_filters(self._recall_file_resource_model, where) + with self._sessions.session() as session: + del_stmt = delete(self._recall_file_resource_model) + if filters: + del_stmt = del_stmt.where(*filters) + session.exec(del_stmt) + session.commit() + removed_ids = {rel.id for rel in removed} + self.relations[:] = [r for r in self.relations if r.id not in removed_ids] + return removed + + def get_resource_categories(self, resource_id: str) -> list[RecallFileResource]: + """Get all category relations for a given resource. + + Args: + resource_id: Resource ID. + + Returns: + List of RecallFileResource relations for the resource. + """ + return self.list_relations({"resource_id": resource_id}) + + def load_existing(self) -> None: + """Load all existing relations from database into cache.""" + self.list_relations() + + +__all__ = ["SQLiteRecallFileResourceRepo"] diff --git a/src/memu/database/sqlite/repositories/recall_file_segment_repo.py b/src/memu/database/sqlite/repositories/recall_file_segment_repo.py new file mode 100644 index 0000000..3ba5714 --- /dev/null +++ b/src/memu/database/sqlite/repositories/recall_file_segment_repo.py @@ -0,0 +1,142 @@ +"""SQLite file-segment repository implementation.""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +from sqlmodel import delete, select + +from memu.database.models import RecallFileSegment +from memu.database.repositories.recall_file_segment import RecallFileSegmentRepo +from memu.database.sqlite.repositories.base import SQLiteRepoBase +from memu.database.sqlite.schema import SQLiteSQLAModels +from memu.database.sqlite.session import SQLiteSessionManager +from memu.database.state import DatabaseState + +logger = logging.getLogger(__name__) + + +class SQLiteRecallFileSegmentRepo(SQLiteRepoBase, RecallFileSegmentRepo): + """SQLite implementation of the file-segment repository.""" + + def __init__( + self, + *, + state: DatabaseState, + recall_file_segment_model: type[Any], + sqla_models: SQLiteSQLAModels, + sessions: SQLiteSessionManager, + scope_fields: list[str], + ) -> None: + super().__init__( + state=state, + sqla_models=sqla_models, + sessions=sessions, + scope_fields=scope_fields, + ) + self._recall_file_segment_model = recall_file_segment_model + self.segments = self._state.segments + + def _row_to_record(self, row: Any) -> RecallFileSegment: + return RecallFileSegment( + id=row.id, + recall_file_id=row.recall_file_id, + track=row.track, + text=row.text, + embedding=self._normalize_embedding(row.embedding), + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + + def list_segments(self, where: Mapping[str, Any] | None = None) -> list[RecallFileSegment]: + with self._sessions.session() as session: + stmt = select(self._recall_file_segment_model) + filters = self._build_filters(self._recall_file_segment_model, where) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + result: list[RecallFileSegment] = [] + for row in rows: + seg = self._row_to_record(row) + result.append(seg) + if not any(s.id == seg.id for s in self.segments): + self.segments.append(seg) + return result + + def list_segments_for_file(self, recall_file_id: str) -> list[RecallFileSegment]: + return self.list_segments({"recall_file_id": recall_file_id}) + + def create_segment( + self, + *, + recall_file_id: str, + text: str, + embedding: list[float] | None, + user_data: dict[str, Any], + track: str = "memory", + ) -> RecallFileSegment: + now = self._now() + with self._sessions.session() as session: + row = self._recall_file_segment_model( + recall_file_id=recall_file_id, + track=track, + text=text, + embedding=self._prepare_embedding(embedding), + created_at=now, + updated_at=now, + **user_data, + ) + session.add(row) + session.commit() + session.refresh(row) + seg = self._row_to_record(row) + + self.segments.append(seg) + return seg + + def delete_segment(self, segment_id: str) -> None: + with self._sessions.session() as session: + session.exec( + delete(self._recall_file_segment_model).where(self._recall_file_segment_model.id == segment_id) + ) + session.commit() + self.segments[:] = [seg for seg in self.segments if seg.id != segment_id] + + def delete_segments_for_file(self, recall_file_id: str) -> list[RecallFileSegment]: + removed = self.list_segments_for_file(recall_file_id) + if not removed: + return [] + with self._sessions.session() as session: + session.exec( + delete(self._recall_file_segment_model).where( + self._recall_file_segment_model.recall_file_id == recall_file_id + ) + ) + session.commit() + self.segments[:] = [seg for seg in self.segments if seg.recall_file_id != recall_file_id] + return removed + + def clear_segments(self, where: Mapping[str, Any] | None = None) -> list[RecallFileSegment]: + removed = self.list_segments(where) + if not removed: + return [] + filters = self._build_filters(self._recall_file_segment_model, where) + with self._sessions.session() as session: + del_stmt = delete(self._recall_file_segment_model) + if filters: + del_stmt = del_stmt.where(*filters) + session.exec(del_stmt) + session.commit() + removed_ids = {seg.id for seg in removed} + self.segments[:] = [seg for seg in self.segments if seg.id not in removed_ids] + return removed + + def load_existing(self) -> None: + self.list_segments() + + +__all__ = ["SQLiteRecallFileSegmentRepo"] diff --git a/src/memu/database/sqlite/repositories/resource_repo.py b/src/memu/database/sqlite/repositories/resource_repo.py new file mode 100644 index 0000000..9dccb70 --- /dev/null +++ b/src/memu/database/sqlite/repositories/resource_repo.py @@ -0,0 +1,224 @@ +"""SQLite resource repository implementation.""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from typing import Any + +from sqlmodel import delete, select + +from memu.database.models import Resource +from memu.database.repositories.resource import ResourceRepo +from memu.database.sqlite.repositories.base import SQLiteRepoBase +from memu.database.sqlite.schema import SQLiteSQLAModels +from memu.database.sqlite.session import SQLiteSessionManager +from memu.database.state import DatabaseState +from memu.vector import cosine_topk + +logger = logging.getLogger(__name__) + + +class SQLiteResourceRepo(SQLiteRepoBase, ResourceRepo): + """SQLite implementation of resource repository.""" + + def __init__( + self, + *, + state: DatabaseState, + resource_model: type[Any], + sqla_models: SQLiteSQLAModels, + sessions: SQLiteSessionManager, + scope_fields: list[str], + ) -> None: + """Initialize resource repository. + + Args: + state: Shared database state for caching. + resource_model: SQLModel class for resources. + sqla_models: SQLAlchemy model container. + sessions: Session manager for database connections. + scope_fields: List of user scope field names. + """ + super().__init__( + state=state, + sqla_models=sqla_models, + sessions=sessions, + scope_fields=scope_fields, + ) + self._resource_model = resource_model + self.resources = self._state.resources + + def list_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, Resource]: + """List resources matching the where clause. + + Args: + where: Optional filter conditions. + + Returns: + Dictionary of resource ID to Resource mapping. + """ + # Prefer cached data if available and no filter + if not where and self.resources: + return dict(self.resources) + + with self._sessions.session() as session: + stmt = select(self._resource_model) + filters = self._build_filters(self._resource_model, where) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + result: dict[str, Resource] = {} + for row in rows: + res = Resource( + id=row.id, + url=row.url, + modality=row.modality, + local_path=row.local_path, + caption=row.caption, + embedding=self._normalize_embedding(row.embedding), + track=row.track, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + result[row.id] = res + self.resources[row.id] = res + + return result + + def clear_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, Resource]: + """Clear resources matching the where clause. + + Args: + where: Optional filter conditions. + + Returns: + Dictionary of deleted resource ID to Resource mapping. + """ + filters = self._build_filters(self._resource_model, where) + with self._sessions.session() as session: + # First get the objects to delete + stmt = select(self._resource_model) + if filters: + stmt = stmt.where(*filters) + rows = session.exec(stmt).all() + + deleted: dict[str, Resource] = {} + for row in rows: + res = Resource( + id=row.id, + url=row.url, + modality=row.modality, + local_path=row.local_path, + caption=row.caption, + embedding=self._normalize_embedding(row.embedding), + track=row.track, + created_at=row.created_at, + updated_at=row.updated_at, + **self._scope_kwargs_from(row), + ) + deleted[row.id] = res + + if not deleted: + return {} + + # Delete from database + del_stmt = delete(self._resource_model) + if filters: + del_stmt = del_stmt.where(*filters) + session.exec(del_stmt) + session.commit() + + # Clean up cache + for res_id in deleted: + self.resources.pop(res_id, None) + + return deleted + + def delete_resource(self, resource_id: str) -> None: + """Delete a single resource by id (used for cascade sync).""" + with self._sessions.session() as session: + session.exec(delete(self._resource_model).where(self._resource_model.id == resource_id)) + session.commit() + self.resources.pop(resource_id, None) + + def create_resource( + self, + *, + url: str, + modality: str, + local_path: str, + caption: str | None, + embedding: list[float] | None, + user_data: dict[str, Any], + track: str | None = None, + ) -> Resource: + """Create a new resource record. + + Args: + url: Resource URL. + modality: Resource modality type. + local_path: Local file path. + caption: Optional caption text. + embedding: Optional embedding vector. + user_data: User scope data. + track: Optional workspace track ("chat"/"skill"/"workspace"). + + Returns: + Created Resource object. + """ + now = self._now() + row = self._resource_model( + url=url, + modality=modality, + local_path=local_path, + caption=caption, + embedding=self._prepare_embedding(embedding), + track=track, + created_at=now, + updated_at=now, + **user_data, + ) + with self._sessions.session() as session: + session.add(row) + session.commit() + session.refresh(row) + + res = Resource( + id=row.id, + url=row.url, + modality=row.modality, + local_path=row.local_path, + caption=row.caption, + embedding=embedding, + track=row.track, + created_at=row.created_at, + updated_at=row.updated_at, + **user_data, + ) + self.resources[row.id] = res + return res + + def vector_search_resources( + self, + query_vec: list[float], + top_k: int, + where: Mapping[str, Any] | None = None, + ) -> list[tuple[str, float]]: + """Rank resources by brute-force cosine similarity. + + SQLite has no native vector support, so this mirrors the item repo and + scores stored embeddings in Python. + """ + pool = self.list_resources(where) + corpus = [(rid, res.embedding) for rid, res in pool.items() if res.embedding] + return cosine_topk(query_vec, corpus, k=top_k) + + def load_existing(self) -> None: + """Load all existing resources from database into cache.""" + self.list_resources() + + +__all__ = ["SQLiteResourceRepo"] diff --git a/src/memu/database/sqlite/schema.py b/src/memu/database/sqlite/schema.py new file mode 100644 index 0000000..fc3b67f --- /dev/null +++ b/src/memu/database/sqlite/schema.py @@ -0,0 +1,126 @@ +"""SQLAlchemy schema definitions for SQLite backend.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from pydantic import BaseModel +from sqlalchemy import MetaData +from sqlmodel import SQLModel + +from memu.database.sqlite.models import ( + SQLiteRecallEntryModel, + SQLiteRecallFileEntryModel, + SQLiteRecallFileModel, + SQLiteRecallFileResourceModel, + SQLiteRecallFileSegmentModel, + SQLiteResourceModel, + build_sqlite_table_model, +) + + +@dataclass +class SQLiteSQLAModels: + """Container for SQLite SQLAlchemy/SQLModel models.""" + + Base: type[Any] + Resource: type[Any] + RecallFile: type[Any] + RecallEntry: type[Any] + RecallFileEntry: type[Any] + RecallFileResource: type[Any] + RecallFileSegment: type[Any] + + +_MODEL_CACHE: dict[type[Any], SQLiteSQLAModels] = {} + + +def get_sqlite_sqlalchemy_models(*, scope_model: type[BaseModel] | None = None) -> SQLiteSQLAModels: + """Build (and cache) SQLModel ORM models for SQLite storage. + + Args: + scope_model: Optional Pydantic model defining user scope fields. + + Returns: + SQLiteSQLAModels containing all table models. + """ + scope = scope_model or BaseModel + cache_key = scope + cached = _MODEL_CACHE.get(cache_key) + if cached: + return cached + + metadata_obj = MetaData() + + # NOTE: SQLite reserves any table name beginning with "sqlite_", so the tables + # must use a different prefix. + resource_model = build_sqlite_table_model( + scope, + SQLiteResourceModel, + tablename="memu_resources", + metadata=metadata_obj, + ) + recall_file_model = build_sqlite_table_model( + scope, + SQLiteRecallFileModel, + tablename="memu_memory_categories", + metadata=metadata_obj, + ) + recall_entry_model = build_sqlite_table_model( + scope, + SQLiteRecallEntryModel, + tablename="memu_memory_items", + metadata=metadata_obj, + ) + recall_file_entry_model = build_sqlite_table_model( + scope, + SQLiteRecallFileEntryModel, + tablename="memu_category_items", + metadata=metadata_obj, + ) + recall_file_resource_model = build_sqlite_table_model( + scope, + SQLiteRecallFileResourceModel, + tablename="memu_resource_categories", + metadata=metadata_obj, + ) + recall_file_segment_model = build_sqlite_table_model( + scope, + SQLiteRecallFileSegmentModel, + tablename="memu_file_segments", + metadata=metadata_obj, + ) + + class SQLiteBase(SQLModel): + __abstract__ = True + metadata = metadata_obj + + models = SQLiteSQLAModels( + Base=SQLiteBase, + Resource=resource_model, + RecallFile=recall_file_model, + RecallEntry=recall_entry_model, + RecallFileEntry=recall_file_entry_model, + RecallFileResource=recall_file_resource_model, + RecallFileSegment=recall_file_segment_model, + ) + _MODEL_CACHE[cache_key] = models + return models + + +def get_sqlite_metadata(scope_model: type[BaseModel] | None = None) -> MetaData: + """Get SQLAlchemy metadata for SQLite tables. + + Args: + scope_model: Optional Pydantic model defining user scope fields. + + Returns: + SQLAlchemy MetaData object. + """ + from typing import cast + + return cast(MetaData, get_sqlite_sqlalchemy_models(scope_model=scope_model).Base.metadata) + + +__all__ = ["SQLiteSQLAModels", "get_sqlite_metadata", "get_sqlite_sqlalchemy_models"] diff --git a/src/memu/database/sqlite/session.py b/src/memu/database/sqlite/session.py new file mode 100644 index 0000000..7aa1aee --- /dev/null +++ b/src/memu/database/sqlite/session.py @@ -0,0 +1,48 @@ +"""SQLite session manager for database connections.""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy.exc import SQLAlchemyError +from sqlmodel import Session, create_engine + +logger = logging.getLogger(__name__) + + +class SQLiteSessionManager: + """Handle engine lifecycle and session creation for SQLite store.""" + + def __init__(self, *, dsn: str, engine_kwargs: dict[str, Any] | None = None) -> None: + """Initialize SQLite session manager. + + Args: + dsn: SQLite connection string (e.g., "sqlite:///path/to/db.sqlite"). + engine_kwargs: Optional keyword arguments for create_engine. + """ + kw: dict[str, Any] = { + "connect_args": {"check_same_thread": False}, # Allow multi-threaded access + } + if engine_kwargs: + kw.update(engine_kwargs) + self._engine = create_engine(dsn, **kw) + + def session(self) -> Session: + """Create a new database session.""" + return Session(self._engine, expire_on_commit=False) + + def close(self) -> None: + """Close the database engine and release resources.""" + try: + self._engine.dispose() + except SQLAlchemyError: + logger.exception("Failed to close SQLite engine") + + @property + def engine(self) -> Any: + """Return the underlying SQLAlchemy engine.""" + return self._engine + + +__all__ = ["SQLiteSessionManager"] diff --git a/src/memu/database/sqlite/sqlite.py b/src/memu/database/sqlite/sqlite.py new file mode 100644 index 0000000..2c3ebbf --- /dev/null +++ b/src/memu/database/sqlite/sqlite.py @@ -0,0 +1,187 @@ +"""SQLite database store implementation for MemU.""" + +from __future__ import annotations + +import logging +from typing import Any + +from pydantic import BaseModel +from sqlmodel import SQLModel + +from memu.database.interfaces import Database +from memu.database.models import ( + RecallEntry, + RecallFile, + RecallFileEntry, + RecallFileResource, + RecallFileSegment, + Resource, +) +from memu.database.repositories import ( + RecallEntryRepo, + RecallFileEntryRepo, + RecallFileRepo, + RecallFileResourceRepo, + RecallFileSegmentRepo, + ResourceRepo, +) +from memu.database.sqlite.repositories.recall_entry_repo import SQLiteRecallEntryRepo +from memu.database.sqlite.repositories.recall_file_entry_repo import SQLiteRecallFileEntryRepo +from memu.database.sqlite.repositories.recall_file_repo import SQLiteRecallFileRepo +from memu.database.sqlite.repositories.recall_file_resource_repo import SQLiteRecallFileResourceRepo +from memu.database.sqlite.repositories.recall_file_segment_repo import SQLiteRecallFileSegmentRepo +from memu.database.sqlite.repositories.resource_repo import SQLiteResourceRepo +from memu.database.sqlite.schema import SQLiteSQLAModels, get_sqlite_sqlalchemy_models +from memu.database.sqlite.session import SQLiteSessionManager +from memu.database.state import DatabaseState + +logger = logging.getLogger(__name__) + + +class SQLiteStore(Database): + """SQLite database store implementation. + + This store provides a lightweight, file-based database backend for MemU. + It uses SQLite for metadata storage and brute-force cosine similarity + for vector search (native vector support is not available in SQLite). + + Attributes: + resource_repo: Repository for resource records. + recall_file_repo: Repository for memory categories. + recall_entry_repo: Repository for memory items. + recall_file_entry_repo: Repository for category-item relations. + resources: Dict cache of resource records. + items: Dict cache of memory item records. + categories: Dict cache of memory category records. + relations: List cache of category-item relations. + """ + + resource_repo: ResourceRepo + recall_file_repo: RecallFileRepo + recall_entry_repo: RecallEntryRepo + recall_file_entry_repo: RecallFileEntryRepo + recall_file_resource_repo: RecallFileResourceRepo + recall_file_segment_repo: RecallFileSegmentRepo + resources: dict[str, Resource] + items: dict[str, RecallEntry] + categories: dict[str, RecallFile] + relations: list[RecallFileEntry] + resource_relations: list[RecallFileResource] + segments: list[RecallFileSegment] + + def __init__( + self, + *, + dsn: str, + scope_model: type[BaseModel] | None = None, + resource_model: type[Any] | None = None, + recall_file_model: type[Any] | None = None, + recall_entry_model: type[Any] | None = None, + recall_file_entry_model: type[Any] | None = None, + recall_file_resource_model: type[Any] | None = None, + recall_file_segment_model: type[Any] | None = None, + sqla_models: SQLiteSQLAModels | None = None, + ) -> None: + """Initialize SQLite database store. + + Args: + dsn: SQLite connection string (e.g., "sqlite:///path/to/db.sqlite"). + scope_model: Pydantic model defining user scope fields. + resource_model: Optional custom resource model. + recall_file_model: Optional custom memory category model. + recall_entry_model: Optional custom memory item model. + recall_file_entry_model: Optional custom category-item model. + sqla_models: Pre-built SQLAlchemy models container. + """ + self.dsn = dsn + self._scope_model: type[BaseModel] = scope_model or BaseModel + self._scope_fields = list(getattr(self._scope_model, "model_fields", {}).keys()) + self._state = DatabaseState() + self._sessions = SQLiteSessionManager(dsn=self.dsn) + self._sqla_models: SQLiteSQLAModels = sqla_models or get_sqlite_sqlalchemy_models(scope_model=self._scope_model) + + # Create tables + self._create_tables() + + # Use provided models or defaults from sqla_models + resource_model = resource_model or self._sqla_models.Resource + recall_file_model = recall_file_model or self._sqla_models.RecallFile + recall_entry_model = recall_entry_model or self._sqla_models.RecallEntry + recall_file_entry_model = recall_file_entry_model or self._sqla_models.RecallFileEntry + recall_file_resource_model = recall_file_resource_model or self._sqla_models.RecallFileResource + recall_file_segment_model = recall_file_segment_model or self._sqla_models.RecallFileSegment + + # Initialize repositories + self.resource_repo = SQLiteResourceRepo( + state=self._state, + resource_model=resource_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + self.recall_file_repo = SQLiteRecallFileRepo( + state=self._state, + recall_file_model=recall_file_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + self.recall_entry_repo = SQLiteRecallEntryRepo( + state=self._state, + recall_entry_model=recall_entry_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + self.recall_file_entry_repo = SQLiteRecallFileEntryRepo( + state=self._state, + recall_file_entry_model=recall_file_entry_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + self.recall_file_resource_repo = SQLiteRecallFileResourceRepo( + state=self._state, + recall_file_resource_model=recall_file_resource_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + self.recall_file_segment_repo = SQLiteRecallFileSegmentRepo( + state=self._state, + recall_file_segment_model=recall_file_segment_model, + sqla_models=self._sqla_models, + sessions=self._sessions, + scope_fields=self._scope_fields, + ) + + # Set up cache references + self.resources = self._state.resources + self.items = self._state.items + self.categories = self._state.categories + self.relations = self._state.relations + self.resource_relations = self._state.resource_relations + self.segments = self._state.segments + + def _create_tables(self) -> None: + """Create SQLite tables if they don't exist.""" + SQLModel.metadata.create_all(self._sessions.engine) + # Also create tables from our custom metadata + self._sqla_models.Base.metadata.create_all(self._sessions.engine) + logger.debug("SQLite tables created/verified") + + def close(self) -> None: + """Close the database connection and release resources.""" + self._sessions.close() + + def load_existing(self) -> None: + """Load all existing data from database into cache.""" + self.resource_repo.load_existing() + self.recall_file_repo.load_existing() + self.recall_entry_repo.load_existing() + self.recall_file_entry_repo.load_existing() + self.recall_file_resource_repo.load_existing() + self.recall_file_segment_repo.load_existing() + + +__all__ = ["SQLiteStore"] diff --git a/src/memu/database/state.py b/src/memu/database/state.py new file mode 100644 index 0000000..a17297a --- /dev/null +++ b/src/memu/database/state.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + +from memu.database.models import ( + RecallEntry, + RecallFile, + RecallFileEntry, + RecallFileResource, + RecallFileSegment, + Resource, +) + + +@dataclass +class DatabaseState: + resources: dict[str, Resource] = field(default_factory=dict) + items: dict[str, RecallEntry] = field(default_factory=dict) + categories: dict[str, RecallFile] = field(default_factory=dict) + relations: list[RecallFileEntry] = field(default_factory=list) + resource_relations: list[RecallFileResource] = field(default_factory=list) + segments: list[RecallFileSegment] = field(default_factory=list) + + +__all__ = ["DatabaseState"] diff --git a/src/memu/embedding/__init__.py b/src/memu/embedding/__init__.py new file mode 100644 index 0000000..ee6e82a --- /dev/null +++ b/src/memu/embedding/__init__.py @@ -0,0 +1,32 @@ +"""Embedding (vectorization) clients. + +Sibling package to :mod:`memu.llm` and :mod:`memu.vlm`, scoped to the embedding +capability used by vector search. It mirrors their package layout: + +- ``backends/``: per-provider embedding request/response shapes (HTTP transport). +- ``http_client``/``openai_sdk``: transport clients. +- ``gateway``: build a client from a :class:`memu.app.settings.EmbeddingConfig`. +- ``defaults``: per-provider default embedding models / endpoints. +""" + +from __future__ import annotations + +from memu.embedding.base import EmbeddingClient +from memu.embedding.defaults import ( + EMBEDDING_PROVIDER_DEFAULTS, + EMBEDDING_PROVIDER_ENDPOINTS, + default_embedding_model, +) +from memu.embedding.gateway import build_embedding_client +from memu.embedding.http_client import HTTPEmbeddingClient +from memu.embedding.openai_sdk import OpenAIEmbeddingSDKClient + +__all__ = [ + "EMBEDDING_PROVIDER_DEFAULTS", + "EMBEDDING_PROVIDER_ENDPOINTS", + "EmbeddingClient", + "HTTPEmbeddingClient", + "OpenAIEmbeddingSDKClient", + "build_embedding_client", + "default_embedding_model", +] diff --git a/src/memu/embedding/backends/__init__.py b/src/memu/embedding/backends/__init__.py new file mode 100644 index 0000000..bed0537 --- /dev/null +++ b/src/memu/embedding/backends/__init__.py @@ -0,0 +1,16 @@ +from memu.embedding.backends.base import EmbeddingBackend +from memu.embedding.backends.doubao import DoubaoEmbeddingBackend, DoubaoMultimodalEmbeddingInput +from memu.embedding.backends.jina import JinaEmbeddingBackend +from memu.embedding.backends.openai import OpenAIEmbeddingBackend +from memu.embedding.backends.openrouter import OpenRouterEmbeddingBackend +from memu.embedding.backends.voyage import VoyageEmbeddingBackend + +__all__ = [ + "DoubaoEmbeddingBackend", + "DoubaoMultimodalEmbeddingInput", + "EmbeddingBackend", + "JinaEmbeddingBackend", + "OpenAIEmbeddingBackend", + "OpenRouterEmbeddingBackend", + "VoyageEmbeddingBackend", +] diff --git a/src/memu/embedding/backends/base.py b/src/memu/embedding/backends/base.py new file mode 100644 index 0000000..5dfcb38 --- /dev/null +++ b/src/memu/embedding/backends/base.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import Any + + +class EmbeddingBackend: + """Defines how to talk to a specific embedding provider. + + Each provider lives in its own module under :mod:`memu.embedding.backends` + and customizes the request endpoint, payload shape, response parsing and + (when needed) the auth headers. Mirrors :class:`memu.llm.backends.base.LLMBackend`. + """ + + name: str = "base" + embedding_endpoint: str = "/embeddings" + + def default_headers(self, api_key: str) -> dict[str, str]: + """Auth/request headers for this provider. + + Defaults to OpenAI-style bearer auth; providers with a different scheme + override this. + """ + return {"Authorization": f"Bearer {api_key}"} + + def build_embedding_payload(self, *, inputs: list[str], embed_model: str) -> dict[str, Any]: + raise NotImplementedError + + def parse_embedding_response(self, data: dict[str, Any]) -> list[list[float]]: + raise NotImplementedError diff --git a/src/memu/embedding/backends/doubao.py b/src/memu/embedding/backends/doubao.py new file mode 100644 index 0000000..ae8a4ae --- /dev/null +++ b/src/memu/embedding/backends/doubao.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from typing import Any, Literal, cast + +from memu.embedding.backends.base import EmbeddingBackend + + +class DoubaoMultimodalEmbeddingInput: + """Represents a single input item for multimodal embedding.""" + + def __init__( + self, + input_type: Literal["text", "image_url", "video_url"], + content: str, + ): + self.input_type = input_type + self.content = content + + def to_dict(self) -> dict[str, Any]: + if self.input_type == "text": + return {"type": "text", "text": self.content} + elif self.input_type == "image_url": + return {"type": "image_url", "image_url": {"url": self.content}} + elif self.input_type == "video_url": + return {"type": "video_url", "video_url": {"url": self.content}} + else: + msg = f"Unsupported input type: {self.input_type}" + raise ValueError(msg) + + +class DoubaoEmbeddingBackend(EmbeddingBackend): + """Backend for Doubao embedding API (including multimodal embedding).""" + + name = "doubao" + embedding_endpoint = "/api/v3/embeddings" + multimodal_embedding_endpoint = "/api/v3/embeddings/multimodal" + + def build_embedding_payload(self, *, inputs: list[str], embed_model: str) -> dict[str, Any]: + """Build payload for standard text embeddings.""" + return {"model": embed_model, "input": inputs, "encoding_format": "float"} + + def parse_embedding_response(self, data: dict[str, Any]) -> list[list[float]]: + """Parse embedding response.""" + return [cast(list[float], d["embedding"]) for d in data["data"]] + + def build_multimodal_embedding_payload( + self, + *, + inputs: list[DoubaoMultimodalEmbeddingInput], + embed_model: str, + encoding_format: str = "float", + ) -> dict[str, Any]: + """ + Build payload for multimodal embedding API. + + Args: + inputs: List of multimodal inputs (text, image_url, video_url) + embed_model: Model name (e.g., 'doubao-embedding-vision-250615') + encoding_format: Encoding format ('float' or 'base64') + + Returns: + Request payload dict + """ + return { + "model": embed_model, + "encoding_format": encoding_format, + "input": [inp.to_dict() for inp in inputs], + } + + def parse_multimodal_embedding_response(self, data: dict[str, Any]) -> list[list[float]]: + """Parse multimodal embedding response.""" + return [cast(list[float], d["embedding"]) for d in data["data"]] diff --git a/src/memu/embedding/backends/jina.py b/src/memu/embedding/backends/jina.py new file mode 100644 index 0000000..3b30a58 --- /dev/null +++ b/src/memu/embedding/backends/jina.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.embedding.backends.base import EmbeddingBackend + + +class JinaEmbeddingBackend(EmbeddingBackend): + """Backend for Jina AI embedding API. + + Jina exposes an OpenAI-compatible ``/embeddings`` surface (bearer auth, + ``{"model", "input"}`` payload, ``data[].embedding`` response) at + ``https://api.jina.ai/v1``. See https://jina.ai/embeddings/. + """ + + name = "jina" + embedding_endpoint = "/embeddings" + + def build_embedding_payload(self, *, inputs: list[str], embed_model: str) -> dict[str, Any]: + return {"model": embed_model, "input": inputs} + + def parse_embedding_response(self, data: dict[str, Any]) -> list[list[float]]: + return [cast(list[float], d["embedding"]) for d in data["data"]] diff --git a/src/memu/embedding/backends/openai.py b/src/memu/embedding/backends/openai.py new file mode 100644 index 0000000..d2b5259 --- /dev/null +++ b/src/memu/embedding/backends/openai.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.embedding.backends.base import EmbeddingBackend + + +class OpenAIEmbeddingBackend(EmbeddingBackend): + """Backend for OpenAI-compatible embedding API.""" + + name = "openai" + embedding_endpoint = "/embeddings" + + def build_embedding_payload(self, *, inputs: list[str], embed_model: str) -> dict[str, Any]: + return {"model": embed_model, "input": inputs} + + def parse_embedding_response(self, data: dict[str, Any]) -> list[list[float]]: + return [cast(list[float], d["embedding"]) for d in data["data"]] diff --git a/src/memu/embedding/backends/openrouter.py b/src/memu/embedding/backends/openrouter.py new file mode 100644 index 0000000..21b2e44 --- /dev/null +++ b/src/memu/embedding/backends/openrouter.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.embedding.backends.base import EmbeddingBackend + + +class OpenRouterEmbeddingBackend(EmbeddingBackend): + """Backend for OpenRouter's OpenAI-compatible embedding API. + + OpenRouter serves embeddings at ``/api/v1/embeddings`` relative to + ``https://openrouter.ai``. + """ + + name = "openrouter" + embedding_endpoint = "/api/v1/embeddings" + + def build_embedding_payload(self, *, inputs: list[str], embed_model: str) -> dict[str, Any]: + return {"model": embed_model, "input": inputs} + + def parse_embedding_response(self, data: dict[str, Any]) -> list[list[float]]: + return [cast(list[float], d["embedding"]) for d in data["data"]] diff --git a/src/memu/embedding/backends/voyage.py b/src/memu/embedding/backends/voyage.py new file mode 100644 index 0000000..0937240 --- /dev/null +++ b/src/memu/embedding/backends/voyage.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.embedding.backends.base import EmbeddingBackend + + +class VoyageEmbeddingBackend(EmbeddingBackend): + """Backend for Voyage AI embedding API. + + Voyage exposes an OpenAI-compatible ``/embeddings`` surface (bearer auth, + ``{"model", "input"}`` payload, ``data[].embedding`` response) at + ``https://api.voyageai.com/v1``. See https://docs.voyageai.com/. + """ + + name = "voyage" + embedding_endpoint = "/embeddings" + + def build_embedding_payload(self, *, inputs: list[str], embed_model: str) -> dict[str, Any]: + return {"model": embed_model, "input": inputs} + + def parse_embedding_response(self, data: dict[str, Any]) -> list[list[float]]: + return [cast(list[float], d["embedding"]) for d in data["data"]] diff --git a/src/memu/embedding/base.py b/src/memu/embedding/base.py new file mode 100644 index 0000000..5bca8b1 --- /dev/null +++ b/src/memu/embedding/base.py @@ -0,0 +1,26 @@ +"""Shared types for embedding (vectorization) clients. + +Embedding clients expose a single capability, :meth:`EmbeddingClient.embed`, +which turns a batch of texts into dense vectors. Each transport (official SDK, +raw HTTP, or framework backend) implements this surface so it can be +wrapped/swapped like the text :mod:`memu.llm` and vision :mod:`memu.vlm` clients. +""" + +from __future__ import annotations + +from typing import Any + + +class EmbeddingClient: + """Base interface for embedding clients.""" + + embed_model: str + + async def embed(self, inputs: list[str]) -> tuple[list[list[float]], Any]: + """Embed ``inputs`` and return ``(vectors, raw_response)``. + + ``raw_response`` carries provider usage metadata (token counts) so the + :class:`memu.llm.wrapper.LLMClientWrapper` can record consumption; it may + be ``None`` for transports that do not expose usage. + """ + raise NotImplementedError diff --git a/src/memu/embedding/defaults.py b/src/memu/embedding/defaults.py new file mode 100644 index 0000000..474616e --- /dev/null +++ b/src/memu/embedding/defaults.py @@ -0,0 +1,30 @@ +"""Per-provider default embedding models and endpoints. + +Maps a provider identifier to its latest general-purpose text embedding model, +used by :class:`memu.app.settings.EmbeddingConfig` to pick a sensible default. +Embedding-only providers (Jina, Voyage) also need their own base URL / API key +env, since they are absent from the shared chat ``_PROVIDER_DEFAULTS`` table. +Verified via provider docs, June 2026. +""" + +from __future__ import annotations + +EMBEDDING_PROVIDER_DEFAULTS: dict[str, str] = { + "openai": "text-embedding-3-small", + "jina": "jina-embeddings-v3", + "voyage": "voyage-3.5", + "doubao": "doubao-embedding-large-text-250515", + "openrouter": "openai/text-embedding-3-small", +} + +# base_url + API key env for embedding-only providers (not chat providers, so +# they are not in ``memu.app.settings._PROVIDER_DEFAULTS``). +EMBEDDING_PROVIDER_ENDPOINTS: dict[str, tuple[str, str]] = { + "jina": ("https://api.jina.ai/v1", "JINA_API_KEY"), + "voyage": ("https://api.voyageai.com/v1", "VOYAGE_API_KEY"), +} + + +def default_embedding_model(provider: str) -> str | None: + """Return the default embedding model for ``provider`` (``None`` if unknown).""" + return EMBEDDING_PROVIDER_DEFAULTS.get(provider.lower()) diff --git a/src/memu/embedding/gateway.py b/src/memu/embedding/gateway.py new file mode 100644 index 0000000..528f57b --- /dev/null +++ b/src/memu/embedding/gateway.py @@ -0,0 +1,78 @@ +"""Embedding gateway: select and build a transport-specific embedding client. + +Mirrors :mod:`memu.llm.gateway` and :mod:`memu.vlm.gateway`. Adding a new +embedding transport means registering a builder in ``EMBEDDING_CLIENT_BUILDERS`` +here rather than editing the service composition root. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from memu.app.settings import EmbeddingConfig + + +def _build_sdk_client(cfg: EmbeddingConfig) -> Any: + from memu.embedding.openai_sdk import OpenAIEmbeddingSDKClient + + return OpenAIEmbeddingSDKClient( + base_url=cfg.base_url, + api_key=cfg.api_key, + embed_model=cfg.embed_model, + batch_size=cfg.embed_batch_size, + ) + + +def _build_httpx_client(cfg: EmbeddingConfig) -> Any: + from memu.embedding.http_client import HTTPEmbeddingClient + + return HTTPEmbeddingClient( + base_url=cfg.base_url, + api_key=cfg.api_key, + embed_model=cfg.embed_model, + provider=cfg.provider, + endpoint_overrides=cfg.endpoint_overrides, + ) + + +def _build_lazyllm_client(cfg: EmbeddingConfig) -> Any: + from memu.llm.lazyllm_client import LazyLLMClient + + source = cfg.lazyllm_source + return LazyLLMClient( + embed_source=source.embed_source or source.source, + embed_model=cfg.embed_model, + ) + + +def _build_anthropic_client(cfg: EmbeddingConfig) -> Any: + msg = ( + "Anthropic does not provide an embeddings API. Configure an embedding " + "profile with a dedicated provider (e.g. 'openai', 'jina', 'voyage')." + ) + raise ValueError(msg) + + +# Registry mapping ``client_backend`` identifiers to embedding client builders. +EMBEDDING_CLIENT_BUILDERS: dict[str, Callable[[EmbeddingConfig], Any]] = { + "sdk": _build_sdk_client, + "httpx": _build_httpx_client, + "lazyllm_backend": _build_lazyllm_client, + "anthropic": _build_anthropic_client, +} + + +def build_embedding_client(cfg: EmbeddingConfig) -> Any: + """Build an embedding client for ``cfg.client_backend``. + + Raises: + ValueError: if ``cfg.client_backend`` is not registered. + """ + builder = EMBEDDING_CLIENT_BUILDERS.get(cfg.client_backend) + if builder is None: + available = ", ".join(sorted(EMBEDDING_CLIENT_BUILDERS)) + msg = f"Unknown embedding client_backend '{cfg.client_backend}'. Available: {available}" + raise ValueError(msg) + return builder(cfg) diff --git a/src/memu/embedding/http_client.py b/src/memu/embedding/http_client.py new file mode 100644 index 0000000..04bc11f --- /dev/null +++ b/src/memu/embedding/http_client.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import logging +import os +from collections.abc import Callable +from typing import Any, Literal + +import httpx + +from memu.embedding.backends.base import EmbeddingBackend +from memu.embedding.backends.doubao import DoubaoEmbeddingBackend, DoubaoMultimodalEmbeddingInput +from memu.embedding.backends.jina import JinaEmbeddingBackend +from memu.embedding.backends.openai import OpenAIEmbeddingBackend +from memu.embedding.backends.openrouter import OpenRouterEmbeddingBackend +from memu.embedding.backends.voyage import VoyageEmbeddingBackend + + +def _load_proxy() -> str | None: + return os.getenv("MEMU_HTTP_PROXY") or os.getenv("HTTP_PROXY") or os.getenv("HTTPS_PROXY") or None + + +logger = logging.getLogger(__name__) + +# Providers with a non-OpenAI endpoint/payload are registered explicitly; any +# other provider is treated as OpenAI-compatible (see ``_load_backend``). +EMBEDDING_BACKENDS: dict[str, Callable[[], EmbeddingBackend]] = { + OpenAIEmbeddingBackend.name: OpenAIEmbeddingBackend, + DoubaoEmbeddingBackend.name: DoubaoEmbeddingBackend, + JinaEmbeddingBackend.name: JinaEmbeddingBackend, + VoyageEmbeddingBackend.name: VoyageEmbeddingBackend, + OpenRouterEmbeddingBackend.name: OpenRouterEmbeddingBackend, +} + + +class HTTPEmbeddingClient: + """HTTP client for embedding APIs.""" + + def __init__( + self, + *, + base_url: str, + api_key: str, + embed_model: str, + provider: str = "openai", + endpoint_overrides: dict[str, str] | None = None, + timeout: int = 60, + ): + # Ensure base_url ends with "/" so httpx doesn't discard the path + # component when joining with endpoint paths. + # See: https://github.com/NevaMind-AI/memU/issues/328 + self.base_url = base_url.rstrip("/") + "/" + self.api_key = api_key or "" + self.embed_model = embed_model + self.provider = provider.lower() + self.backend = self._load_backend(self.provider) + overrides = endpoint_overrides or {} + raw_embedding_ep = ( + overrides.get("embeddings") + or overrides.get("embedding") + or overrides.get("embed") + or self.backend.embedding_endpoint + ) + # Strip leading "/" so httpx resolves relative to base_url + self.embedding_endpoint = raw_embedding_ep.lstrip("/") + self.timeout = timeout + self.proxy = _load_proxy() + + async def embed(self, inputs: list[str]) -> tuple[list[list[float]], dict[str, Any]]: + """ + Create text embeddings. + + Args: + inputs: List of text strings to embed + + Returns: + Tuple of (list of embedding vectors, raw response dict). The raw + response carries provider ``usage`` so callers/interceptors can + track token consumption. + """ + payload = self.backend.build_embedding_payload(inputs=inputs, embed_model=self.embed_model) + async with httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout, proxy=self.proxy) as client: + resp = await client.post(self.embedding_endpoint, json=payload, headers=self._headers()) + resp.raise_for_status() + data = resp.json() + logger.debug("HTTP embedding response: %s", data) + return self.backend.parse_embedding_response(data), data + + async def embed_multimodal( + self, + inputs: list[tuple[Literal["text", "image_url", "video_url"], str]], + *, + encoding_format: str = "float", + ) -> list[list[float]]: + """ + Create multimodal embeddings using Doubao embedding vision API. + + This method supports embedding a mix of text, images, and videos in a single request. + Only available when using the 'doubao' provider. + + Args: + inputs: List of tuples where each tuple contains: + - input_type: One of 'text', 'image_url', or 'video_url' + - content: The text content or URL to the image/video + encoding_format: Encoding format for the embeddings ('float' or 'base64') + + Returns: + List of embedding vectors + + Example: + >>> client = HTTPEmbeddingClient( + ... base_url="https://ark.cn-beijing.volces.com", + ... api_key="your-api-key", + ... embed_model="doubao-embedding-vision-250615", + ... provider="doubao", + ... ) + >>> embeddings = await client.embed_multimodal([ + ... ("text", "What is in the image and video?"), + ... ("image_url", "https://example.com/image.png"), + ... ("video_url", "https://example.com/video.mp4"), + ... ]) + + Raises: + TypeError: If the backend does not support multimodal embeddings + """ + if not isinstance(self.backend, DoubaoEmbeddingBackend): + msg = ( + f"Multimodal embedding is only supported by 'doubao' provider, " + f"but current provider is '{self.provider}'" + ) + raise TypeError(msg) + + multimodal_inputs = [ + DoubaoMultimodalEmbeddingInput(input_type=input_type, content=content) for input_type, content in inputs + ] + + payload = self.backend.build_multimodal_embedding_payload( + inputs=multimodal_inputs, + embed_model=self.embed_model, + encoding_format=encoding_format, + ) + + endpoint = self.backend.multimodal_embedding_endpoint.lstrip("/") + async with httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout, proxy=self.proxy) as client: + resp = await client.post(endpoint, json=payload, headers=self._headers()) + resp.raise_for_status() + data = resp.json() + + logger.debug("HTTP multimodal embedding response: %s", data) + return self.backend.parse_multimodal_embedding_response(data) + + def _headers(self) -> dict[str, str]: + return self.backend.default_headers(self.api_key) + + def _load_backend(self, provider: str) -> EmbeddingBackend: + # Providers with a non-standard endpoint/payload are registered + # explicitly; everything else is treated as OpenAI-compatible (the most + # common case, e.g. grok/deepseek/kimi/minimax). + factory = EMBEDDING_BACKENDS.get(provider, OpenAIEmbeddingBackend) + return factory() diff --git a/src/memu/embedding/openai_sdk.py b/src/memu/embedding/openai_sdk.py new file mode 100644 index 0000000..85ec419 --- /dev/null +++ b/src/memu/embedding/openai_sdk.py @@ -0,0 +1,47 @@ +import logging +from typing import cast + +from openai import AsyncOpenAI +from openai.types import CreateEmbeddingResponse + +logger = logging.getLogger(__name__) + + +class OpenAIEmbeddingSDKClient: + """OpenAI embedding client that relies on the official Python SDK.""" + + def __init__(self, *, base_url: str, api_key: str, embed_model: str, batch_size: int = 1): + self.base_url = base_url.rstrip("/") + self.api_key = api_key or "" + self.embed_model = embed_model + self.batch_size = batch_size + self.client = AsyncOpenAI(api_key=self.api_key, base_url=self.base_url) + + async def embed(self, inputs: list[str]) -> tuple[list[list[float]], CreateEmbeddingResponse | None]: + """ + Create text embeddings via the official SDK. + + Args: + inputs: List of text strings to embed + + Returns: + Tuple of (list of embedding vectors, last raw response). The raw + response carries token ``usage`` so callers/interceptors can track + consumption. For batched requests only the last response is returned. + """ + # Process in batches to handle API limits (e.g., some providers limit batch size) + if len(inputs) <= self.batch_size: + # Single batch - direct call + response = await self.client.embeddings.create(model=self.embed_model, input=inputs) + return [cast(list[float], d.embedding) for d in response.data], response + + # Multiple batches - split and merge + all_embeddings: list[list[float]] = [] + last_response: CreateEmbeddingResponse | None = None + for i in range(0, len(inputs), self.batch_size): + batch = inputs[i : i + self.batch_size] + response = await self.client.embeddings.create(model=self.embed_model, input=batch) + all_embeddings.extend([cast(list[float], d.embedding) for d in response.data]) + last_response = response + + return all_embeddings, last_response diff --git a/src/memu/integrations/__init__.py b/src/memu/integrations/__init__.py new file mode 100644 index 0000000..dd2390a --- /dev/null +++ b/src/memu/integrations/__init__.py @@ -0,0 +1,3 @@ +from .langgraph import MemULangGraphTools + +__all__ = ["MemULangGraphTools"] diff --git a/src/memu/integrations/langgraph.py b/src/memu/integrations/langgraph.py new file mode 100644 index 0000000..2e24ddc --- /dev/null +++ b/src/memu/integrations/langgraph.py @@ -0,0 +1,163 @@ +"""LangGraph integration for MemU.""" + +from __future__ import annotations + +import contextlib +import logging +import os +import tempfile +import uuid +from typing import Any + +# MUST explicitly import langgraph to satisfy DEP002 +import langgraph +from pydantic import BaseModel, Field + +from memu.app.service import MemoryService + +try: + from langchain_core.tools import BaseTool, StructuredTool +except ImportError as e: + msg = "Please install 'langchain-core' (and 'langgraph') to use the LangGraph integration." + raise ImportError(msg) from e + + +# Setup logger +logger = logging.getLogger("memu.integrations.langgraph") + + +class MemUIntegrationError(Exception): + """Base exception for MemU integration issues.""" + + +class SaveRecallInput(BaseModel): + """Input schema for the save_memory tool.""" + + content: str = Field(description="The text content or information to save/remember.") + user_id: str = Field(description="The unique identifier of the user.") + metadata: dict[str, Any] | None = Field(default=None, description="Additional metadata related to the memory.") + + +class SearchRecallInput(BaseModel): + """Input schema for the search_memory tool.""" + + query: str = Field(description="The search query to retrieve relevant memories.") + user_id: str = Field(description="The unique identifier of the user.") + limit: int = Field(default=5, description="Number of memories to retrieve.") + metadata_filter: dict[str, Any] | None = Field( + default=None, description="Optional filter for memory metadata (e.g., {'category': 'work'})." + ) + min_relevance_score: float = Field(default=0.0, description="Minimum relevance score (0.0 to 1.0) for results.") + + +class MemULangGraphTools: + """Adapter to expose MemU as a set of Tools for LangGraph/LangChain agents. + + This class provides a bridge between the MemU MemoryService and LangChain's + tooling ecosystem. + """ + + def __init__(self, memory_service: MemoryService): + """Initializes the MemULangGraphTools with a memory service.""" + self.memory_service = memory_service + # Expose the langgraph module to ensure it's "used" even if just by reference in this class + self._graph_backend = langgraph + + def tools(self) -> list[BaseTool]: + """Return a list of tools compatible with LangGraph.""" + return [ + self.save_memory_tool(), + self.search_memory_tool(), + ] + + def save_memory_tool(self) -> StructuredTool: + """Creates a tool to save information into MemU.""" + + async def _save(content: str, user_id: str, metadata: dict | None = None) -> str: + logger.info("Entering save_memory_tool for user_id: %s", user_id) + filename = f"memu_input_{uuid.uuid4()}.txt" + temp_dir = tempfile.gettempdir() + file_path = os.path.join(temp_dir, filename) + + try: + with open(file_path, "w", encoding="utf-8") as f: + f.write(content) + + logger.debug("Calling memory_service.memorize with temporary file: %s", file_path) + await self.memory_service.memorize( + resource_url=file_path, + modality="conversation", + user={"user_id": user_id, **(metadata or {})}, + ) + logger.info("Successfully saved memory for user_id: %s", user_id) + except Exception as e: + error_msg = f"Failed to save memory for user {user_id}: {e!s}" + logger.exception(error_msg) + return str(MemUIntegrationError(error_msg)) + finally: + if os.path.exists(file_path): + with contextlib.suppress(OSError): + os.remove(file_path) + logger.debug("Cleaned up temporary file: %s", file_path) + + return "Memory saved successfully." + + return StructuredTool.from_function( + func=None, + coroutine=_save, + name="save_memory", + description="Save a piece of information, conversation snippet, or memory for a user.", + args_schema=SaveRecallInput, + ) + + def search_memory_tool(self) -> StructuredTool: + """Creates a tool to search for information in MemU.""" + + async def _search( + query: str, + user_id: str, + limit: int = 5, + metadata_filter: dict | None = None, + min_relevance_score: float = 0.0, + ) -> str: + logger.info("Entering search_memory_tool for user_id: %s, query: '%s'", user_id, query) + try: + queries = [{"role": "user", "content": query}] + where_filter = {"user_id": user_id} + if metadata_filter: + where_filter.update(metadata_filter) + + logger.debug("Calling memory_service.retrieve with where_filter: %s", where_filter) + result = await self.memory_service.retrieve( + queries=queries, + where=where_filter, + ) + logger.info("Successfully retrieved memories for user_id: %s", user_id) + except Exception as e: + error_msg = f"Failed to search memory for user {user_id}: {e!s}" + logger.exception(error_msg) + return str(MemUIntegrationError(error_msg)) + + items = result.get("items", []) + if min_relevance_score > 0: + items = [item for item in items if item.get("score", 1.0) >= min_relevance_score] + + if not items: + logger.info("No memories found for user_id: %s", user_id) + return "No relevant memories found." + + response_text = "Retrieved Memories:\n" + for idx, item in enumerate(items[:limit]): + summary = item.get("summary", "") + score = item.get("score", "N/A") + response_text += f"{idx + 1}. [Score: {score}] {summary}\n" + + return response_text + + return StructuredTool.from_function( + func=None, + coroutine=_search, + name="search_memory", + description="Search for relevant memories or information for a user based on a query.", + args_schema=SearchRecallInput, + ) diff --git a/src/memu/llm/anthropic_client.py b/src/memu/llm/anthropic_client.py new file mode 100644 index 0000000..52464d8 --- /dev/null +++ b/src/memu/llm/anthropic_client.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import base64 +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from anthropic.types import Message + +logger = logging.getLogger(__name__) + +# Anthropic requires max_tokens on every request; fall back to this when omitted. +_DEFAULT_MAX_TOKENS = 1024 + +_MIME_BY_SUFFIX = { + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", +} + + +class AnthropicClient: + """Claude LLM client that relies on the official Anthropic Python SDK. + + Mirrors the surface of :class:`memu.llm.openai_client.OpenAIClient` so it can + be wrapped by :class:`memu.llm.wrapper.LLMClientWrapper`. Anthropic does not + offer audio transcription, so that method raises. Embedding is handled by the + dedicated :mod:`memu.embedding` clients. + """ + + def __init__( + self, + *, + base_url: str | None, + api_key: str, + chat_model: str, + max_tokens: int | None = None, + ): + # Imported lazily so the optional ``anthropic`` dependency is only + # required when this client is actually used. + from anthropic import AsyncAnthropic + + self.base_url = base_url.rstrip("/") if base_url else None + self.api_key = api_key or "" + self.chat_model = chat_model + self.max_tokens = max_tokens or _DEFAULT_MAX_TOKENS + self.client = AsyncAnthropic(api_key=self.api_key, base_url=self.base_url) + + async def chat( + self, + prompt: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + temperature: float = 0.2, + ) -> tuple[str, Message]: + """Generic chat completion via the Messages API.""" + kwargs: dict[str, Any] = { + "model": self.chat_model, + "max_tokens": max_tokens or self.max_tokens, + "temperature": temperature, + "messages": [{"role": "user", "content": prompt}], + } + if system_prompt is not None: + kwargs["system"] = system_prompt + + response = await self.client.messages.create(**kwargs) + logger.debug("Anthropic chat response: %s", response) + return _extract_text(response), response + + async def summarize( + self, + text: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, Message]: + system = system_prompt or "Summarize the text in one short paragraph." + response = await self.client.messages.create( + model=self.chat_model, + max_tokens=max_tokens or self.max_tokens, + system=system, + messages=[{"role": "user", "content": text}], + ) + logger.debug("Anthropic summarize response: %s", response) + return _extract_text(response), response + + async def vision( + self, + prompt: str, + image_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, Message]: + """Analyze an image alongside a text prompt via the Messages API.""" + image_data = Path(image_path).read_bytes() + base64_image = base64.b64encode(image_data).decode("utf-8") + mime_type = _MIME_BY_SUFFIX.get(Path(image_path).suffix.lower(), "image/jpeg") + + kwargs: dict[str, Any] = { + "model": self.chat_model, + "max_tokens": max_tokens or self.max_tokens, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": mime_type, + "data": base64_image, + }, + }, + ], + } + ], + } + if system_prompt is not None: + kwargs["system"] = system_prompt + + response = await self.client.messages.create(**kwargs) + logger.debug("Anthropic vision response: %s", response) + return _extract_text(response), response + + async def transcribe( + self, + audio_path: str, + *, + prompt: str | None = None, + language: str | None = None, + response_format: str = "text", + ) -> tuple[str, None]: + msg = "Anthropic does not provide an audio transcription API." + raise NotImplementedError(msg) + + +def _extract_text(response: Message) -> str: + """Concatenate text blocks from an Anthropic Messages response.""" + parts: list[str] = [] + for block in response.content: + if getattr(block, "type", None) == "text": + parts.append(getattr(block, "text", "")) + return "".join(parts) diff --git a/src/memu/llm/backends/__init__.py b/src/memu/llm/backends/__init__.py new file mode 100644 index 0000000..2a80b89 --- /dev/null +++ b/src/memu/llm/backends/__init__.py @@ -0,0 +1,21 @@ +from memu.llm.backends.base import LLMBackend +from memu.llm.backends.claude import ClaudeLLMBackend +from memu.llm.backends.deepseek import DeepSeekLLMBackend +from memu.llm.backends.doubao import DoubaoLLMBackend +from memu.llm.backends.grok import GrokBackend +from memu.llm.backends.kimi import KimiLLMBackend +from memu.llm.backends.minimax import MiniMaxLLMBackend +from memu.llm.backends.openai import OpenAILLMBackend +from memu.llm.backends.openrouter import OpenRouterLLMBackend + +__all__ = [ + "ClaudeLLMBackend", + "DeepSeekLLMBackend", + "DoubaoLLMBackend", + "GrokBackend", + "KimiLLMBackend", + "LLMBackend", + "MiniMaxLLMBackend", + "OpenAILLMBackend", + "OpenRouterLLMBackend", +] diff --git a/src/memu/llm/backends/base.py b/src/memu/llm/backends/base.py new file mode 100644 index 0000000..8763621 --- /dev/null +++ b/src/memu/llm/backends/base.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Any + + +class LLMBackend: + """Defines how to talk to a specific HTTP LLM provider. + + Each provider lives in its own module under :mod:`memu.llm.backends` and + customizes the request endpoint, payload shape, response parsing and (when + needed) the auth headers. + """ + + name: str = "base" + summary_endpoint: str = "/chat/completions" + + def default_headers(self, api_key: str) -> dict[str, str]: + """Auth/request headers for this provider. + + Defaults to OpenAI-style bearer auth; providers with a different scheme + (e.g. Anthropic's ``x-api-key``) override this. + """ + return {"Authorization": f"Bearer {api_key}"} + + def build_summary_payload( + self, *, text: str, system_prompt: str | None, chat_model: str, max_tokens: int | None + ) -> dict[str, Any]: + raise NotImplementedError + + def parse_summary_response(self, data: dict[str, Any]) -> str: + raise NotImplementedError + + def build_vision_payload( + self, + *, + prompt: str, + base64_image: str, + mime_type: str, + system_prompt: str | None, + chat_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + raise NotImplementedError diff --git a/src/memu/llm/backends/claude.py b/src/memu/llm/backends/claude.py new file mode 100644 index 0000000..938f870 --- /dev/null +++ b/src/memu/llm/backends/claude.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.llm.backends.base import LLMBackend + +# Anthropic requires max_tokens; fall back to this when the caller omits it. +_DEFAULT_MAX_TOKENS = 1024 +_ANTHROPIC_VERSION = "2023-06-01" + + +class ClaudeLLMBackend(LLMBackend): + """Backend for Anthropic Claude (native Messages API). + + Unlike the OpenAI-compatible providers, Claude uses ``x-api-key`` auth, a + top-level ``system`` field, a required ``max_tokens``, and a different + response/vision shape. + + Default base_url: ``https://api.anthropic.com`` with model + ``claude-haiku-4-5``. + """ + + name = "claude" + summary_endpoint = "/v1/messages" + + def default_headers(self, api_key: str) -> dict[str, str]: + return { + "x-api-key": api_key, + "anthropic-version": _ANTHROPIC_VERSION, + "content-type": "application/json", + } + + def build_summary_payload( + self, *, text: str, system_prompt: str | None, chat_model: str, max_tokens: int | None + ) -> dict[str, Any]: + payload: dict[str, Any] = { + "model": chat_model, + "max_tokens": max_tokens or _DEFAULT_MAX_TOKENS, + "messages": [{"role": "user", "content": text}], + } + if system_prompt: + payload["system"] = system_prompt + return payload + + def parse_summary_response(self, data: dict[str, Any]) -> str: + blocks = data.get("content") or [] + for block in blocks: + if isinstance(block, dict) and block.get("type") == "text": + return cast(str, block.get("text", "")) + return "" + + def build_vision_payload( + self, + *, + prompt: str, + base64_image: str, + mime_type: str, + system_prompt: str | None, + chat_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + payload: dict[str, Any] = { + "model": chat_model, + "max_tokens": max_tokens or _DEFAULT_MAX_TOKENS, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": mime_type, + "data": base64_image, + }, + }, + ], + } + ], + } + if system_prompt: + payload["system"] = system_prompt + return payload diff --git a/src/memu/llm/backends/deepseek.py b/src/memu/llm/backends/deepseek.py new file mode 100644 index 0000000..b003e55 --- /dev/null +++ b/src/memu/llm/backends/deepseek.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from memu.llm.backends.openai import OpenAILLMBackend + + +class DeepSeekLLMBackend(OpenAILLMBackend): + """Backend for DeepSeek LLM API (OpenAI-compatible). + + Default base_url: ``https://api.deepseek.com/v1`` with model ``deepseek-v4-flash``. + Inherits OpenAI's payload/response handling. + """ + + name = "deepseek" + summary_endpoint = "/chat/completions" diff --git a/src/memu/llm/backends/doubao.py b/src/memu/llm/backends/doubao.py new file mode 100644 index 0000000..9dd7012 --- /dev/null +++ b/src/memu/llm/backends/doubao.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.llm.backends.base import LLMBackend + + +class DoubaoLLMBackend(LLMBackend): + """Backend for Doubao LLM API.""" + + name = "doubao" + summary_endpoint = "/api/v3/chat/completions" + + def build_summary_payload( + self, *, text: str, system_prompt: str | None, chat_model: str, max_tokens: int | None + ) -> dict[str, Any]: + """Build payload for Doubao chat completions (OpenAI-compatible).""" + prompt = system_prompt or "Summarize the text in one short paragraph." + payload: dict[str, Any] = { + "model": chat_model, + "messages": [ + {"role": "system", "content": prompt}, + {"role": "user", "content": text}, + ], + "temperature": 0.2, + } + if max_tokens is not None: + payload["max_tokens"] = max_tokens + return payload + + def parse_summary_response(self, data: dict[str, Any]) -> str: + return cast(str, data["choices"][0]["message"]["content"]) + + def build_vision_payload( + self, + *, + prompt: str, + base64_image: str, + mime_type: str, + system_prompt: str | None, + chat_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + """Build payload for Doubao Vision API (OpenAI-compatible).""" + messages: list[dict[str, Any]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + + messages.append({ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{base64_image}", + }, + }, + ], + }) + + payload: dict[str, Any] = { + "model": chat_model, + "messages": messages, + "temperature": 0.2, + } + if max_tokens is not None: + payload["max_tokens"] = max_tokens + return payload diff --git a/src/memu/llm/backends/grok.py b/src/memu/llm/backends/grok.py new file mode 100644 index 0000000..cdcffdc --- /dev/null +++ b/src/memu/llm/backends/grok.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from memu.llm.backends.openai import OpenAILLMBackend + + +class GrokBackend(OpenAILLMBackend): + """Backend for Grok (xAI) LLM API.""" + + name = "grok" + # Grok uses the same payload structure as OpenAI + # We inherits build_summary_payload, parse_summary_response, etc. diff --git a/src/memu/llm/backends/kimi.py b/src/memu/llm/backends/kimi.py new file mode 100644 index 0000000..f5e260e --- /dev/null +++ b/src/memu/llm/backends/kimi.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from memu.llm.backends.openai import OpenAILLMBackend + + +class KimiLLMBackend(OpenAILLMBackend): + """Backend for Kimi / Moonshot LLM API (OpenAI-compatible). + + Default base_url: ``https://api.moonshot.cn/v1`` with model ``kimi-k2.7-code-highspeed``. + Inherits OpenAI's payload/response handling. + """ + + name = "kimi" + summary_endpoint = "/chat/completions" diff --git a/src/memu/llm/backends/minimax.py b/src/memu/llm/backends/minimax.py new file mode 100644 index 0000000..2801d2b --- /dev/null +++ b/src/memu/llm/backends/minimax.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from memu.llm.backends.openai import OpenAILLMBackend + + +class MiniMaxLLMBackend(OpenAILLMBackend): + """Backend for MiniMax LLM API (OpenAI-compatible chat completions v2). + + Default base_url: ``https://api.minimax.io/v1`` with model ``MiniMax-M3``. + MiniMax exposes an OpenAI-compatible chat endpoint at ``/text/chatcompletion_v2``. + """ + + name = "minimax" + summary_endpoint = "/text/chatcompletion_v2" diff --git a/src/memu/llm/backends/openai.py b/src/memu/llm/backends/openai.py new file mode 100644 index 0000000..4b322cf --- /dev/null +++ b/src/memu/llm/backends/openai.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.llm.backends.base import LLMBackend + + +class OpenAILLMBackend(LLMBackend): + """Backend for OpenAI-compatible LLM API.""" + + name = "openai" + summary_endpoint = "/chat/completions" + + def build_summary_payload( + self, *, text: str, system_prompt: str | None, chat_model: str, max_tokens: int | None + ) -> dict[str, Any]: + prompt = system_prompt or "Summarize the text in one short paragraph." + payload: dict[str, Any] = { + "model": chat_model, + "messages": [ + {"role": "system", "content": prompt}, + {"role": "user", "content": text}, + ], + "temperature": 0.2, + } + # Omit when unset: newer models reject ``max_tokens=None``. + if max_tokens is not None: + payload["max_tokens"] = max_tokens + return payload + + def parse_summary_response(self, data: dict[str, Any]) -> str: + return cast(str, data["choices"][0]["message"]["content"]) + + def build_vision_payload( + self, + *, + prompt: str, + base64_image: str, + mime_type: str, + system_prompt: str | None, + chat_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + """Build payload for OpenAI Vision API.""" + messages: list[dict[str, Any]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + + messages.append({ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{base64_image}", + }, + }, + ], + }) + + payload: dict[str, Any] = { + "model": chat_model, + "messages": messages, + "temperature": 0.2, + } + # Omit when unset: newer models reject ``max_tokens=None``. + if max_tokens is not None: + payload["max_tokens"] = max_tokens + return payload diff --git a/src/memu/llm/backends/openrouter.py b/src/memu/llm/backends/openrouter.py new file mode 100644 index 0000000..1a8cdee --- /dev/null +++ b/src/memu/llm/backends/openrouter.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.llm.backends.base import LLMBackend + + +class OpenRouterLLMBackend(LLMBackend): + """Backend for OpenRouter LLM API (OpenAI-compatible).""" + + name = "openrouter" + summary_endpoint = "/api/v1/chat/completions" + + def build_summary_payload( + self, *, text: str, system_prompt: str | None, chat_model: str, max_tokens: int | None + ) -> dict[str, Any]: + """Build payload for OpenRouter chat completions (OpenAI-compatible).""" + prompt = system_prompt or "Summarize the text in one short paragraph." + payload: dict[str, Any] = { + "model": chat_model, + "messages": [ + {"role": "system", "content": prompt}, + {"role": "user", "content": text}, + ], + "temperature": 0.2, + } + if max_tokens is not None: + payload["max_tokens"] = max_tokens + return payload + + def parse_summary_response(self, data: dict[str, Any]) -> str: + """Parse OpenRouter response (OpenAI-compatible format).""" + return cast(str, data["choices"][0]["message"]["content"]) + + def build_vision_payload( + self, + *, + prompt: str, + base64_image: str, + mime_type: str, + system_prompt: str | None, + chat_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + """Build payload for OpenRouter Vision API (OpenAI-compatible).""" + messages: list[dict[str, Any]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + + messages.append({ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{base64_image}", + }, + }, + ], + }) + + payload: dict[str, Any] = { + "model": chat_model, + "messages": messages, + "temperature": 0.2, + } + if max_tokens is not None: + payload["max_tokens"] = max_tokens + return payload diff --git a/src/memu/llm/gateway.py b/src/memu/llm/gateway.py new file mode 100644 index 0000000..25fe083 --- /dev/null +++ b/src/memu/llm/gateway.py @@ -0,0 +1,90 @@ +"""LLM gateway: select and build a transport-specific LLM client. + +This module centralizes the dispatch from configuration (``LLMConfig``) to a +concrete client implementation under :mod:`memu.llm`. Adding a new LLM +implementation means registering a builder in ``LLM_CLIENT_BUILDERS`` here rather +than editing the service composition root. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from memu.app.settings import LLMConfig + + +def _build_sdk_client(cfg: LLMConfig) -> Any: + from memu.llm.openai_client import OpenAIClient + + return OpenAIClient( + base_url=cfg.base_url, + api_key=cfg.api_key, + chat_model=cfg.chat_model, + ) + + +def _build_anthropic_client(cfg: LLMConfig) -> Any: + from memu.llm.anthropic_client import AnthropicClient + + # The OpenAI default base_url is meaningless for Anthropic; let the SDK use + # its own default (https://api.anthropic.com) unless explicitly overridden. + base_url = None if cfg.base_url == "https://api.openai.com/v1" else cfg.base_url + return AnthropicClient( + base_url=base_url, + api_key=cfg.api_key, + chat_model=cfg.chat_model, + ) + + +def _build_httpx_client(cfg: LLMConfig) -> Any: + from memu.llm.http_client import HTTPLLMClient + + return HTTPLLMClient( + base_url=cfg.base_url, + api_key=cfg.api_key, + chat_model=cfg.chat_model, + provider=cfg.provider, + endpoint_overrides=cfg.endpoint_overrides, + ) + + +def _build_lazyllm_client(cfg: LLMConfig) -> Any: + from memu.llm.lazyllm_client import LazyLLMClient + + source = cfg.lazyllm_source + return LazyLLMClient( + llm_source=source.llm_source or source.source, + vlm_source=source.vlm_source or source.source, + embed_source=source.embed_source or source.source, + stt_source=source.stt_source or source.source, + chat_model=cfg.chat_model, + embed_model=cfg.embed_model, + vlm_model=source.vlm_model, + stt_model=source.stt_model, + ) + + +# Registry mapping ``client_backend`` identifiers to client builders. Register +# new LLM implementations here. +LLM_CLIENT_BUILDERS: dict[str, Callable[[LLMConfig], Any]] = { + "sdk": _build_sdk_client, + "anthropic": _build_anthropic_client, + "httpx": _build_httpx_client, + "lazyllm_backend": _build_lazyllm_client, +} + + +def build_llm_client(cfg: LLMConfig) -> Any: + """Build an LLM client for ``cfg.client_backend``. + + Raises: + ValueError: if ``cfg.client_backend`` is not registered. + """ + builder = LLM_CLIENT_BUILDERS.get(cfg.client_backend) + if builder is None: + available = ", ".join(sorted(LLM_CLIENT_BUILDERS)) + msg = f"Unknown llm_client_backend '{cfg.client_backend}'. Available: {available}" + raise ValueError(msg) + return builder(cfg) diff --git a/src/memu/llm/http_client.py b/src/memu/llm/http_client.py new file mode 100644 index 0000000..c09bad2 --- /dev/null +++ b/src/memu/llm/http_client.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import base64 +import logging +import os +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import httpx + +from memu.llm.backends.base import LLMBackend +from memu.llm.backends.claude import ClaudeLLMBackend +from memu.llm.backends.deepseek import DeepSeekLLMBackend +from memu.llm.backends.doubao import DoubaoLLMBackend +from memu.llm.backends.grok import GrokBackend +from memu.llm.backends.kimi import KimiLLMBackend +from memu.llm.backends.minimax import MiniMaxLLMBackend +from memu.llm.backends.openai import OpenAILLMBackend +from memu.llm.backends.openrouter import OpenRouterLLMBackend + + +def _load_proxy() -> str | None: + return os.getenv("MEMU_HTTP_PROXY") or os.getenv("HTTP_PROXY") or os.getenv("HTTPS_PROXY") or None + + +logger = logging.getLogger(__name__) + +LLM_BACKENDS: dict[str, Callable[[], LLMBackend]] = { + OpenAILLMBackend.name: OpenAILLMBackend, + ClaudeLLMBackend.name: ClaudeLLMBackend, + GrokBackend.name: GrokBackend, + DeepSeekLLMBackend.name: DeepSeekLLMBackend, + KimiLLMBackend.name: KimiLLMBackend, + MiniMaxLLMBackend.name: MiniMaxLLMBackend, + DoubaoLLMBackend.name: DoubaoLLMBackend, + OpenRouterLLMBackend.name: OpenRouterLLMBackend, +} + + +class HTTPLLMClient: + """HTTP client for LLM APIs (chat, vision, transcription). + + Scoped to text capabilities; embedding is handled by the dedicated + :mod:`memu.embedding` clients. + """ + + def __init__( + self, + *, + base_url: str, + api_key: str, + chat_model: str, + provider: str = "openai", + endpoint_overrides: dict[str, str] | None = None, + timeout: int = 60, + ): + # Ensure base_url ends with "/" so httpx doesn't discard the path + # component when joining with endpoint paths. + # See: https://github.com/NevaMind-AI/memU/issues/328 + self.base_url = base_url.rstrip("/") + "/" + self.api_key = api_key or "" + self.chat_model = chat_model + self.provider = provider.lower() + self.backend = self._load_backend(self.provider) + overrides = endpoint_overrides or {} + raw_summary_ep = overrides.get("chat") or overrides.get("summary") or self.backend.summary_endpoint + # Strip leading "/" from endpoints so httpx resolves them relative to + # base_url instead of treating them as absolute paths. + self.summary_endpoint = raw_summary_ep.lstrip("/") + self.timeout = timeout + self.proxy = _load_proxy() + + async def chat( + self, + prompt: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + temperature: float = 0.2, + ) -> tuple[str, dict[str, Any]]: + """Generic chat completion.""" + messages: list[dict[str, Any]] = [] + if system_prompt is not None: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + payload: dict[str, Any] = { + "model": self.chat_model, + "messages": messages, + "temperature": temperature, + } + if max_tokens is not None: + payload["max_tokens"] = max_tokens + + async with httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout) as client: + resp = await client.post(self.summary_endpoint, json=payload, headers=self._headers()) + resp.raise_for_status() + data = resp.json() + logger.debug("HTTP LLM chat response: %s", data) + return self.backend.parse_summary_response(data), data + + async def summarize( + self, text: str, max_tokens: int | None = None, system_prompt: str | None = None + ) -> tuple[str, dict[str, Any]]: + payload = self.backend.build_summary_payload( + text=text, system_prompt=system_prompt, chat_model=self.chat_model, max_tokens=max_tokens + ) + async with httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout, proxy=self.proxy) as client: + resp = await client.post(self.summary_endpoint, json=payload, headers=self._headers()) + resp.raise_for_status() + data = resp.json() + logger.debug("HTTP LLM summarize response: %s", data) + return self.backend.parse_summary_response(data), data + + async def vision( + self, + prompt: str, + image_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, dict[str, Any]]: + """ + Call Vision API with an image. + + Args: + prompt: Text prompt to send with the image + image_path: Path to the image file + max_tokens: Maximum tokens in response + system_prompt: Optional system prompt + + Returns: + Tuple of (LLM response text, raw response dict) + """ + # Read and encode image as base64 + image_data = Path(image_path).read_bytes() + base64_image = base64.b64encode(image_data).decode("utf-8") + + # Detect image format + suffix = Path(image_path).suffix.lower() + mime_type = { + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", + }.get(suffix, "image/jpeg") + + payload = self.backend.build_vision_payload( + prompt=prompt, + base64_image=base64_image, + mime_type=mime_type, + system_prompt=system_prompt, + chat_model=self.chat_model, + max_tokens=max_tokens, + ) + + async with httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout, proxy=self.proxy) as client: + resp = await client.post(self.summary_endpoint, json=payload, headers=self._headers()) + resp.raise_for_status() + data = resp.json() + logger.debug("HTTP LLM vision response: %s", data) + return self.backend.parse_summary_response(data), data + + async def transcribe( + self, + audio_path: str, + *, + prompt: str | None = None, + language: str | None = None, + response_format: str = "text", + ) -> tuple[str, dict[str, Any] | None]: + """ + Transcribe audio file using OpenAI Audio API. + + Args: + audio_path: Path to the audio file + prompt: Optional prompt to guide the transcription + language: Optional language code (e.g., 'en', 'zh') + response_format: Response format ('text', 'json', 'verbose_json') + + Returns: + Tuple of (transcribed text, raw response dict or None for text format) + """ + try: + raw_response: dict[str, Any] | None = None + # Prepare multipart form data + with open(audio_path, "rb") as audio_file: + files = {"file": (Path(audio_path).name, audio_file, "application/octet-stream")} + data = { + "model": "gpt-4o-mini-transcribe", + "response_format": response_format, + } + if prompt: + data["prompt"] = prompt + if language: + data["language"] = language + + async with httpx.AsyncClient( + base_url=self.base_url, timeout=self.timeout * 3, proxy=self.proxy + ) as client: + resp = await client.post( + "/v1/audio/transcriptions", + files=files, + data=data, + headers=self._headers(), + ) + resp.raise_for_status() + + if response_format == "text": + result = resp.text + else: + raw_response = resp.json() + result = raw_response.get("text", "") + + logger.debug("HTTP audio transcribe response for %s: %s chars", audio_path, len(result)) + except Exception: + logger.exception("Audio transcription failed for %s", audio_path) + raise + else: + return result or "", raw_response + + def _headers(self) -> dict[str, str]: + return self.backend.default_headers(self.api_key) + + def _load_backend(self, provider: str) -> LLMBackend: + factory = LLM_BACKENDS.get(provider) + if not factory: + msg = f"Unsupported LLM provider '{provider}'. Available: {', '.join(LLM_BACKENDS.keys())}" + raise ValueError(msg) + return factory() diff --git a/src/memu/llm/lazyllm_client.py b/src/memu/llm/lazyllm_client.py new file mode 100644 index 0000000..8446b6a --- /dev/null +++ b/src/memu/llm/lazyllm_client.py @@ -0,0 +1,159 @@ +import asyncio +import functools +from typing import Any, cast + +import lazyllm +from lazyllm import LOG + + +class LazyLLMClient: + """LAZYLLM client that relies on the LazyLLM framework.""" + + DEFAULT_SOURCE = "qwen" + + def __init__( + self, + *, + llm_source: str | None = None, + vlm_source: str | None = None, + embed_source: str | None = None, + stt_source: str | None = None, + chat_model: str | None = None, + vlm_model: str | None = None, + embed_model: str | None = None, + stt_model: str | None = None, + ): + self.llm_source = llm_source or self.DEFAULT_SOURCE + self.vlm_source = vlm_source or self.DEFAULT_SOURCE + self.embed_source = embed_source or self.DEFAULT_SOURCE + self.stt_source = stt_source or self.DEFAULT_SOURCE + self.chat_model = chat_model + self.vlm_model = vlm_model + self.embed_model = embed_model + self.stt_model = stt_model + + async def _call_async(self, client: Any, *args: Any, **kwargs: Any) -> Any: + """ + Asynchronously call a LazyLLM client with given arguments and keyword arguments. + """ + if kwargs: + return await asyncio.to_thread(functools.partial(client, *args, **kwargs)) + else: + return await asyncio.to_thread(client, *args) + + async def chat( + self, + text: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + temperature: float = 0.2, + ) -> str: + """ + Generate a summary or response for the input text using the configured LLM backend. + + Args: + text: The input text to summarize or process. + max_tokens: (Optional) Maximum number of tokens to generate. + system_prompt: (Optional) System instruction to guide the LLM behavior. + Return: + The generated summary text as a string. + """ + client = lazyllm.namespace("MEMU").OnlineModule(source=self.llm_source, model=self.chat_model, type="llm") + prompt = f"{system_prompt}\n\n" if system_prompt else "" + full_prompt = f"{prompt}text:\n{text}" + LOG.debug(f"Summarizing text with {self.llm_source}/{self.chat_model}") + response = await self._call_async(client, full_prompt) + return cast(str, response) + + async def summarize( + self, + text: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> str: + """ + Generate a summary or response for the input text using the configured LLM backend. + + Args: + text: The input text to summarize or process. + max_tokens: (Optional) Maximum number of tokens to generate. + system_prompt: (Optional) System instruction to guide the LLM behavior. + Return: + The generated summary text as a string. + """ + client = lazyllm.namespace("MEMU").OnlineModule(source=self.llm_source, model=self.chat_model, type="llm") + prompt = system_prompt or "Summarize the text in one short paragraph." + full_prompt = f"{prompt}\n\ntext:\n{text}" + LOG.debug(f"Summarizing text with {self.llm_source}/{self.chat_model}") + response = await self._call_async(client, full_prompt) + return cast(str, response) + + async def vision( + self, + prompt: str, + image_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, Any]: + """ + Process an image with a text prompt using the configured VLM (Vision-Language Model). + + + Args: + prompt: Text prompt describing the request or question about the image. + image_path: Path to the image file to be analyzed. + max_tokens: (Optional) Maximum number of tokens to generate. + system_prompt: (Optional) System instruction to guide the VLM behavior. + Return: + A tuple containing the generated text response and None (reserved for metadata). + """ + client = lazyllm.namespace("MEMU").OnlineModule(source=self.vlm_source, model=self.vlm_model, type="vlm") + LOG.debug(f"Processing image with {self.vlm_source}/{self.vlm_model}: {image_path}") + # LazyLLM VLM accepts prompt as first positional argument and image_path as keyword argument + response = await self._call_async(client, prompt, lazyllm_files=image_path) + return response, None + + async def embed( + self, + texts: list[str], + batch_size: int = 10, + ) -> list[list[float]]: + """ + Generate vector embeddings for a list of text strings. + + Args: + texts: List of text strings to embed. + batch_size: (Optional) Batch size for processing embeddings (default: 10). + Return: + A list of embedding vectors (list of floats), one for each input text. + """ + client = lazyllm.namespace("MEMU").OnlineModule( + source=self.embed_source, model=self.embed_model, type="embed", batch_size=batch_size + ) + LOG.debug(f"embed {len(texts)} texts with {self.embed_source}/{self.embed_model}") + response = await self._call_async(client, texts) + return cast(list[list[float]], response) + + async def transcribe( + self, + audio_path: str, + language: str | None = None, + prompt: str | None = None, + ) -> str: + """ + Transcribe audio content to text using the configured STT (Speech-to-Text) backend. + + Args: + audio_path: Path to the audio file to transcribe. + language: (Optional) Language code of the audio content. + prompt: (Optional) Text prompt to guide the transcription or translation. + Return: + The transcribed text as a string. + """ + client = lazyllm.namespace("MEMU").OnlineModule(source=self.stt_source, model=self.stt_model, type="stt") + LOG.debug(f"Transcribing audio with {self.stt_source}/{self.stt_model}: {audio_path}") + response = await self._call_async(client, audio_path) + return cast(str, response) diff --git a/src/memu/llm/openai_client.py b/src/memu/llm/openai_client.py new file mode 100644 index 0000000..1eb020d --- /dev/null +++ b/src/memu/llm/openai_client.py @@ -0,0 +1,202 @@ +import base64 +import logging +from pathlib import Path +from typing import Any, Literal + +from openai import AsyncOpenAI, omit +from openai.types.chat import ( + ChatCompletion, + ChatCompletionContentPartImageParam, + ChatCompletionContentPartTextParam, + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, + ChatCompletionUserMessageParam, +) + +logger = logging.getLogger(__name__) + + +class OpenAIClient: + """OpenAI LLM client that relies on the official Python SDK. + + Scoped to text capabilities (chat/summarize/vision/transcribe). Embedding is + handled by the dedicated :mod:`memu.embedding` clients. + """ + + def __init__( + self, + *, + base_url: str, + api_key: str, + chat_model: str, + ): + self.base_url = base_url.rstrip("/") + self.api_key = api_key or "" + self.chat_model = chat_model + self.client = AsyncOpenAI(api_key=self.api_key, base_url=self.base_url) + + async def chat( + self, + prompt: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + temperature: float = 0.2, + ) -> tuple[str, ChatCompletion]: + """Generic chat completion.""" + messages: list[ChatCompletionMessageParam] = [] + if system_prompt is not None: + system_message: ChatCompletionSystemMessageParam = {"role": "system", "content": system_prompt} + messages.append(system_message) + + user_message: ChatCompletionUserMessageParam = {"role": "user", "content": prompt} + messages.append(user_message) + + response = await self.client.chat.completions.create( + model=self.chat_model, + messages=messages, + temperature=temperature, + # GPT-5 models reject ``max_tokens`` (and ``max_tokens=None``); omit when + # unset, otherwise use ``max_completion_tokens`` (works for GPT-4o too). + max_completion_tokens=max_tokens if max_tokens is not None else omit, + ) + content = response.choices[0].message.content + logger.debug("OpenAI chat response: %s", response) + return content or "", response + + async def summarize( + self, + text: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, ChatCompletion]: + prompt = system_prompt or "Summarize the text in one short paragraph." + + system_message: ChatCompletionSystemMessageParam = {"role": "system", "content": prompt} + user_message: ChatCompletionUserMessageParam = {"role": "user", "content": text} + messages: list[ChatCompletionMessageParam] = [system_message, user_message] + + response = await self.client.chat.completions.create( + model=self.chat_model, + messages=messages, + temperature=1, + max_completion_tokens=max_tokens if max_tokens is not None else omit, + ) + content = response.choices[0].message.content + logger.debug("OpenAI summarize response: %s", response) + return content or "", response + + async def vision( + self, + prompt: str, + image_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, ChatCompletion]: + """ + Call OpenAI Vision API with an image. + + Args: + prompt: Text prompt to send with the image + image_path: Path to the image file + max_tokens: Maximum tokens in response + system_prompt: Optional system prompt + + Returns: + Tuple of (LLM response text, raw ChatCompletion response) + """ + # Read and encode image as base64 + image_data = Path(image_path).read_bytes() + base64_image = base64.b64encode(image_data).decode("utf-8") + + # Detect image format + suffix = Path(image_path).suffix.lower() + mime_type = { + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", + }.get(suffix, "image/jpeg") + + # Build messages with image + messages: list[ChatCompletionMessageParam] = [] + if system_prompt: + system_message: ChatCompletionSystemMessageParam = { + "role": "system", + "content": system_prompt, + } + messages.append(system_message) + + text_part: ChatCompletionContentPartTextParam = {"type": "text", "text": prompt} + image_part: ChatCompletionContentPartImageParam = { + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{base64_image}", + }, + } + user_message: ChatCompletionUserMessageParam = { + "role": "user", + "content": [text_part, image_part], + } + messages.append(user_message) + + response = await self.client.chat.completions.create( + model=self.chat_model, + messages=messages, + temperature=1, + max_completion_tokens=max_tokens if max_tokens is not None else omit, + ) + content = response.choices[0].message.content + logger.debug("OpenAI vision response: %s", response) + return content or "", response + + async def transcribe( + self, + audio_path: str, + *, + prompt: str | None = None, + language: str | None = None, + response_format: Literal["text", "json", "verbose_json"] = "text", + ) -> tuple[str, Any]: + """ + Transcribe audio file using OpenAI Audio API. + + Args: + audio_path: Path to the audio file + prompt: Optional prompt to guide the transcription + language: Optional language code (e.g., 'en', 'zh') + response_format: Response format ('text', 'json', 'verbose_json') + + Returns: + Tuple of (transcribed text, raw transcription response) + """ + try: + # Use gpt-4o-mini-transcribe for better performance and cost + kwargs: dict[str, Any] = {} + if prompt is not None: + kwargs["prompt"] = prompt + if language is not None: + kwargs["language"] = language + with open(audio_path, "rb") as audio_stream: + transcription = await self.client.audio.transcriptions.create( + file=audio_stream, + model="gpt-4o-mini-transcribe", + response_format=response_format, + **kwargs, + ) + + # Handle different response formats + if response_format == "text": + result = transcription if isinstance(transcription, str) else transcription.text + else: + result = transcription.text if hasattr(transcription, "text") else str(transcription) + + logger.debug("OpenAI transcribe response for %s: %s chars", audio_path, len(result)) + except Exception: + logger.exception("Audio transcription failed for %s", audio_path) + raise + else: + return result or "", transcription diff --git a/src/memu/llm/wrapper.py b/src/memu/llm/wrapper.py new file mode 100644 index 0000000..90c34ba --- /dev/null +++ b/src/memu/llm/wrapper.py @@ -0,0 +1,804 @@ +from __future__ import annotations + +import hashlib +import inspect +import logging +import threading +import time +import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class LLMCallContext: + profile: str + request_id: str + trace_id: str | None + operation: str | None + step_id: str | None + provider: str | None + model: str | None + tags: Mapping[str, Any] | None + + +@dataclass(frozen=True) +class LLMRequestView: + kind: str + input_items: int | None = None + input_chars: int | None = None + content: str | list[str] | None = None + content_hash: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class LLMResponseView: + output_items: int | None = None + output_chars: int | None = None + content: str | None = None + content_hash: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class LLMUsage: + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + cached_input_tokens: int | None = None + reasoning_tokens: int | None = None + latency_ms: float | None = None + finish_reason: str | None = None + status: str | None = None + tokens_breakdown: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class LLMCallFilter: + operations: set[str] | None = None + step_ids: set[str] | None = None + providers: set[str] | None = None + models: set[str] | None = None + statuses: set[str] | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "operations", _normalize_set(self.operations)) + object.__setattr__(self, "providers", _normalize_set(self.providers)) + object.__setattr__(self, "models", _normalize_set(self.models)) + object.__setattr__(self, "statuses", _normalize_set(self.statuses)) + + def matches(self, ctx: LLMCallContext, status: str | None) -> bool: + if self.operations and (ctx.operation or "").lower() not in self.operations: + return False + if self.step_ids and (ctx.step_id or "") not in self.step_ids: + return False + if self.providers and (ctx.provider or "").lower() not in self.providers: + return False + if self.models and (ctx.model or "").lower() not in self.models: + return False + if self.statuses: + return status is not None and status.lower() in self.statuses + return True + + +@dataclass(frozen=True) +class LLMCallMetadata: + profile: str | None = None + operation: str | None = None + step_id: str | None = None + trace_id: str | None = None + tags: Mapping[str, Any] | None = None + + +@dataclass(frozen=True) +class _LLMInterceptor: + interceptor_id: int + fn: Callable[..., Any] + name: str | None + priority: int + order: int + filter: LLMCallFilter | Callable[[LLMCallContext, str | None], bool] | None + + +@dataclass(frozen=True) +class _LLMInterceptorSnapshot: + before: tuple[_LLMInterceptor, ...] + after: tuple[_LLMInterceptor, ...] + on_error: tuple[_LLMInterceptor, ...] + + +class LLMInterceptorHandle: + def __init__(self, registry: LLMInterceptorRegistry, interceptor_id: int) -> None: + self._registry = registry + self._interceptor_id = interceptor_id + self._disposed = False + + def dispose(self) -> bool: + if self._disposed: + return False + self._disposed = True + return self._registry.remove(self._interceptor_id) + + +class LLMInterceptorRegistry: + def __init__(self, *, strict: bool = False) -> None: + self._before: tuple[_LLMInterceptor, ...] = () + self._after: tuple[_LLMInterceptor, ...] = () + self._on_error: tuple[_LLMInterceptor, ...] = () + self._lock = threading.Lock() + self._seq = 0 + self._strict = strict + + @property + def strict(self) -> bool: + return self._strict + + def register_before( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + priority: int = 0, + where: LLMCallFilter | Callable[[LLMCallContext, str | None], bool] | Mapping[str, Any] | None = None, + ) -> LLMInterceptorHandle: + return self._register("before", fn, name=name, priority=priority, where=where) + + def register_after( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + priority: int = 0, + where: LLMCallFilter | Callable[[LLMCallContext, str | None], bool] | Mapping[str, Any] | None = None, + ) -> LLMInterceptorHandle: + return self._register("after", fn, name=name, priority=priority, where=where) + + def register_on_error( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + priority: int = 0, + where: LLMCallFilter | Callable[[LLMCallContext, str | None], bool] | Mapping[str, Any] | None = None, + ) -> LLMInterceptorHandle: + return self._register("on_error", fn, name=name, priority=priority, where=where) + + def _register( + self, + kind: str, + fn: Callable[..., Any], + *, + name: str | None, + priority: int, + where: LLMCallFilter | Callable[[LLMCallContext, str | None], bool] | Mapping[str, Any] | None, + ) -> LLMInterceptorHandle: + if not callable(fn): + msg = "Interceptor must be callable" + raise TypeError(msg) + where = _coerce_filter(where) + with self._lock: + self._seq += 1 + interceptor = _LLMInterceptor( + interceptor_id=self._seq, + fn=fn, + name=name, + priority=priority, + order=self._seq, + filter=where, + ) + if kind == "before": + self._before = _sorted_interceptors(self._before, interceptor) + elif kind == "after": + self._after = _sorted_interceptors(self._after, interceptor) + elif kind == "on_error": + self._on_error = _sorted_interceptors(self._on_error, interceptor) + else: + msg = f"Unknown interceptor kind '{kind}'" + raise ValueError(msg) + return LLMInterceptorHandle(self, interceptor.interceptor_id) + + def remove(self, interceptor_id: int) -> bool: + with self._lock: + removed = False + before = tuple(i for i in self._before if i.interceptor_id != interceptor_id) + after = tuple(i for i in self._after if i.interceptor_id != interceptor_id) + on_error = tuple(i for i in self._on_error if i.interceptor_id != interceptor_id) + if len(before) != len(self._before): + removed = True + self._before = before + if len(after) != len(self._after): + removed = True + self._after = after + if len(on_error) != len(self._on_error): + removed = True + self._on_error = on_error + return removed + + def snapshot(self) -> _LLMInterceptorSnapshot: + return _LLMInterceptorSnapshot(self._before, self._after, self._on_error) + + +class LLMClientWrapper: + def __init__( + self, + client: Any, + *, + registry: LLMInterceptorRegistry, + metadata: LLMCallMetadata | None = None, + provider: str | None = None, + chat_model: str | None = None, + embed_model: str | None = None, + ) -> None: + self._client = client + self._registry = registry + self._metadata = metadata or LLMCallMetadata() + self._provider = provider + self._chat_model = chat_model or getattr(client, "chat_model", None) + self._embed_model = embed_model or getattr(client, "embed_model", None) + + def __getattr__(self, name: str) -> Any: + return getattr(self._client, name) + + async def summarize( + self, + text: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> Any: + request_view = _build_text_request_view( + "summarize", + text, + metadata={ + "system_prompt_chars": len(system_prompt or ""), + "max_tokens": max_tokens, + }, + ) + + async def _call() -> Any: + return await self._client.summarize(text, max_tokens=max_tokens, system_prompt=system_prompt) + + return await self._invoke( + kind="summarize", + call_fn=_call, + request_view=request_view, + model=self._chat_model, + response_builder=_build_text_response_view, + ) + + async def chat( + self, + prompt: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + temperature: float = 0.2, + ) -> Any: + request_view = _build_text_request_view( + "chat", + prompt, + metadata={ + "system_prompt_chars": len(system_prompt or ""), + "max_tokens": max_tokens, + "temperature": temperature, + }, + ) + + async def _call() -> Any: + return await self._client.chat( + prompt, + max_tokens=max_tokens, + system_prompt=system_prompt, + temperature=temperature, + ) + + return await self._invoke( + kind="chat", + call_fn=_call, + request_view=request_view, + model=self._chat_model, + response_builder=_build_text_response_view, + ) + + async def vision( + self, + prompt: str, + image_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> Any: + metadata = { + "image_path": Path(image_path).name, + "image_bytes": _safe_file_size(image_path), + "system_prompt_chars": len(system_prompt or ""), + "max_tokens": max_tokens, + } + request_view = _build_text_request_view("vision", prompt, metadata=metadata) + + async def _call() -> Any: + return await self._client.vision( + prompt, + image_path, + max_tokens=max_tokens, + system_prompt=system_prompt, + ) + + return await self._invoke( + kind="vision", + call_fn=_call, + request_view=request_view, + model=self._chat_model, + response_builder=_build_text_response_view, + ) + + async def video( + self, + prompt: str, + video_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> Any: + metadata = { + "video_path": Path(video_path).name, + "video_bytes": _safe_file_size(video_path), + "system_prompt_chars": len(system_prompt or ""), + "max_tokens": max_tokens, + } + request_view = _build_text_request_view("video", prompt, metadata=metadata) + + async def _call() -> Any: + return await self._client.video( + prompt, + video_path, + max_tokens=max_tokens, + system_prompt=system_prompt, + ) + + return await self._invoke( + kind="video", + call_fn=_call, + request_view=request_view, + model=self._chat_model, + response_builder=_build_text_response_view, + ) + + async def embed(self, inputs: list[str]) -> Any: + request_view = _build_embedding_request_view(inputs) + + async def _call() -> Any: + return await self._client.embed(inputs) + + return await self._invoke( + kind="embed", + call_fn=_call, + request_view=request_view, + model=self._embed_model, + response_builder=_build_embedding_response_view, + ) + + async def transcribe( + self, + audio_path: str, + *, + prompt: str | None = None, + language: str | None = None, + response_format: str = "text", + ) -> Any: + metadata = { + "audio_path": Path(audio_path).name, + "audio_bytes": _safe_file_size(audio_path), + "prompt_chars": len(prompt or ""), + "language": language, + "response_format": response_format, + } + request_view = _build_text_request_view("transcribe", prompt or "", metadata=metadata) + + async def _call() -> Any: + return await self._client.transcribe( + audio_path, + prompt=prompt, + language=language, + response_format=response_format, + ) + + return await self._invoke( + kind="transcribe", + call_fn=_call, + request_view=request_view, + model=None, + response_builder=_build_text_response_view, + ) + + async def _invoke( + self, + *, + kind: str, + call_fn: Callable[[], Any], + request_view: LLMRequestView, + model: str | None, + response_builder: Callable[[Any], LLMResponseView], + ) -> Any: + call_ctx = self._build_call_context(model) + snapshot = self._registry.snapshot() + await self._run_before(snapshot.before, call_ctx, request_view) + start_time = time.perf_counter() + try: + result = call_fn() + if inspect.isawaitable(result): + result = await result + except Exception as exc: + latency_ms = (time.perf_counter() - start_time) * 1000 + usage = LLMUsage(latency_ms=latency_ms, status="error") + await self._run_on_error(snapshot.on_error, call_ctx, request_view, exc, usage) + raise + else: + latency_ms = (time.perf_counter() - start_time) * 1000 + + # Handle tuple response: (pure_response, raw_response) + pure_result = result + raw_response = None + if isinstance(result, tuple) and len(result) == 2: + pure_result, raw_response = result + + response_view = response_builder(pure_result) + + # Extract token usage from raw response (best-effort) + extracted_usage = _extract_usage_from_raw_response(kind=kind, raw_response=raw_response) + usage = LLMUsage( + input_tokens=extracted_usage.get("input_tokens"), + output_tokens=extracted_usage.get("output_tokens"), + total_tokens=extracted_usage.get("total_tokens"), + cached_input_tokens=extracted_usage.get("cached_input_tokens"), + reasoning_tokens=extracted_usage.get("reasoning_tokens"), + latency_ms=latency_ms, + finish_reason=extracted_usage.get("finish_reason"), + status="success", + tokens_breakdown=extracted_usage.get("tokens_breakdown"), + ) + + await self._run_after(snapshot.after, call_ctx, request_view, response_view, usage) + return pure_result + + def _build_call_context(self, model: str | None) -> LLMCallContext: + request_id = uuid.uuid4().hex + return LLMCallContext( + profile=self._metadata.profile or "", + request_id=request_id, + trace_id=self._metadata.trace_id, + operation=self._metadata.operation, + step_id=self._metadata.step_id, + provider=self._provider, + model=model, + tags=self._metadata.tags, + ) + + async def _run_before( + self, + interceptors: Sequence[_LLMInterceptor], + ctx: LLMCallContext, + request_view: LLMRequestView, + ) -> None: + for interceptor in interceptors: + if not _should_run_interceptor(interceptor, ctx, None): + continue + await _safe_invoke_interceptor( + interceptor, + self._registry.strict, + ctx, + request_view, + ) + + async def _run_after( + self, + interceptors: Sequence[_LLMInterceptor], + ctx: LLMCallContext, + request_view: LLMRequestView, + response_view: LLMResponseView, + usage: LLMUsage, + ) -> None: + for interceptor in reversed(interceptors): + if not _should_run_interceptor(interceptor, ctx, "success"): + continue + await _safe_invoke_interceptor( + interceptor, + self._registry.strict, + ctx, + request_view, + response_view, + usage, + ) + + async def _run_on_error( + self, + interceptors: Sequence[_LLMInterceptor], + ctx: LLMCallContext, + request_view: LLMRequestView, + error: Exception, + usage: LLMUsage, + ) -> None: + for interceptor in reversed(interceptors): + if not _should_run_interceptor(interceptor, ctx, "error"): + continue + await _safe_invoke_interceptor( + interceptor, + self._registry.strict, + ctx, + request_view, + error, + usage, + ) + + +def _normalize_set(values: set[str] | None) -> set[str] | None: + if not values: + return None + return {str(value).lower() for value in values} + + +def _sorted_interceptors( + existing: tuple[_LLMInterceptor, ...], + interceptor: _LLMInterceptor, +) -> tuple[_LLMInterceptor, ...]: + items = list(existing) + items.append(interceptor) + items.sort(key=lambda item: (item.priority, item.order)) + return tuple(items) + + +def _hash_text(value: str | None) -> str | None: + if not value: + return None + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _hash_texts(values: Sequence[str]) -> str | None: + if not values: + return None + sha = hashlib.sha256() + for value in values: + sha.update(value.encode("utf-8")) + sha.update(b"\0") + return sha.hexdigest() + + +def _safe_file_size(path: str) -> int | None: + try: + return Path(path).stat().st_size + except OSError: + return None + + +def _build_text_request_view( + kind: str, + text: str, + *, + metadata: dict[str, Any] | None = None, +) -> LLMRequestView: + return LLMRequestView( + kind=kind, + input_items=1, + input_chars=len(text), + content=text, + content_hash=_hash_text(text), + metadata=metadata or {}, + ) + + +def _build_text_response_view(response: str) -> LLMResponseView: + return LLMResponseView( + output_items=1, + output_chars=len(response), + content=response, + content_hash=_hash_text(response), + metadata={}, + ) + + +def _build_embedding_request_view(inputs: Sequence[str]) -> LLMRequestView: + total_chars = sum(len(text) for text in inputs) + return LLMRequestView( + kind="embed", + input_items=len(inputs), + input_chars=total_chars, + content=list(inputs), + content_hash=_hash_texts(inputs), + metadata={}, + ) + + +def _build_embedding_response_view(response: Sequence[Sequence[float]]) -> LLMResponseView: + vector_dim = len(response[0]) if response else 0 + return LLMResponseView( + output_items=len(response), + output_chars=None, + content_hash=None, + metadata={"vector_dim": vector_dim}, + ) + + +def _get_attr_or_key(obj: Any, key: str) -> Any: + """Extract value from object attribute or dict key.""" + if obj is None: + return None + if hasattr(obj, key): + return getattr(obj, key) + if isinstance(obj, dict): + return obj.get(key) + return None + + +def _extract_finish_reason(raw_response: Any) -> str | None: + """Extract finish_reason from choices[0] if available.""" + choices = _get_attr_or_key(raw_response, "choices") + if choices and len(choices) > 0: + result = _get_attr_or_key(choices[0], "finish_reason") + return str(result) if result is not None else None + return None + + +def _get_usage_object(raw_response: Any) -> Any: + """Get usage object/dict from response.""" + if hasattr(raw_response, "usage") and raw_response.usage is not None: + return raw_response.usage + if isinstance(raw_response, dict) and "usage" in raw_response: + return raw_response["usage"] + return None + + +def _convert_to_dict(obj: Any) -> dict[str, Any] | None: + """Convert object to dict using available methods.""" + if hasattr(obj, "model_dump"): + result: dict[str, Any] = obj.model_dump() + return result + if hasattr(obj, "__dict__"): + return dict(obj.__dict__) + if isinstance(obj, dict): + return obj + return None + + +def _extract_token_details(usage_obj: Any, usage_data: dict[str, Any]) -> None: + """Extract token breakdown and cached tokens from usage object.""" + completion_tokens_details = _get_attr_or_key(usage_obj, "completion_tokens_details") + if completion_tokens_details is not None: + breakdown = _convert_to_dict(completion_tokens_details) + if breakdown is not None: + usage_data["tokens_breakdown"] = breakdown + reasoning_tokens = _get_attr_or_key(completion_tokens_details, "reasoning_tokens") + if reasoning_tokens is not None: + usage_data["reasoning_tokens"] = reasoning_tokens + + prompt_tokens_details = _get_attr_or_key(usage_obj, "prompt_tokens_details") + if prompt_tokens_details is not None: + cached_tokens = _get_attr_or_key(prompt_tokens_details, "cached_tokens") + if cached_tokens is not None: + usage_data["cached_input_tokens"] = cached_tokens + + +def _extract_usage_from_raw_response(kind: str, raw_response: Any) -> dict[str, Any]: + """ + Best-effort extraction of token usage from raw LLM response. + + Supports OpenAI SDK response objects and JSON dict responses. + Mapping: + input_tokens <- prompt_tokens + output_tokens <- completion_tokens + total_tokens <- total_tokens + tokens_breakdown <- completion_tokens_details + """ + usage_data: dict[str, Any] = {} + + if raw_response is None: + return usage_data + + try: + finish_reason = _extract_finish_reason(raw_response) + if finish_reason is not None: + usage_data["finish_reason"] = finish_reason + + usage_obj = _get_usage_object(raw_response) + if usage_obj is None: + return usage_data + + # Map prompt_tokens -> input_tokens + prompt_tokens = _get_attr_or_key(usage_obj, "prompt_tokens") + if prompt_tokens is not None: + usage_data["input_tokens"] = prompt_tokens + + # Map completion_tokens -> output_tokens + completion_tokens = _get_attr_or_key(usage_obj, "completion_tokens") + if completion_tokens is not None: + usage_data["output_tokens"] = completion_tokens + + # total_tokens stays the same + total_tokens = _get_attr_or_key(usage_obj, "total_tokens") + if total_tokens is not None: + usage_data["total_tokens"] = total_tokens + + # Some providers does not explicitly return input_tokens for embedding calls + if kind == "embed" and usage_data.get("total_tokens") and not usage_data.get("input_tokens"): + usage_data["input_tokens"] = usage_data["total_tokens"] + + _extract_token_details(usage_obj, usage_data) + + except Exception: + # Best-effort: silently ignore extraction errors + logger.debug("Failed to extract usage from raw response", exc_info=True) + + return usage_data + + +def _coerce_filter( + where: LLMCallFilter | Callable[[LLMCallContext, str | None], bool] | Mapping[str, Any] | None, +) -> LLMCallFilter | Callable[[LLMCallContext, str | None], bool] | None: + if where is None or callable(where) or isinstance(where, LLMCallFilter): + return where + if isinstance(where, Mapping): + return LLMCallFilter( + operations=_ensure_set(where.get("operations") or where.get("operation")), + step_ids=_ensure_set(where.get("step_ids") or where.get("step_id")), + providers=_ensure_set(where.get("providers") or where.get("provider")), + models=_ensure_set(where.get("models") or where.get("model")), + statuses=_ensure_set(where.get("statuses") or where.get("status")), + ) + msg = "Filter must be a callable, mapping, or LLMCallFilter" + raise TypeError(msg) + + +def _ensure_set(value: Any) -> set[str] | None: + if value is None: + return None + if isinstance(value, set): + return {str(item) for item in value} + if isinstance(value, (list, tuple)): + return {str(item) for item in value} + return {str(value)} + + +def _should_run_interceptor( + interceptor: _LLMInterceptor, + ctx: LLMCallContext, + status: str | None, +) -> bool: + filt = interceptor.filter + if filt is None: + return True + if isinstance(filt, LLMCallFilter): + try: + return filt.matches(ctx, status) + except Exception: + logger.exception("LLM interceptor filter failed: %s", interceptor.name or interceptor.interceptor_id) + return False + try: + return bool(filt(ctx, status)) + except TypeError: + try: + return bool(filt(ctx, None)) + except Exception: + logger.exception("LLM interceptor filter failed: %s", interceptor.name or interceptor.interceptor_id) + return False + except Exception: + logger.exception("LLM interceptor filter failed: %s", interceptor.name or interceptor.interceptor_id) + return False + + +async def _safe_invoke_interceptor( + interceptor: _LLMInterceptor, + strict: bool, + *args: Any, +) -> None: + try: + result = interceptor.fn(*args) + if inspect.isawaitable(result): + await result + except Exception: + if strict: + raise + logger.exception("LLM interceptor failed: %s", interceptor.name or interceptor.interceptor_id) diff --git a/src/memu/memory_fs/__init__.py b/src/memu/memory_fs/__init__.py new file mode 100644 index 0000000..9e1af3f --- /dev/null +++ b/src/memu/memory_fs/__init__.py @@ -0,0 +1,23 @@ +"""Markdown "memory file system" artifact layer. + +Renders the structured memory store (folders/files/sources) into browsable +markdown artifacts on disk. Read-only against the database and fully optional. +""" + +from memu.memory_fs.exporter import ( + ExistingArtifacts, + ExportResult, + FileDescription, + MemoryFileExporter, + slugify, +) +from memu.memory_fs.synthesizer import MemorySynthesizer + +__all__ = [ + "ExistingArtifacts", + "ExportResult", + "FileDescription", + "MemoryFileExporter", + "MemorySynthesizer", + "slugify", +] diff --git a/src/memu/memory_fs/exporter.py b/src/memu/memory_fs/exporter.py new file mode 100644 index 0000000..27d63cc --- /dev/null +++ b/src/memu/memory_fs/exporter.py @@ -0,0 +1,489 @@ +"""Render the structured memory store into memU's markdown "memory file system". + +The exporter projects the (optionally scoped) store into the navigable file tree +described in the README: + + / + ├── INDEX.md ← index of the raw files under resource/ + ├── MEMORY.md ← overall overview + index of memory/ + ├── SKILL.md ← overall overview + index of skill/ + ├── resource/ + │ └── ← one copied raw source file + ├── memory/ + │ └── .md ← one memory-track file (description + content) + └── skill/ + └── .md ← one skill-track file (description + content) + +Each artifact is a different aggregation of what the agent has stored. The three +root indexes (``INDEX.md`` / ``MEMORY.md`` / ``SKILL.md``) each point at a sibling +directory of payloads (``resource/`` / ``memory/`` / ``skill/``): + +- ``resource/`` : the raw source files copied verbatim out of the blob store + (``Resource.local_path``), so the actual ingested bytes live next to the memory. +- ``INDEX.md`` : an index of those raw files (name, modality, description, link + into ``resource/``), so an agent knows which raw resources exist. +- ``memory/`` : the living memory split one file per memory-track + :class:`~memu.database.models.RecallFile` (its description + content). +- ``MEMORY.md`` : an overall overview that links to each ``memory/.md`` file + (or a synthesized overview body when one is supplied). +- ``skill/`` : reusable skills split one file per skill-track ``RecallFile`` (its + description + content), exactly mirroring ``memory/`` (ADR 0006). Skills are + generated and persisted by the memorize workflow, not derived at export time. +- ``SKILL.md`` (root): an overview that links to each ``skill/.md`` file (or a + synthesized overview body when one is supplied) — the mirror of ``MEMORY.md``. + +This layer is read-only against the database and never mutates memory records. +A sidecar manifest (``.memufs_manifest.json``) records the content hash of every +artifact this exporter owns, so each export only rewrites files whose rendered +content actually changed (diff detection) — no database schema change required. +Rendered content avoids volatile values (e.g. "now" timestamps) so re-exporting +an unchanged store is detected as a no-op. +""" + +from __future__ import annotations + +import hashlib +import json +import pathlib +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from memu.database.interfaces import Database + from memu.database.models import RecallEntry, RecallFile, Resource + +MANIFEST_NAME = ".memufs_manifest.json" +SKILL_DIRNAME = "skill" +RESOURCE_DIRNAME = "resource" +MEMORY_DIRNAME = "memory" +SKILL_INDEX_FILENAME = "SKILL.md" +INDEX_FILENAME = "INDEX.md" +MEMORY_FILENAME = "MEMORY.md" +_GENERATED_NOTICE = "" + + +@dataclass +class FileDescription: + """A single source file's multimodal description (the shared trunk). + + Derived from a :class:`~memu.database.models.Resource`: ``caption`` is the + text/caption produced by preprocessing, regardless of original modality. + ``local_path`` points at the ingested file inside the resources dir. + """ + + url: str + modality: str + description: str + resource_id: str + local_path: str = "" + + @property + def file_name(self) -> str: + """The ingested file's name in the resources dir (fallback: the url).""" + name = pathlib.Path(self.local_path).name if self.local_path else "" + return name or self.url + + +@dataclass +class ExistingArtifacts: + """The prior on-disk overview bodies the synthesizer merges new content into. + + An empty instance (the default) signals a from-scratch (re)build: nothing is + carried forward, and the exporter's manifest diff prunes any now-stale files. + """ + + memory_body: str = "" + skill_body: str = "" + + +@dataclass +class ExportResult: + """Outcome of a single export pass.""" + + output_dir: str + written: list[str] = field(default_factory=list) + unchanged: list[str] = field(default_factory=list) + removed: list[str] = field(default_factory=list) + + @property + def changed(self) -> bool: + return bool(self.written or self.removed) + + def to_dict(self) -> dict[str, Any]: + return { + "output_dir": self.output_dir, + "changed": self.changed, + "written": sorted(self.written), + "unchanged": sorted(self.unchanged), + "removed": sorted(self.removed), + } + + +def slugify(name: str) -> str: + """Turn a name into a filesystem-safe slug.""" + slug = re.sub(r"[^a-z0-9]+", "-", name.strip().lower()).strip("-") + return slug or "untitled" + + +def _content_hash(content: str | bytes) -> str: + data = content if isinstance(content, bytes) else content.encode("utf-8") + return hashlib.sha256(data).hexdigest() + + +class MemoryFileExporter: + """Render a :class:`~memu.database.interfaces.Database` into the markdown tree. + + The exporter is stateless apart from the on-disk manifest, so a single + instance can be reused across scopes/exports. Callers are responsible for + serializing concurrent exports that target the same ``output_dir``. + """ + + def __init__(self, output_dir: str) -> None: + self.output_dir = pathlib.Path(output_dir) + + def export( + self, + database: Database, + *, + where: Mapping[str, Any] | None = None, + memory_body: str | None = None, + skill_body: str | None = None, + ) -> ExportResult: + """Render the (optionally scoped) store and write only changed artifacts. + + ``memory_body``/``skill_body`` are optional synthesized overviews for the root + ``MEMORY.md``/``SKILL.md``; when omitted, each is rendered deterministically as + an index linking to its per-file payloads. The ``memory/`` and ``skill/`` trees + (one file per memory- and skill-track :class:`~memu.database.models.RecallFile` + respectively), the ``resource/`` raw copies, and ``INDEX.md`` are always + deterministic. + """ + self.output_dir.mkdir(parents=True, exist_ok=True) + scope = dict(where) if where else None + + # memory/ and skill/ render their own RecallFile track; the two tracks share + # the table and are split here by the ``track`` column (ADR 0006). + memory_files = list( + database.recall_file_repo.list_categories(where={**(scope or {}), "track": "memory"}).values() + ) + skill_files = list( + database.recall_file_repo.list_categories(where={**(scope or {}), "track": "skill"}).values() + ) + resources = list(database.resource_repo.list_resources(where=scope).values()) + + # The shared trunk: one multimodal description per source file. + descriptions = self._build_descriptions(resources) + + # resource/: copy the raw source bytes verbatim; ``links`` maps each + # resource id to its relative path under resource/ for the INDEX.md links. + raw_artifacts, links = self._resource_artifacts(descriptions) + + # memory/ and skill/: one file per RecallFile, indexed by MEMORY.md / SKILL.md. + ordered_memory, memory_slugs = self._recall_file_slugs(memory_files) + ordered_skills, skill_slugs = self._recall_file_slugs(skill_files) + memory_artifacts = { + f"{MEMORY_DIRNAME}/{slug}.md": self._recall_file_document(rf) + for rf, slug in zip(ordered_memory, memory_slugs, strict=True) + } + skill_artifacts = { + f"{SKILL_DIRNAME}/{slug}.md": self._recall_file_document(rf) + for rf, slug in zip(ordered_skills, skill_slugs, strict=True) + } + + artifacts: dict[str, str | bytes] = {} + artifacts.update(raw_artifacts) + artifacts.update(memory_artifacts) + artifacts.update(skill_artifacts) + if memory_body is not None: + artifacts[MEMORY_FILENAME] = self._overview_document("Memory", memory_body) + else: + artifacts[MEMORY_FILENAME] = self._recall_file_index("Memory", MEMORY_DIRNAME, ordered_memory, memory_slugs) + if skill_body is not None: + artifacts[SKILL_INDEX_FILENAME] = self._overview_document("Skills", skill_body) + else: + artifacts[SKILL_INDEX_FILENAME] = self._recall_file_index( + "Skills", SKILL_DIRNAME, ordered_skills, skill_slugs + ) + artifacts[INDEX_FILENAME] = self._index_bypass(descriptions, links) + + return self._sync(artifacts) + + # -- trunk ------------------------------------------------------------- + + @staticmethod + def _build_descriptions(resources: list[Resource]) -> list[FileDescription]: + descriptions: list[FileDescription] = [] + for resource in sorted(resources, key=lambda r: (r.url, r.id)): + descriptions.append( + FileDescription( + url=resource.url, + modality=resource.modality, + description=" ".join((resource.caption or "").split()), + resource_id=resource.id, + local_path=resource.local_path, + ) + ) + return descriptions + + @staticmethod + def build_synthesis_descriptions(resources: list[Resource], items: list[RecallEntry]) -> list[FileDescription]: + """Build synthesizer input from the structured store (extracted items). + + The synthesizer is fed the extracted memory items per source so that the + structured store is the single source of truth for synthesis, rather than the + lossy per-source caption. A source's caption is used only as a fallback when it + has no extracted items yet (e.g. it failed extraction or has not been processed). + """ + items_by_resource: dict[str, list[RecallEntry]] = {} + for item in items: + if item.resource_id: + items_by_resource.setdefault(item.resource_id, []).append(item) + + descriptions: list[FileDescription] = [] + for resource in sorted(resources, key=lambda r: (r.url, r.id)): + res_items = sorted( + items_by_resource.get(resource.id, []), + key=lambda i: (i.memory_type, i.created_at, i.id), + ) + parts = [ + f"[{item.memory_type}] {' '.join((item.summary or '').split())}" + for item in res_items + if (item.summary or "").strip() + ] + description = "; ".join(parts) if parts else " ".join((resource.caption or "").split()) + descriptions.append( + FileDescription( + url=resource.url, + modality=resource.modality, + description=description, + resource_id=resource.id, + local_path=resource.local_path, + ) + ) + return descriptions + + # -- MEMORY / SKILL: per-file payloads + root overview ----------------- + # + # memory/ and skill/ are isomorphic: each is one file per RecallFile of its track, + # indexed by a root overview (MEMORY.md / SKILL.md) that is either synthesized or a + # deterministic link index. The helpers below are track-agnostic and shared. + + @staticmethod + def _overview_document(title: str, body: str) -> str: + """A synthesized root overview (MEMORY.md / SKILL.md) wrapping ``body``.""" + body = body.strip() or f"_No {title.lower()} yet._" + return f"# {title}\n\n{_GENERATED_NOTICE}\n\n{body}\n" + + @staticmethod + def _recall_file_slugs(files: list[RecallFile]) -> tuple[list[RecallFile], list[str]]: + """Order files deterministically and assign a unique slug to each. + + Returns the ordered files alongside a parallel list of slugs (used for the + ``/.md`` file names and the root-index links). Slug clashes are + de-duplicated with a numeric suffix so files never collide. + """ + ordered = sorted(files, key=lambda c: (c.name.lower(), c.id)) + slugs: list[str] = [] + used: dict[str, int] = {} + for rf in ordered: + base = slugify(rf.name) + count = used.get(base, 0) + used[base] = count + 1 + slugs.append(base if count == 0 else f"{base}-{count + 1}") + return ordered, slugs + + def _recall_file_document(self, rf: RecallFile) -> str: + """A single ``/.md`` file: the RecallFile's description + content.""" + description = self._inline((rf.description or "").strip()) + content = (rf.content or "").strip() + lines = [f"# {rf.name}", "", _GENERATED_NOTICE, ""] + if description: + lines.append(f"_{description}_") + lines.append("") + lines.append(content or "_No content yet._") + lines.append("") + return "\n".join(lines) + + def _recall_file_index(self, title: str, dirname: str, ordered: list[RecallFile], slugs: list[str]) -> str: + """The deterministic root index: an overview linking to each per-file payload.""" + lines = [f"# {title}", "", _GENERATED_NOTICE, "", "## Overview", ""] + if ordered: + for rf, slug in zip(ordered, slugs, strict=True): + description = self._inline((rf.description or "").strip()) + link = f"{dirname}/{slug}.md" + line = f"- [**{rf.name}**]({link})" + if description: + line = f"{line} — {description}" + lines.append(line) + else: + lines.append(f"_No {title.lower()} yet._") + lines.append("") + return "\n".join(lines) + + # -- bypass: RESOURCE + INDEX ------------------------------------------ + + @staticmethod + def _resource_artifacts( + descriptions: list[FileDescription], + ) -> tuple[dict[str, bytes], dict[str, str]]: + """Copy each readable source file into ``resource/`` (verbatim bytes). + + Returns the ``resource/`` -> raw bytes artifact map plus a + ``resource_id`` -> relative path map used to link ``INDEX.md`` entries to + their copies. Resources without a readable ``local_path`` are skipped (they + are still listed in INDEX.md, just without a link). File-name collisions are + de-duplicated with a numeric suffix so copies never clobber each other. + """ + raw: dict[str, bytes] = {} + links: dict[str, str] = {} + used: dict[str, int] = {} + for desc in descriptions: + if not desc.local_path: + continue + path = pathlib.Path(desc.local_path) + if not path.is_file(): + continue + try: + content = path.read_bytes() + except OSError: + continue + name = desc.file_name + count = used.get(name, 0) + used[name] = count + 1 + out_name = name if count == 0 else MemoryFileExporter._suffix_name(name, count + 1) + rel = f"{RESOURCE_DIRNAME}/{out_name}" + raw[rel] = content + links[desc.resource_id] = rel + return raw, links + + @staticmethod + def _suffix_name(name: str, n: int) -> str: + """Insert a ``-n`` disambiguator before the extension (``a.txt`` -> ``a-2.txt``).""" + stem, dot, ext = name.rpartition(".") + if dot: + return f"{stem}-{n}.{ext}" + return f"{name}-{n}" + + def _index_bypass(self, descriptions: list[FileDescription], links: dict[str, str]) -> str: + """An index of the raw source files copied into ``resource/``.""" + lines = ["# Index", "", _GENERATED_NOTICE, "", "## Files", ""] + if descriptions: + for desc in descriptions: + text = desc.description or "_No description._" + rel = links.get(desc.resource_id) + label = f"[`{desc.file_name}`]({rel})" if rel else f"`{desc.file_name}`" + lines.append(f"- {label} ({desc.modality}) — {text}") + else: + lines.append("_No source files yet._") + lines.append("") + return "\n".join(lines) + + # -- shared helpers ---------------------------------------------------- + + @staticmethod + def _inline(text: str) -> str: + """Collapse a value to a single safe line.""" + return " ".join(text.split()) + + # -- diff / write ------------------------------------------------------ + + def _sync(self, artifacts: dict[str, str | bytes]) -> ExportResult: + manifest = self._load_manifest() + result = ExportResult(output_dir=str(self.output_dir)) + new_manifest: dict[str, str] = {} + + for rel_path, content in artifacts.items(): + digest = _content_hash(content) + new_manifest[rel_path] = digest + target = self.output_dir / rel_path + if manifest.get(rel_path) == digest and target.exists(): + result.unchanged.append(rel_path) + continue + target.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + target.write_bytes(content) + else: + target.write_text(content, encoding="utf-8") + result.written.append(rel_path) + + for rel_path in manifest: + if rel_path in new_manifest: + continue + stale = self.output_dir / rel_path + if stale.exists(): + stale.unlink() + self._prune_empty_dirs(stale.parent) + result.removed.append(rel_path) + + self._save_manifest(new_manifest) + return result + + def _prune_empty_dirs(self, directory: pathlib.Path) -> None: + """Remove now-empty directories created for nested artifacts (e.g. skill/).""" + root = self.output_dir.resolve() + current = directory + while current.resolve() != root and root in current.resolve().parents: + try: + next(current.iterdir()) + except StopIteration: + current.rmdir() + current = current.parent + except OSError: + break + else: + break + + # -- reading existing artifacts (for incremental update) --------------- + + def artifacts_exist(self) -> bool: + """Whether a prior memory tree is already present (init vs update).""" + return (self.output_dir / MEMORY_FILENAME).exists() + + def read_existing(self) -> ExistingArtifacts: + """Load the prior MEMORY/SKILL overview bodies as a single bundle for merging.""" + return ExistingArtifacts(memory_body=self.read_memory_body(), skill_body=self.read_skill_body()) + + def read_memory_body(self) -> str: + """Read MEMORY.md and strip the heading/notice, returning just the body.""" + return self._read_overview_body(MEMORY_FILENAME, drop_heading="# Memory") + + def read_skill_body(self) -> str: + """Read SKILL.md and strip the heading/notice, returning just the body.""" + return self._read_overview_body(SKILL_INDEX_FILENAME, drop_heading="# Skills") + + def _read_overview_body(self, filename: str, *, drop_heading: str) -> str: + path = self.output_dir / filename + if not path.exists(): + return "" + return self._strip_chrome(path.read_text(encoding="utf-8"), drop_heading=drop_heading) + + @staticmethod + def _strip_chrome(text: str, *, drop_heading: str | None = None) -> str: + lines = text.splitlines() + kept = [ + line + for line in lines + if line.strip() != _GENERATED_NOTICE and (drop_heading is None or line.strip() != drop_heading) + ] + return "\n".join(kept).strip() + + def _load_manifest(self) -> dict[str, str]: + manifest_path = self.output_dir / MANIFEST_NAME + if not manifest_path.exists(): + return {} + try: + data = json.loads(manifest_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return {} + if not isinstance(data, dict): + return {} + return {str(key): str(value) for key, value in data.items()} + + def _save_manifest(self, manifest: dict[str, str]) -> None: + manifest_path = self.output_dir / MANIFEST_NAME + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + + +__all__ = ["ExistingArtifacts", "ExportResult", "FileDescription", "MemoryFileExporter", "slugify"] diff --git a/src/memu/memory_fs/synthesizer.py b/src/memu/memory_fs/synthesizer.py new file mode 100644 index 0000000..65cfd80 --- /dev/null +++ b/src/memu/memory_fs/synthesizer.py @@ -0,0 +1,115 @@ +"""LLM synthesis of the MEMORY/SKILL overview documents. + +This is the optional, opt-in counterpart to the deterministic exporter. It synthesizes +the two root overview documents — ``MEMORY.md`` and ``SKILL.md`` — that sit above the +per-file payloads: + +- ``MEMORY.md`` is synthesized from the per-source multimodal descriptions (the source + trunk), merged into the prior overview. +- ``SKILL.md`` is synthesized from the bodies of the skill-track ``RecallFile``s (the + skill trunk), merged into the prior overview. + +The per-file payloads themselves (``memory/.md``, ``skill/.md``) and +``INDEX.md`` stay deterministic and are handled by the exporter. Skills are generated +and persisted inside the memorize workflow (ADR 0006); this layer only summarizes them. +""" + +from __future__ import annotations + +import re +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING + +from memu.prompts.memory_fs import ( + DESCRIPTIONS_PLACEHOLDER, + EXISTING_PLACEHOLDER, + MEMORY_SYNTHESIS_PROMPT, + SKILL_OVERVIEW_SYNTHESIS_PROMPT, +) + +if TYPE_CHECKING: + from memu.database.models import RecallFile + from memu.memory_fs.exporter import FileDescription + +ChatFn = Callable[[str], Awaitable[str]] + + +class MemorySynthesizer: + """Synthesize the MEMORY/SKILL overview documents via an LLM.""" + + def __init__( + self, + *, + memory_prompt: str = MEMORY_SYNTHESIS_PROMPT, + skill_prompt: str = SKILL_OVERVIEW_SYNTHESIS_PROMPT, + ) -> None: + self._memory_prompt = memory_prompt + self._skill_prompt = skill_prompt + + async def synthesize_memory( + self, + descriptions: list[FileDescription], + *, + existing_memory: str = "", + chat: ChatFn, + ) -> str: + """Synthesize the ``MEMORY.md`` body from the per-source descriptions. + + Empty ``existing_memory`` builds from scratch; a populated value is merged into. + Returns the prior body unchanged when there is nothing to synthesize. + """ + formatted = self._format_descriptions(descriptions) + if not formatted: + return existing_memory + prompt = self._memory_prompt.replace(EXISTING_PLACEHOLDER, existing_memory.strip() or "(empty)").replace( + DESCRIPTIONS_PLACEHOLDER, formatted + ) + return self._clean_markdown(await chat(prompt)) + + async def synthesize_skill_overview( + self, + skills: list[RecallFile], + *, + existing_skill: str = "", + chat: ChatFn, + ) -> str: + """Synthesize the ``SKILL.md`` overview from the skill-track files' bodies. + + The skill files are the full accumulated library (the memorize workflow merges + per source), so the overview is regenerated from the whole set each time, merged + into any prior overview. Returns the prior overview unchanged when empty. + """ + formatted = self._format_skills(skills) + if not formatted: + return existing_skill + prompt = self._skill_prompt.replace(EXISTING_PLACEHOLDER, existing_skill.strip() or "(empty)").replace( + DESCRIPTIONS_PLACEHOLDER, formatted + ) + return self._clean_markdown(await chat(prompt)) + + @staticmethod + def _format_descriptions(descriptions: list[FileDescription]) -> str: + lines = [ + f"- [{desc.modality}] {desc.url}: {desc.description}" for desc in descriptions if desc.description.strip() + ] + return "\n".join(lines) + + @staticmethod + def _format_skills(skills: list[RecallFile]) -> str: + blocks = [ + f"## {skill.name}\n{(skill.content or '').strip()}".strip() + for skill in sorted(skills, key=lambda s: s.name) + if (skill.content or "").strip() + ] + return "\n\n".join(blocks) + + @staticmethod + def _clean_markdown(raw: str) -> str: + text = (raw or "").strip() + if text.startswith("```"): + text = re.sub(r"^```[a-zA-Z]*\n", "", text) + text = re.sub(r"\n```$", "", text).strip() + return text + + +__all__ = ["ChatFn", "MemorySynthesizer"] diff --git a/src/memu/preprocess/__init__.py b/src/memu/preprocess/__init__.py new file mode 100644 index 0000000..9f58c71 --- /dev/null +++ b/src/memu/preprocess/__init__.py @@ -0,0 +1,99 @@ +"""Resource preprocessing, split by format/modality. + +This package extracts the per-format preprocessing logic out of the memorize +flow. Each modality has a dedicated :class:`~memu.preprocess.base.Preprocessor` +implementation, and :func:`preprocess_resource` dispatches to the right one based +on the resource modality. +""" + +from __future__ import annotations + +from typing import Any + +from memu.preprocess.audio import AudioPreprocessor, prepare_audio_text +from memu.preprocess.base import ( + PreprocessContext, + Preprocessor, + PreprocessResult, + parse_multimodal_response, +) +from memu.preprocess.conversation import ConversationPreprocessor +from memu.preprocess.document import DocumentPreprocessor +from memu.preprocess.image import ImagePreprocessor +from memu.preprocess.video import VideoPreprocessor +from memu.prompts.preprocess import PROMPTS as PREPROCESS_PROMPTS + +# Registry mapping modality -> preprocessor. Register new formats here. +PREPROCESSORS: dict[str, Preprocessor] = { + ConversationPreprocessor.modality: ConversationPreprocessor(), + VideoPreprocessor.modality: VideoPreprocessor(), + ImagePreprocessor.modality: ImagePreprocessor(), + DocumentPreprocessor.modality: DocumentPreprocessor(), + AudioPreprocessor.modality: AudioPreprocessor(), +} + +__all__ = [ + "PREPROCESSORS", + "AudioPreprocessor", + "ConversationPreprocessor", + "DocumentPreprocessor", + "ImagePreprocessor", + "PreprocessContext", + "PreprocessResult", + "Preprocessor", + "VideoPreprocessor", + "parse_multimodal_response", + "preprocess_resource", +] + + +def _resolve_template(modality: str, ctx: PreprocessContext) -> str | None: + configured_prompt = ctx.multimodal_preprocess_prompts.get(modality) + if configured_prompt is None: + return PREPROCESS_PROMPTS.get(modality) + if isinstance(configured_prompt, str): + return configured_prompt + # No custom prompts configured for preprocessing for now. If the user decides + # to use their custom prompt, they must provide ALL prompt blocks. + return ctx.resolve_custom_prompt(configured_prompt, {}) + + +async def preprocess_resource( + *, + modality: str, + local_path: str, + text: str | None, + ctx: PreprocessContext, + llm_client: Any | None = None, +) -> PreprocessResult: + """Preprocess a resource based on its modality. + + - Text-based modalities (conversation, document): require text content. + - Audio: transcribe (or read sidecar text) first, then process as text. + - Media modalities (video, image): process media files directly. + + Returns a list of preprocessed resources, each with ``text`` and ``caption``. + """ + template = _resolve_template(modality, ctx) + if not template: + return [{"text": text, "caption": None}] + + if modality == "audio": + text = await prepare_audio_text(local_path, text, ctx, llm_client=llm_client) + if text is None: + return [{"text": None, "caption": None}] + + preprocessor = PREPROCESSORS.get(modality) + if preprocessor is None: + return [{"text": text, "caption": None}] + + if preprocessor.requires_text and not text: + return [{"text": text, "caption": None}] + + return await preprocessor.run( + local_path=local_path, + text=text, + template=template, + ctx=ctx, + llm_client=llm_client, + ) diff --git a/src/memu/preprocess/audio.py b/src/memu/preprocess/audio.py new file mode 100644 index 0000000..8b8e793 --- /dev/null +++ b/src/memu/preprocess/audio.py @@ -0,0 +1,83 @@ +"""Audio preprocessing. + +Audio is first turned into text (via transcription or a sidecar text file) by +``prepare_audio_text`` in :mod:`memu.preprocess`, then formatted and captioned here. +""" + +from __future__ import annotations + +import logging +import pathlib +from typing import Any, cast + +from memu.preprocess.base import ( + PreprocessContext, + Preprocessor, + PreprocessResult, + parse_multimodal_response, +) + +logger = logging.getLogger(__name__) + +AUDIO_EXTENSIONS = {".mp3", ".mp4", ".mpeg", ".mpga", ".m4a", ".wav", ".webm"} +TEXT_EXTENSIONS = {".txt", ".text"} + + +class AudioPreprocessor(Preprocessor): + modality = "audio" + requires_text = False + + async def run( + self, + *, + local_path: str, + text: str | None, + template: str, + ctx: PreprocessContext, + llm_client: Any | None = None, + ) -> PreprocessResult: + assert text is not None # transcription resolved before dispatch # noqa: S101 + prompt = template.format(transcription=ctx.escape_prompt_value(text)) + client = llm_client or ctx.get_llm_client() + processed = await client.chat(prompt) + processed_content, caption = parse_multimodal_response(processed, "processed_content", "caption") + return [{"text": processed_content or text, "caption": caption}] + + +async def prepare_audio_text( + local_path: str, + text: str | None, + ctx: PreprocessContext, + llm_client: Any | None = None, +) -> str | None: + """Ensure audio resources provide text via transcription or file read.""" + if text: + return text + + file_ext = pathlib.Path(local_path).suffix.lower() + + if file_ext in AUDIO_EXTENSIONS: + try: + logger.info(f"Transcribing audio file: {local_path}") + client = llm_client or ctx.get_llm_client() + transcribed = cast(str, await client.transcribe(local_path)) + logger.info(f"Audio transcription completed: {len(transcribed)} characters") + except Exception: + logger.exception("Audio transcription failed for %s", local_path) + return None + else: + return transcribed + + if file_ext in TEXT_EXTENSIONS: + path_obj = pathlib.Path(local_path) + try: + text_content = path_obj.read_text(encoding="utf-8") + logger.info(f"Read pre-transcribed text file: {len(text_content)} characters") + except Exception: + logger.exception("Failed to read text file %s", local_path) + return None + else: + return text_content + + logger.warning(f"Unknown audio file type: {file_ext}, skipping transcription") + return None diff --git a/src/memu/preprocess/base.py b/src/memu/preprocess/base.py new file mode 100644 index 0000000..9e493a9 --- /dev/null +++ b/src/memu/preprocess/base.py @@ -0,0 +1,84 @@ +"""Shared types and parsing helpers for resource preprocessing. + +Preprocessors turn a fetched resource (text and/or a local file) into a list of +``{"text", "caption"}`` segments that downstream memory extraction consumes. +Each modality (conversation, document, video, image, audio) has its own +implementation under this package; :mod:`memu.preprocess` wires them together. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any, ClassVar + +# A preprocessed resource is a list of segments, each carrying optional text and +# an optional short caption. +PreprocessResult = list[dict[str, str | None]] + + +@dataclass(frozen=True) +class PreprocessContext: + """Dependencies a preprocessor needs from the owning memory service. + + Bundled explicitly so the per-format logic stays decoupled from the large + ``MemorizeMixin`` while preserving its exact behavior. + """ + + get_llm_client: Callable[[], Any] + get_vlm_client: Callable[[], Any] + escape_prompt_value: Callable[[str], str] + extract_json_blob: Callable[[str], str] + resolve_custom_prompt: Callable[[Any, Mapping[str, str]], str] + multimodal_preprocess_prompts: Mapping[str, Any] + + +class Preprocessor: + """Base class for modality-specific preprocessors.""" + + modality: ClassVar[str] = "base" + # Whether the modality cannot proceed without resolved text content. + requires_text: ClassVar[bool] = False + + async def run( + self, + *, + local_path: str, + text: str | None, + template: str, + ctx: PreprocessContext, + llm_client: Any | None = None, + ) -> PreprocessResult: + raise NotImplementedError + + +def extract_tag_content(raw: str, tag: str) -> str | None: + """Extract inner text of an XML-like ``...`` block.""" + pattern = re.compile(rf"<{tag}>(.*?)", re.IGNORECASE | re.DOTALL) + match = pattern.search(raw) + if not match: + return None + content = match.group(1).strip() + return content or None + + +def parse_multimodal_response(raw: str, content_tag: str, caption_tag: str) -> tuple[str | None, str | None]: + """Parse a multimodal preprocessing response (video, image, document, audio). + + Extracts content and caption from XML-like tags, with fallbacks when tags are + absent. + """ + content = extract_tag_content(raw, content_tag) + caption = extract_tag_content(raw, caption_tag) + + # Fallback: if no tags found, use the raw response as content. + if not content: + content = raw.strip() + + # Fallback for caption: use the first sentence of content if none found. + if not caption and content: + first_sentence = content.split(".")[0] + caption = first_sentence if len(first_sentence) <= 200 else first_sentence[:200] + + return content, caption diff --git a/src/memu/preprocess/conversation.py b/src/memu/preprocess/conversation.py new file mode 100644 index 0000000..b5559b9 --- /dev/null +++ b/src/memu/preprocess/conversation.py @@ -0,0 +1,42 @@ +"""Conversation (chat) preprocessing. + +Normalizes a chat log into an indexed, line-based transcript and returns it as a +single resource. The conversation is intentionally kept whole (no segmentation): +downstream memory extraction sees the complete, original transcript instead of an +LLM-rewritten or split version. +""" + +from __future__ import annotations + +from typing import Any + +from memu.preprocess.base import ( + PreprocessContext, + Preprocessor, + PreprocessResult, +) +from memu.utils.conversation import format_conversation_for_preprocess + + +class ConversationPreprocessor(Preprocessor): + modality = "conversation" + requires_text = True + + async def run( + self, + *, + local_path: str, + text: str | None, + template: str, + ctx: PreprocessContext, + llm_client: Any | None = None, + ) -> PreprocessResult: + assert text is not None # guaranteed by requires_text dispatch # noqa: S101 + # Use the original JSON-derived, indexed conversation text as-is. We do not + # segment or summarize: keeping the whole transcript avoids dropping fields + # (e.g. created_at) that an LLM rewrite/segmentation pass might lose. + # ``template``/``llm_client`` are unused here but kept to satisfy the + # ``Preprocessor.run`` contract (and template must stay non-empty so this + # modality is dispatched through the text-normalization path). + conversation_text = format_conversation_for_preprocess(text) + return [{"text": conversation_text, "caption": None}] diff --git a/src/memu/preprocess/document.py b/src/memu/preprocess/document.py new file mode 100644 index 0000000..ca5276f --- /dev/null +++ b/src/memu/preprocess/document.py @@ -0,0 +1,36 @@ +"""Document (log / text) preprocessing. + +Condenses document text and extracts a short caption. +""" + +from __future__ import annotations + +from typing import Any + +from memu.preprocess.base import ( + PreprocessContext, + Preprocessor, + PreprocessResult, + parse_multimodal_response, +) + + +class DocumentPreprocessor(Preprocessor): + modality = "document" + requires_text = True + + async def run( + self, + *, + local_path: str, + text: str | None, + template: str, + ctx: PreprocessContext, + llm_client: Any | None = None, + ) -> PreprocessResult: + assert text is not None # guaranteed by requires_text dispatch # noqa: S101 + prompt = template.format(document_text=ctx.escape_prompt_value(text)) + client = llm_client or ctx.get_llm_client() + processed = await client.chat(prompt) + processed_content, caption = parse_multimodal_response(processed, "processed_content", "caption") + return [{"text": processed_content or text, "caption": caption}] diff --git a/src/memu/preprocess/image.py b/src/memu/preprocess/image.py new file mode 100644 index 0000000..e6b7537 --- /dev/null +++ b/src/memu/preprocess/image.py @@ -0,0 +1,34 @@ +"""Image preprocessing. + +Extracts a detailed description and caption from an image using the Vision API. +""" + +from __future__ import annotations + +from typing import Any + +from memu.preprocess.base import ( + PreprocessContext, + Preprocessor, + PreprocessResult, + parse_multimodal_response, +) + + +class ImagePreprocessor(Preprocessor): + modality = "image" + requires_text = False + + async def run( + self, + *, + local_path: str, + text: str | None, + template: str, + ctx: PreprocessContext, + llm_client: Any | None = None, + ) -> PreprocessResult: + client = llm_client or ctx.get_vlm_client() + processed = await client.vision(prompt=template, image_path=local_path, system_prompt=None) + description, caption = parse_multimodal_response(processed, "detailed_description", "caption") + return [{"text": description, "caption": caption}] diff --git a/src/memu/preprocess/video.py b/src/memu/preprocess/video.py new file mode 100644 index 0000000..a18363f --- /dev/null +++ b/src/memu/preprocess/video.py @@ -0,0 +1,60 @@ +"""Video preprocessing. + +The whole video file is analyzed natively in a single call (e.g. video-capable +models via OpenRouter's ``video_url`` content type) so its sampled frames and +temporal changes inform the description. Native video understanding is required: +when the configured VLM client does not support it, the video is skipped rather +than degraded to a single still frame. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from memu.preprocess.base import ( + PreprocessContext, + Preprocessor, + PreprocessResult, + parse_multimodal_response, +) + +logger = logging.getLogger(__name__) + + +class VideoPreprocessor(Preprocessor): + modality = "video" + requires_text = False + + async def run( + self, + *, + local_path: str, + text: str | None, + template: str, + ctx: PreprocessContext, + llm_client: Any | None = None, + ) -> PreprocessResult: + try: + client = llm_client or ctx.get_vlm_client() + + if not getattr(client, "supports_video", False): + logger.error( + "Video preprocessing requires a VLM client with native video support " + "(supports_video=True), e.g. an OpenRouter video-capable model. " + "Middle-frame fallback is disabled; skipping video: %s", + local_path, + ) + return [{"text": None, "caption": None}] + + return await self._run_native(local_path=local_path, template=template, client=client) + except Exception as e: + logger.error(f"Video preprocessing failed: {e}", exc_info=True) + return [{"text": None, "caption": None}] + + async def _run_native(self, *, local_path: str, template: str, client: Any) -> PreprocessResult: + """Analyze the whole video natively (frames + audio) in a single call.""" + logger.info(f"Analyzing video natively with VLM video API: {local_path}") + processed = await client.video(prompt=template, video_path=local_path, system_prompt=None) + description, caption = parse_multimodal_response(processed, "detailed_description", "caption") + return [{"text": description, "caption": caption}] diff --git a/src/memu/prompts/__init__.py b/src/memu/prompts/__init__.py new file mode 100644 index 0000000..fbfe2dc --- /dev/null +++ b/src/memu/prompts/__init__.py @@ -0,0 +1,13 @@ +from memu.prompts.category_summary import PROMPT as CATEGORY_SUMMARY_PROMPT +from memu.prompts.memory_type import DEFAULT_MEMORY_TYPES +from memu.prompts.memory_type import PROMPTS as MEMORY_TYPE_PROMPTS +from memu.prompts.preprocess import PROMPTS as PREPROCESS_PROMPTS +from memu.prompts.retrieve.judger import PROMPT as RETRIEVE_JUDGER_PROMPT + +__all__ = [ + "CATEGORY_SUMMARY_PROMPT", + "DEFAULT_MEMORY_TYPES", + "MEMORY_TYPE_PROMPTS", + "PREPROCESS_PROMPTS", + "RETRIEVE_JUDGER_PROMPT", +] diff --git a/src/memu/prompts/category_patch/__init__.py b/src/memu/prompts/category_patch/__init__.py new file mode 100644 index 0000000..63ab090 --- /dev/null +++ b/src/memu/prompts/category_patch/__init__.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from memu.prompts.category_patch.category import PROMPT + +CATEGORY_PATCH_PROMPT = PROMPT.strip() + +__all__ = ["CATEGORY_PATCH_PROMPT"] diff --git a/src/memu/prompts/category_patch/category.py b/src/memu/prompts/category_patch/category.py new file mode 100644 index 0000000..50ade9c --- /dev/null +++ b/src/memu/prompts/category_patch/category.py @@ -0,0 +1,45 @@ +PROMPT = """ +# Task Objective +Your task is to read an existing user profile and an update related to a specific memory topic, then determine whether the profile needs to be updated. +If an update is required, you must generate the updated version of the profile based on the rules below. + +# Workflow +1. Understand the Topic +Focus only on memories relevant to the specified Topic. + +2. Analyze Original Content +Review the existing profile content enclosed in .... + +3. Analyze Update +Determine whether the update represents: +- A new memory +- A variation of an existing memory +- A discarded (invalidated) memory + +4. Decision Making +Judge whether the profile requires modification based on relevance and importance. + +5. Generate Output +- If an update is required, produce the revised profile content. +- If not, explicitly indicate that no update is needed. + + +# Response Format (JSON): +{{ + "need_update": [bool, whether the profile needs to be updated] + "updated_content": [str, the updated content of the profile if need_update is true, otherwise empty] +}} + + +# Input +Topic: +{category} + +Original content: + +{original_content} + + +Update: +{update_content} +""" diff --git a/src/memu/prompts/category_summary/__init__.py b/src/memu/prompts/category_summary/__init__.py new file mode 100644 index 0000000..a3e6040 --- /dev/null +++ b/src/memu/prompts/category_summary/__init__.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from memu.prompts.category_summary.category import CUSTOM_PROMPT, PROMPT +from memu.prompts.category_summary.category_with_refs import CUSTOM_PROMPT as CUSTOM_PROMPT_WITH_REFS +from memu.prompts.category_summary.category_with_refs import PROMPT as PROMPT_WITH_REFS + +DEFAULT_CATEGORY_SUMMARY_PROMPT_ORDINAL: dict[str, int] = { + "objective": 10, + "workflow": 20, + "rules": 30, + "output": 40, + "examples": 50, + "input": 90, +} + +__all__ = [ + "CUSTOM_PROMPT", + "CUSTOM_PROMPT_WITH_REFS", + "DEFAULT_CATEGORY_SUMMARY_PROMPT_ORDINAL", + "PROMPT", + "PROMPT_WITH_REFS", +] diff --git a/src/memu/prompts/category_summary/category.py b/src/memu/prompts/category_summary/category.py new file mode 100644 index 0000000..509de40 --- /dev/null +++ b/src/memu/prompts/category_summary/category.py @@ -0,0 +1,296 @@ +PROMPT_LEGACY = """ +# Task Objective +You are a professional User Profile Synchronization Specialist. Your core objective is to accurately merge newly extracted user information items into the user's initial profile using only two operations: add and update. +Because no original conversation text is provided, active deletion is not allowed; only implicit replacement through newer items is permitted. The final output must be the updated, complete user profile. + +# Workflow +## Step 1: Preprocessing & Parsing +- Input sources +User Initial Profile: structured, categorized, confirmed long-term user information. +Newly Extracted User Information Items. +- Structure parsing +Initial profile: extract categories and core content; preserve original wording style and format; build a category-content mapping. +New items: validate completeness and category correctness; mark each as Add or Update; distinguish stable facts from event-type information; extract dates/times (events only). +- Pre-validation +Verify subject accuracy: clearly distinguish the user from related persons (family, friends, etc.). +Remove invalid items: vague, miscategorized, or non-user-information items. +Remove one-off events: temporary actions without long-term relevance (e.g., what the user ate today). + +## Step 2: Core Operations (Update / Add) +A. Update +Conflict detection: compare new items with existing ones in the same category for semantic overlap (e.g., age update). +Validity priority: retain information that is more specific, clearer, and more certain. +Overwrite / supplement: replace outdated entries with new ones, ensuring no loss of core information. +Time integration (events only): retain dates/times and integrate them naturally; multiple events at the same time may be layered, but each entry must remain independently understandable. +B. Add +Deduplication check: ensure the new item is not identical or semantically similar to existing or updated items. +Category matching: place the item into the correct predefined category. +Insertion: add the item following the original profile's language and formatting style, concise and clear. + +## Step 3: Merge & Formatting +Structured ordering: present content by category order; omit empty categories. +Formatting rules: strictly use Markdown (# for main title, ## for category titles). +Final validation +Consistency: no contradictions or duplicates. +Compliance: correct categories only; no explanatory or operational text. +Accuracy: subject clarity; natural time embedding; proper format. + + +## Step 4: Summarize +Target length: {target_length} +Summarize the updated user markdown profile to the target length. +Use Markdown hierarchy. +Do not include explanations, operation traces, or meta text. +Control item length strictly; prioritize core information if needed. + +## Step 5: Output +Output only the updated user markdown profile. +Use Markdown hierarchy. +Do not include explanations, operation traces, or meta text. +Control item length strictly; prioritize core information if needed. + + + +# Output Format (Markdown) +```markdown +# {category} +## +- User information item +- User information item +... +## +- User information item +- User information item +... +``` + +# Examples (Input / Output / Explanation) +- Example 1: Basic Add & Update + + +Topic: +Personal Basic Information + +Original content: + +# Personal Basic Information +## Basic Information +- The user is 28 years old +- The user currently lives in Beijing +## Basic Preferences +- The user likes spicy food +## Core Traits +- The user is extroverted + + +New memory items: + +- The user is 30 years old +- The user currently lives in Shanghai +- The user prefers Sichuan-style spicy food and dislikes sweet-spicy flavors +- The user enjoys hiking on weekends +- The user is meticulous +- The user ate Malatang today + + +Output +# Personal Basic Information +## Basic Information +- The user is 30 years old +- The user currently lives in Shanghai +## Basic Preferences +- The user prefers Sichuan-style spicy food and dislikes sweet-spicy flavors +- The user enjoys hiking on weekends +## Core Traits +- The user is extroverted +- The user is meticulous + +Explanation +The "The user ate Malatang today" is a one-time daily action without long-term value and is therefore excluded. + + +Your task is to read and analyze existing content and some new memory items, and then selectively update the content to reflect both the existing and new information. + + +# Input + +Topic: +{category} + +Original content: + +{original_content} + + +New memory items: + +{new_recall_entries_text} + + + +# Output format (Markdown) +```markdown +# {category} +## +- User information item +- User information item +... +## +- User information item +- User information item +... +``` +""" + + +PROMPT_BLOCK_OBJECTIVE = """ +# Task Objective +You are a professional User Profile Synchronization Specialist. Your core objective is to accurately merge newly extracted user information items into the user's initial profile using only two operations: add and update. +Because no original conversation text is provided, active deletion is not allowed; only implicit replacement through newer items is permitted. The final output must be the updated, complete user profile. +""" + +PROMPT_BLOCK_WORKFLOW = """ +# Workflow +## Step 1: Preprocessing & Parsing +- Input sources +User Initial Profile: structured, categorized, confirmed long-term user information. +Newly Extracted User Information Items. +- Structure parsing +Initial profile: extract categories and core content; preserve original wording style and format; build a category-content mapping. +New items: validate completeness and category correctness; mark each as Add or Update; distinguish stable facts from event-type information; extract dates/times (events only). +- Pre-validation +Verify subject accuracy: clearly distinguish the user from related persons (family, friends, etc.). +Remove invalid items: vague, miscategorized, or non-user-information items. +Remove one-off events: temporary actions without long-term relevance (e.g., what the user ate today). + +## Step 2: Core Operations (Update / Add) +A. Update +Conflict detection: compare new items with existing ones in the same category for semantic overlap (e.g., age update). +Validity priority: retain information that is more specific, clearer, and more certain. +Overwrite / supplement: replace outdated entries with new ones, ensuring no loss of core information. +Time integration (events only): retain dates/times and integrate them naturally; multiple events at the same time may be layered, but each entry must remain independently understandable. +B. Add +Deduplication check: ensure the new item is not identical or semantically similar to existing or updated items. +Category matching: place the item into the correct predefined category. +Insertion: add the item following the original profile's language and formatting style, concise and clear. + +## Step 3: Merge & Formatting +Structured ordering: present content by category order; omit empty categories. +Formatting rules: strictly use Markdown (# for main title, ## for category titles). +Final validation +Consistency: no contradictions or duplicates. +Compliance: correct categories only; no explanatory or operational text. +Accuracy: subject clarity; natural time embedding; proper format. + + +## Step 4: Summarize +Target length: {target_length} +Summarize the updated user markdown profile to the target length. +Use Markdown hierarchy. +Cluster the memory items and update the . + +## Step 5: Output +Output only the updated user markdown profile. +Use Markdown hierarchy. +Do not include explanations, operation traces, or meta text. +Control item length strictly; prioritize core information if needed. +""" + +PROMPT_BLOCK_RULES = """ +""" + +PROMPT_BLOCK_OUTPUT = """ +# Output Format (Markdown) +```markdown +# {category} +## +- User information item +- User information item +## +- User information item +- User information item +``` + +# Critical +Always ensure that your output does not exceed {target_length} tokens. +You may merge or omit unimportant information to meet this limit. +""" + +PROMPT_BLOCK_EXAMPLES = """ +# Examples (Input / Output / Explanation) +- Example 1: Basic Add & Update + +Topic: +Personal Basic Information + +Original content: + +# Personal Basic Information +## Basic Information +- The user is 28 years old +- The user currently lives in Beijing +## Basic Preferences +- The user likes spicy food +## Core Traits +- The user is extroverted + + +New memory items: + +- The user is 30 years old +- The user currently lives in Shanghai +- The user prefers Sichuan-style spicy food and dislikes sweet-spicy flavors +- The user enjoys hiking on weekends +- The user is meticulous +- The user ate Malatang today + + +Output +# Personal Basic Information +## Basic Information +- The user is 30 years old +- The user currently lives in Shanghai +## Basic Preferences +- The user prefers Sichuan-style spicy food and dislikes sweet-spicy flavors +- The user enjoys hiking on weekends +## Core Traits +- The user is extroverted +- The user is meticulous + +Explanation +The "The user ate Malatang today" is a one-time daily action without long-term value and is therefore excluded. +""" + +PROMPT_BLOCK_INPUT = """ +# Input +Topic: +{category} + +Original content: + +{original_content} + + +New memory items: + +{new_recall_entries_text} + +""" + +PROMPT = "\n\n".join([ + PROMPT_BLOCK_OBJECTIVE.strip(), + PROMPT_BLOCK_WORKFLOW.strip(), + PROMPT_BLOCK_RULES.strip(), + PROMPT_BLOCK_OUTPUT.strip(), + PROMPT_BLOCK_EXAMPLES.strip(), + PROMPT_BLOCK_INPUT.strip(), +]) + +CUSTOM_PROMPT = { + "objective": PROMPT_BLOCK_OBJECTIVE.strip(), + "workflow": PROMPT_BLOCK_WORKFLOW.strip(), + "rules": PROMPT_BLOCK_RULES.strip(), + "output": PROMPT_BLOCK_OUTPUT.strip(), + "examples": PROMPT_BLOCK_EXAMPLES.strip(), + "input": PROMPT_BLOCK_INPUT.strip(), +} diff --git a/src/memu/prompts/category_summary/category_with_refs.py b/src/memu/prompts/category_summary/category_with_refs.py new file mode 100644 index 0000000..2240fcd --- /dev/null +++ b/src/memu/prompts/category_summary/category_with_refs.py @@ -0,0 +1,140 @@ +""" +Category summary prompt with inline references to memory items. + +This prompt instructs the LLM to include [ref:ITEM_ID] citations +when summarizing category content, linking statements to their +source memory items. +""" + +PROMPT_BLOCK_OBJECTIVE = """ +# Task Objective +You are a professional User Profile Synchronization Specialist. Your core objective is to accurately merge newly extracted user information items into the user's initial profile using only two operations: add and update. + +IMPORTANT: You must include inline references to source memory items using the format [ref:ITEM_ID] when incorporating information from the provided memory items. This creates a traceable link between summary statements and their sources. +""" + +PROMPT_BLOCK_WORKFLOW = """ +# Workflow +## Step 1: Preprocessing & Parsing +- Input sources: + - User Initial Profile: structured, categorized, confirmed long-term user information. + - Newly Extracted User Information Items: each item has an ID that MUST be referenced. +- Structure parsing: + - Initial profile: extract categories and core content; preserve original wording style and format. + - New items: note the item ID for each piece of information to include as [ref:ID]. + +## Step 2: Core Operations (Update / Add) +A. Update +- When updating existing information with new data, add the reference: "User is 30 years old [ref:item_abc123]" +- If multiple items support the same fact, include multiple refs: [ref:id1,id2] + +B. Add +- When adding new information, always include the source reference +- Format: "User enjoys hiking on weekends [ref:item_xyz789]" + +## Step 3: Merge & Formatting +- Structured ordering: present content by category order; omit empty categories. +- Formatting rules: strictly use Markdown (# for main title, ## for category titles). +- References: ensure every new or updated fact has at least one [ref:ITEM_ID] citation. + +## Step 4: Summarize +Target length: {target_length} +- Summarize the updated user markdown profile to the target length. +- PRESERVE all [ref:ITEM_ID] citations in the summary. +- Use Markdown hierarchy. + +## Step 5: Output +- Output only the updated user markdown profile with inline references. +- Use Markdown hierarchy. +- Do not include explanations, operation traces, or meta text. +""" + +PROMPT_BLOCK_RULES = """ +# Reference Rules +1. Every piece of information from new memory items MUST have a [ref:ITEM_ID] citation +2. Use the exact item ID provided in the input +3. Place references immediately after the relevant statement +4. Multiple sources can be cited: [ref:id1,id2] +5. Existing information without new updates does not need references +""" + +PROMPT_BLOCK_OUTPUT = """ +# Output Format (Markdown with References) +```markdown +# {category} +## +- User information item [ref:ITEM_ID] +- User information item [ref:ITEM_ID] +## +- User information item [ref:ITEM_ID,ITEM_ID2] +``` + +# Critical +- Always ensure that your output does not exceed {target_length} tokens. +- ALWAYS include [ref:ITEM_ID] for information from new memory items. +- You may merge or omit unimportant information to meet the length limit. +""" + +PROMPT_BLOCK_EXAMPLES = """ +# Examples (Input / Output) + +Topic: +Personal Basic Information + +Original content: + +# Personal Basic Information +## Basic Information +- The user is 28 years old +- The user currently lives in Beijing + + +New memory items with IDs: + +- [item_a1b2c3] The user is 30 years old +- [item_d4e5f6] The user currently lives in Shanghai +- [item_g7h8i9] The user prefers Sichuan-style spicy food + + +Output: +# Personal Basic Information +## Basic Information +- The user is 30 years old [ref:item_a1b2c3] +- The user currently lives in Shanghai [ref:item_d4e5f6] +## Basic Preferences +- The user prefers Sichuan-style spicy food [ref:item_g7h8i9] +""" + +PROMPT_BLOCK_INPUT = """ +# Input +Topic: +{category} + +Original content: + +{original_content} + + +New memory items with IDs: + +{new_recall_entries_text} + +""" + +PROMPT = "\n\n".join([ + PROMPT_BLOCK_OBJECTIVE.strip(), + PROMPT_BLOCK_WORKFLOW.strip(), + PROMPT_BLOCK_RULES.strip(), + PROMPT_BLOCK_OUTPUT.strip(), + PROMPT_BLOCK_EXAMPLES.strip(), + PROMPT_BLOCK_INPUT.strip(), +]) + +CUSTOM_PROMPT = { + "objective": PROMPT_BLOCK_OBJECTIVE.strip(), + "workflow": PROMPT_BLOCK_WORKFLOW.strip(), + "rules": PROMPT_BLOCK_RULES.strip(), + "output": PROMPT_BLOCK_OUTPUT.strip(), + "examples": PROMPT_BLOCK_EXAMPLES.strip(), + "input": PROMPT_BLOCK_INPUT.strip(), +} diff --git a/src/memu/prompts/memory_fs/__init__.py b/src/memu/prompts/memory_fs/__init__.py new file mode 100644 index 0000000..2170a11 --- /dev/null +++ b/src/memu/prompts/memory_fs/__init__.py @@ -0,0 +1,213 @@ +"""Prompts for the resource -> file memorize path (ADR 0007 phase 1) and the +legacy memory_fs synthesis bypass. + +Two families live here: + +- The single-shot synthesis prompts (``*_SYNTHESIS_PROMPT``) — the legacy bypass + that consumes the shared trunk plus the current artifact and emits it wholesale. +- The two-step resource -> file prompts (``ROUTE_PROMPTS`` / ``SYNTHESIS_PROMPTS``), + keyed by track (``"memory"`` / ``"skill"``): step (a) routes a source to the set of + files to update/create; step (b) writes each target file's body. Both are track + parametric so the workspace workflow drives chat and skill through one code path. + +The literal placeholder tokens (``__DESCRIPTIONS__``, ``__EXISTING__``, ``__CONTENT__``, +``__NAME__``, ``__DESCRIPTION__``) are replaced (not ``str.format``) so source text +containing braces is safe. +""" + +from __future__ import annotations + +DESCRIPTIONS_PLACEHOLDER = "__DESCRIPTIONS__" +EXISTING_PLACEHOLDER = "__EXISTING__" +CONTENT_PLACEHOLDER = "__CONTENT__" +NAME_PLACEHOLDER = "__NAME__" +DESCRIPTION_PLACEHOLDER = "__DESCRIPTION__" + +MEMORY_SYNTHESIS_PROMPT = """You are maintaining an AI agent's long-term memory document about a user. + +Below is the CURRENT memory document, followed by NEW source descriptions — one per +source file the agent has seen. Produce the updated memory document. + +Requirements: +- Merge new facts, revise statements the new descriptions make outdated, and keep + existing content that is still valid. If the CURRENT document is empty, synthesize + a fresh document from the descriptions alone. +- Output the FULL Markdown document only. Do not wrap it in code fences. +- Use second-level headings (##) for sections such as Profile, Preferences, Goals, + and Key Events. Include a section only if there is real content for it. +- Be concise and factual. Do not invent details that are not supported by the + descriptions. Write in the same language as the descriptions. + +CURRENT memory document: +__EXISTING__ + +NEW source descriptions: +__DESCRIPTIONS__ +""" + +SKILL_OVERVIEW_SYNTHESIS_PROMPT = """You are maintaining the overview page of an AI agent's skill library. + +Below is the CURRENT overview, followed by the full set of SKILLS currently in the +library (name + body). Produce the updated overview document. + +Requirements: +- Write a concise overview of what the agent knows how to do: group related skills, + call out the most important ones, and orient a reader to the library. Do not paste + each skill's full body — the bodies live in their own files. +- Output the FULL Markdown document only. Do not wrap it in code fences. +- Use second-level headings (##) for groupings when there are enough skills to warrant + them. If there are no skills, output a short note that the library is empty. +- Be concise and factual. Write in the same language as the skills. + +CURRENT overview: +__EXISTING__ + +SKILLS in the library: +__DESCRIPTIONS__ +""" + +SKILL_FILE_SYNTHESIS_PROMPT = """You are maintaining an AI agent's skill library. + +Below are the EXISTING skills (name + body), followed by the CONTENT of a single source +the agent just processed. From the content, identify concrete, repeatable skills or tool +usage patterns (what worked, how to repeat it, what to avoid). Ignore one-off facts, +preferences, or trivia — those belong in the memory document, not here. + +Return ONLY a JSON array of skills to add or replace. Each element is an object: + {"name": "kebab-case-skill-name", "description": "one-line summary of the skill", "body": "Markdown body for this skill"} +- To revise an existing skill, reuse its exact name and return the full new body. +- To add a new skill, use a new name. +- Only include skills actually affected by this content; untouched existing skills are + kept automatically. +- If there are no genuine skills to add or change, return an empty array: [] + +EXISTING skills: +__EXISTING__ + +NEW source content: +__DESCRIPTIONS__ +""" + +# --- Two-step resource -> file prompts (ADR 0007 phase 1) --------------------- +# +# Step (a): route a single source to the set of files to update/create. The model +# sees the existing files (name + one-line description) and the source content, and +# returns a JSON plan. Step (b): given one target file (name + description + current +# body) and the source content, write the file's full body. + +_MEMORY_ROUTE_PROMPT = """You are maintaining an AI agent's long-term memory about a user, organized as a set +of memory files (each a themed document — e.g. Profile, Preferences, Goals, Work). + +Below are the EXISTING memory files (name + one-line description), followed by the +CONTENT of a single source the agent just processed. Decide which files this source +should update, and whether any new file should be created for facts that fit no +existing file. Capture durable facts, preferences, goals, and notable events; ignore +throwaway chatter. + +Return ONLY a JSON array of operations. Each element is an object: + {"op": "update", "name": ""} + {"op": "create", "name": "", "description": "one-line summary of the file"} +- Use "update" with a file's EXACT existing name to route the source there. +- Use "create" only when no existing file fits; give a reusable name and a description. +- Prefer updating an existing file over creating a near-duplicate. +- List a file at most once. If the source has nothing memory-worthy, return []. + +EXISTING memory files: +__EXISTING__ + +SOURCE content: +__CONTENT__ +""" + +_SKILL_ROUTE_PROMPT = """You are maintaining an AI agent's skill library — a set of skill files, each a +concrete, repeatable how-to (what worked, how to repeat it, what to avoid). + +Below are the EXISTING skills (name + one-line description), followed by the CONTENT of +a single source the agent just processed. Identify concrete, repeatable skills or tool +usage patterns in the content and decide which skills to update or create. Ignore +one-off facts, preferences, or trivia — those belong in memory, not here. + +Return ONLY a JSON array of operations. Each element is an object: + {"op": "update", "name": ""} + {"op": "create", "name": "kebab-case-skill-name", "description": "one-line summary of the skill"} +- Use "update" with a skill's EXACT existing name to revise it. +- Use "create" only when no existing skill fits; give a new kebab-case name and a description. +- Prefer updating an existing skill over creating a near-duplicate. +- List a skill at most once. If the source has no genuine skills, return []. + +EXISTING skills: +__EXISTING__ + +SOURCE content: +__CONTENT__ +""" + +_MEMORY_FILE_SYNTHESIS_PROMPT = """You are maintaining a single memory file about a user. + +FILE name: __NAME__ +FILE description: __DESCRIPTION__ + +Below is the CURRENT content of this file (empty if it is being created), followed by the +CONTENT of a new source. Produce the updated file. + +Requirements: +- Merge in facts from the source that belong in THIS file, revise statements the source + makes outdated, and keep existing content that is still valid. If the CURRENT content + is empty, synthesize a fresh document from the source alone. +- Only include material relevant to this file's topic; leave unrelated facts out. +- Output the FULL Markdown document only. Do not wrap it in code fences. +- Be concise and factual. Do not invent details not supported by the source. Write in the + same language as the source. + +CURRENT content: +__EXISTING__ + +NEW source content: +__CONTENT__ +""" + +_SKILL_FILE_SYNTHESIS_PROMPT = """You are maintaining a single skill file in an AI agent's skill library. + +SKILL name: __NAME__ +SKILL description: __DESCRIPTION__ + +Below is the CURRENT body of this skill (empty if it is being created), followed by the +CONTENT of a new source. Produce the updated skill body. + +Requirements: +- Capture the concrete, repeatable procedure this skill describes: what it accomplishes, + the steps to repeat it, and pitfalls to avoid. Merge in what the source adds and revise + anything it supersedes. If the CURRENT body is empty, write it fresh from the source. +- Output the FULL Markdown body only. Do not wrap it in code fences. +- Be concise and actionable. Do not invent steps not supported by the source. Write in the + same language as the source. + +CURRENT body: +__EXISTING__ + +NEW source content: +__CONTENT__ +""" + +# Track-keyed dispatch tables used by the workspace memorize workflow. +ROUTE_PROMPTS: dict[str, str] = { + "memory": _MEMORY_ROUTE_PROMPT, + "skill": _SKILL_ROUTE_PROMPT, +} +SYNTHESIS_PROMPTS: dict[str, str] = { + "memory": _MEMORY_FILE_SYNTHESIS_PROMPT, + "skill": _SKILL_FILE_SYNTHESIS_PROMPT, +} + +__all__ = [ + "CONTENT_PLACEHOLDER", + "DESCRIPTIONS_PLACEHOLDER", + "DESCRIPTION_PLACEHOLDER", + "EXISTING_PLACEHOLDER", + "MEMORY_SYNTHESIS_PROMPT", + "NAME_PLACEHOLDER", + "ROUTE_PROMPTS", + "SKILL_FILE_SYNTHESIS_PROMPT", + "SKILL_OVERVIEW_SYNTHESIS_PROMPT", + "SYNTHESIS_PROMPTS", +] diff --git a/src/memu/prompts/memory_type/__init__.py b/src/memu/prompts/memory_type/__init__.py new file mode 100644 index 0000000..ea584bd --- /dev/null +++ b/src/memu/prompts/memory_type/__init__.py @@ -0,0 +1,46 @@ +from memu.prompts.memory_type import behavior, event, knowledge, profile, skill, tool + +# DEFAULT_MEMORY_TYPES: list[str] = ["profile", "event", "knowledge", "behavior"] +DEFAULT_MEMORY_TYPES: list[str] = ["profile", "event"] + +PROMPTS: dict[str, str] = { + "profile": profile.PROMPT.strip(), + "event": event.PROMPT.strip(), + "knowledge": knowledge.PROMPT.strip(), + "behavior": behavior.PROMPT.strip(), + "skill": skill.PROMPT.strip(), + "tool": tool.PROMPT.strip(), +} + +CUSTOM_PROMPTS: dict[str, dict[str, str]] = { + "profile": profile.CUSTOM_PROMPT, + "event": event.CUSTOM_PROMPT, + "knowledge": knowledge.CUSTOM_PROMPT, + "behavior": behavior.CUSTOM_PROMPT, + "skill": skill.CUSTOM_PROMPT, + "tool": tool.CUSTOM_PROMPT, +} + +CUSTOM_TYPE_CUSTOM_PROMPTS: dict[str, str] = { + "category": profile.CUSTOM_PROMPT["category"], + "output": profile.CUSTOM_PROMPT["output"], + "input": profile.CUSTOM_PROMPT["input"], +} + +DEFAULT_MEMORY_CUSTOM_PROMPT_ORDINAL: dict[str, int] = { + "objective": 10, + "workflow": 20, + "rules": 30, + "category": 40, + "output": 50, + "examples": 60, + "input": 90, +} + +__all__ = [ + "CUSTOM_PROMPTS", + "CUSTOM_TYPE_CUSTOM_PROMPTS", + "DEFAULT_MEMORY_CUSTOM_PROMPT_ORDINAL", + "DEFAULT_MEMORY_TYPES", + "PROMPTS", +] diff --git a/src/memu/prompts/memory_type/behavior.py b/src/memu/prompts/memory_type/behavior.py new file mode 100644 index 0000000..b19d55f --- /dev/null +++ b/src/memu/prompts/memory_type/behavior.py @@ -0,0 +1,176 @@ +PROMPT_LEGACY = """ +Your task is to read and understand the resource content between the user and the assistant, and, based on the given memory categories, extract behavioral patterns, routines, and solutions about the user. + +## Original Resource: + +{resource} + + +## Memory Categories: +{categories_str} + +## Critical Requirements: +The core extraction target is behavioral memory items that record patterns, routines, and solutions characterizing how the user acts or behaves to solve specific problems. + +## Memory Item Requirements: +- Use the same language as the resource in . +- Extract patterns of behavior, routines, and solutions +- Focus on how the user typically acts, their preferences, and regular activities +- Each item can be either a single sentence concisely describing the pattern, routine, or solution, or a multi-line record with each line recording a specific step of the pattern, routine, or solution. +- Only extract meaningful behaviors, skip one-time actions unless significant +- Return empty array if no meaningful behaviors found + +## About Memory Categories: +- You can put identical or similar memory items into multiple memory categories. +- Do not create new memory categories. Please only generate in the given memory categories. +- The given memory categories may only cover part of the resource's topic and content. You don't need to summarize resource's content unrelated to the given memory categories. +- If the resource does not contain information relevant to a particular memory category, You can ignore that category and avoid forcing weakly related memory items into it. Simply skip that memory category and DO NOT output contents like "no relevant memory item". + +## Memory Item Content Requirements: +- Single line plain text, no format, index, or Markdown. +- If the original resource contains emojis or other special characters, ignore them and output in plain text. +- *ALWAYS* use the same language as the resource. + +# Response Format (JSON): +{{ + "memories_items": [ + {{ + "content": "the content of the memory item", + "categories": [list of memory categories that this memory item should belongs to, can be empty] + }} + ] +}} +""" + +PROMPT_BLOCK_OBJECTIVE = """ +# Task Objective +You are a professional User Memory Extractor. Your core task is to extract behavioral patterns, routines, and solutions that characterize how the user acts or behaves to solve specific problems. +""" + +PROMPT_BLOCK_WORKFLOW = """ +# Workflow +Read the full conversation to understand topics and meanings. +## Extract memories +Select turns that contain valuable Behavior Information and extract behavioral memory items. +## Review & validate +Merge semantically similar items. +Resolve contradictions by keeping the latest / most certain item. +## Final output +Output Behavior Information. +""" + +PROMPT_BLOCK_RULES = """ +# Rules +## General requirements (must satisfy all) +- Use "user" to refer to the user consistently. +- Each memory item must be complete and self-contained, written as a declarative descriptive sentence. +- Each memory item must express one single complete piece of information and be understandable without context. +- Similar/redundant items must be merged into one, and assigned to only one category. +- Each memory item must be < 50 words worth of length (keep it concise but include relevant details). +- Focus on patterns of behavior, routines, and solutions. +- Focus on how the user typically acts, their preferences, and regular activities. +- Can include multi-line records with each line describing a specific step of the pattern, routine, or solution. +Important: Extract only behaviors directly stated or confirmed by the user. No guesses, no suggestions, and no content introduced only by the assistant. +Important: Accurately reflect whether the subject is the user or someone around the user. + +## Special rules for Behavior Information +- One-time actions or specific events are forbidden in Behavior Information unless they demonstrate a significant pattern. +- Focus on recurring patterns, typical approaches, and established routines. +- Do not extract content that was obtained only through the model's follow-up questions unless the user shows strong proactive intent. + +## Forbidden content +- Knowledge Q&A without a clear user behavior pattern. +- One-time events that do not reflect recurring behavior. +- Turns where the user did not respond and only the assistant spoke. +- Illegal / harmful sensitive topics (violence, politics, drugs, etc.). +- Private financial accounts, IDs, addresses, military/defense/government job details, precise street addresses—unless explicitly requested by the user (still avoid if not necessary). +- Any content mentioned only by the assistant and not explicitly confirmed by the user. + +## Review & validation rules +- Merge similar items: keep only one and assign a single category. +- Resolve conflicts: keep the latest / most certain item. +- Final check: every item must comply with all extraction rules. +""" + +PROMPT_BLOCK_CATEGORY = """ +## Memory Categories: +{categories_str} +""" + +PROMPT_BLOCK_OUTPUT = """ +# Output Format (XML) +Return all memories wrapped in a single element: + + + Behavior memory item content 1 + + Category Name + + + + Behavior memory item content 2 + + Category Name + + + +""" + +PROMPT_BLOCK_EXAMPLES = """ +# Examples (Input / Output / Explanation) +Example 1: Behavior Information Extraction +## Input +user: Hi, are you busy? I just got off work and I'm going to the supermarket to buy some groceries. +assistant: Not busy. Are you cooking for yourself? +user: Yes. It's healthier. I work as a product manager in an internet company. I'm 30 this year. After work I like experimenting with cooking, I often figure out dishes by myself. +assistant: Being a PM is tough. You're so disciplined to cook at 30! +user: It's fine. Cooking relaxes me. It's better than takeout. Also I'm traveling next weekend. +assistant: You can check the weather ahead. Your sunscreen can finally be used. +user: I haven't started packing yet. It's annoying. +## Output + + + The user typically cooks for themselves after work instead of ordering takeout + + Daily Routine + + + + The user often experiments with cooking and figures out dishes by themselves + + Daily Routine + + + +## Explanation +Only behavioral patterns explicitly stated by the user are extracted. +Cooking after work and experimenting with dishes are recurring behaviors/routines. +User's job, age are stable traits (not behaviors). The travel plan is a one-time event, not a behavioral pattern. +""" + +PROMPT_BLOCK_INPUT = """ +# Original Resource: + +{resource} + +""" + +PROMPT = "\n\n".join([ + PROMPT_BLOCK_OBJECTIVE.strip(), + PROMPT_BLOCK_WORKFLOW.strip(), + PROMPT_BLOCK_RULES.strip(), + PROMPT_BLOCK_CATEGORY.strip(), + PROMPT_BLOCK_OUTPUT.strip(), + PROMPT_BLOCK_EXAMPLES.strip(), + PROMPT_BLOCK_INPUT.strip(), +]) + +CUSTOM_PROMPT = { + "objective": PROMPT_BLOCK_OBJECTIVE.strip(), + "workflow": PROMPT_BLOCK_WORKFLOW.strip(), + "rules": PROMPT_BLOCK_RULES.strip(), + "category": PROMPT_BLOCK_CATEGORY.strip(), + "output": PROMPT_BLOCK_OUTPUT.strip(), + "examples": PROMPT_BLOCK_EXAMPLES.strip(), + "input": PROMPT_BLOCK_INPUT.strip(), +} diff --git a/src/memu/prompts/memory_type/event.py b/src/memu/prompts/memory_type/event.py new file mode 100644 index 0000000..5b7caeb --- /dev/null +++ b/src/memu/prompts/memory_type/event.py @@ -0,0 +1,182 @@ +PROMPT_LEGACY = """ +Your task is to read and understand the resource content between the user and the assistant, and, based on the given memory categories, extract specific events and experiences that happened to or involved the user. + +## Original Resource: + +{resource} + + +## Memory Categories: +{categories_str} + +## Critical Requirements: +The core extraction target is eventful memory items about specific events, experiences, and occurrences that happened at a particular time and involve the user. + +## Memory Item Requirements: +- Use the same language as the resource in . +- Each memory item should be complete and standalone. +- Each memory item should express a complete piece of information, and is understandable without context and reading other memory items. +- Always use declarative and descriptive sentences. +- Use "the user" (or that in the target language, e.g., "用户") to refer to the user. +- Focus on specific events that happened at a particular time or period. +- Extract concrete happenings, activities, and experiences. +- Include relevant details such as time, location, and participants where available. +- Carefully judge whether an event is narrated by the user or the assistant. You should only extract memory items for events directly narrated or confirmed by the user. +- DO NOT include behavioral patterns, habits, or factual knowledge. +- DO NOT record temporary, ephemeral situations or trivial daily activities unless significant. + +## Example (good): +- The user and his family went on a hike at a nature park outside the city last weekend. They had a picnic there, and had a great time. + +## Example (bad): +- The user went on a hike. (The time, place, and people are missing.) +- They had a great time. (The reference to "they" is unclear and does not constitute a self-contained memory item.) + +## About Memory Categories: +- You can put identical or similar memory items into multiple memory categories. +- Do not create new memory categories. Please only generate in the given memory categories. +- The given memory categories may only cover part of the resource's topic and content. You don't need to summarize resource's content unrelated to the given memory categories. +- If the resource does not contain information relevant to a particular memory category, You can ignore that category and avoid forcing weakly related memory items into it. Simply skip that memory category and DO NOT output contents like "no relevant memory item". + +## Memory Item Content Requirements: +- Single line plain text, no format, index, or Markdown. +- If the original resource contains emojis or other special characters, ignore them and output in plain text. +- *ALWAYS* use the same language as the resource. + +# Response Format (JSON): +{{ + "memories_items": [ + {{ + "content": "the content of the memory item", + "categories": [list of memory categories that this memory item should belongs to, can be empty] + }} + ] +}} +""" + +PROMPT_BLOCK_OBJECTIVE = """ +# Task Objective +You are a professional User Memory Extractor. Your core task is to extract specific events and experiences that happened to or involved the user (e.g., activities, occurrences, experiences at particular times). +""" + +PROMPT_BLOCK_WORKFLOW = """ +# Workflow +Read the full conversation to understand topics and meanings. +## Extract memories +Select turns that contain valuable Event Information and extract event memory items. +## Review & validate +Merge semantically similar items. +Resolve contradictions by keeping the latest / most certain item. +## Final output +Output Event Information. +""" + +PROMPT_BLOCK_RULES = """ +# Rules +## General requirements (must satisfy all) +- Use "user" to refer to the user consistently. +- Each memory item must be complete and self-contained, written as a declarative descriptive sentence. +- Each memory item must express one single complete piece of information and be understandable without context. +- Similar/redundant items must be merged into one, and assigned to only one category. +- Each memory item must be < 50 words worth of length (keep it concise but include relevant details). +- Focus on specific events that happened at a particular time or period. +- Include relevant details such as time, location, and participants where available. +Important: Extract only events directly stated or confirmed by the user. No guesses, no suggestions, and no content introduced only by the assistant. +Important: Accurately reflect whether the subject is the user or someone around the user. + +## Special rules for Event Information +- Behavioral patterns, habits, preferences, or factual knowledge are forbidden in Event Information. +- Focus on concrete happenings, activities, and experiences. +- Do not extract content that was obtained only through the model's follow-up questions unless the user shows strong proactive intent. + +## Forbidden content +- Knowledge Q&A without a clear user event. +- Trivial daily activities unless significant (e.g., routine meals, commuting). +- Temporary, ephemeral situations that lack meaningful significance. +- Turns where the user did not respond and only the assistant spoke. +- Illegal / harmful sensitive topics (violence, politics, drugs, etc.). +- Private financial accounts, IDs, addresses, military/defense/government job details, precise street addresses—unless explicitly requested by the user (still avoid if not necessary). +- Any content mentioned only by the assistant and not explicitly confirmed by the user. + +## Review & validation rules +- Merge similar items: keep only one and assign a single category. +- Resolve conflicts: keep the latest / most certain item. +- Final check: every item must comply with all extraction rules. +""" + +PROMPT_BLOCK_CATEGORY = """ +## Memory Categories: +{categories_str} +""" + +PROMPT_BLOCK_OUTPUT = """ +# Output Format (XML) +Return all memories wrapped in a single element: + + + Event memory item content 1 + + Category Name + + + + Event memory item content 2 + + Category Name + + + +""" + +PROMPT_BLOCK_EXAMPLES = """ +# Examples (Input / Output / Explanation) +Example 1: Event Information Extraction +## Input +user: Hi, are you busy? I just got off work and I'm going to the supermarket to buy some groceries. +assistant: Not busy. Are you cooking for yourself? +user: Yes. It's healthier. I work as a product manager in an internet company. I'm 30 this year. After work I like experimenting with cooking, I often figure out dishes by myself. +assistant: Being a PM is tough. You're so disciplined to cook at 30! +user: It's fine. Cooking relaxes me. It's better than takeout. Also I'm traveling next weekend. +assistant: You can check the weather ahead. Your sunscreen can finally be used. +user: I haven't started packing yet. It's annoying. +## Output + + + The user is planning a trip next weekend and hasn't started packing yet + + Travel + + + +## Explanation +Only specific events explicitly stated by the user are extracted. +The travel plan is an event with a specific time reference (next weekend). +User's job, age, and cooking habits are stable user traits, so they are not extracted as Event Information. +""" + +PROMPT_BLOCK_INPUT = """ +# Original Resource: + +{resource} + +""" + +PROMPT = "\n\n".join([ + PROMPT_BLOCK_OBJECTIVE.strip(), + PROMPT_BLOCK_WORKFLOW.strip(), + PROMPT_BLOCK_RULES.strip(), + PROMPT_BLOCK_CATEGORY.strip(), + PROMPT_BLOCK_OUTPUT.strip(), + PROMPT_BLOCK_EXAMPLES.strip(), + PROMPT_BLOCK_INPUT.strip(), +]) + +CUSTOM_PROMPT = { + "objective": PROMPT_BLOCK_OBJECTIVE.strip(), + "workflow": PROMPT_BLOCK_WORKFLOW.strip(), + "rules": PROMPT_BLOCK_RULES.strip(), + "category": PROMPT_BLOCK_CATEGORY.strip(), + "output": PROMPT_BLOCK_OUTPUT.strip(), + "examples": PROMPT_BLOCK_EXAMPLES.strip(), + "input": PROMPT_BLOCK_INPUT.strip(), +} diff --git a/src/memu/prompts/memory_type/knowledge.py b/src/memu/prompts/memory_type/knowledge.py new file mode 100644 index 0000000..eb5fb8b --- /dev/null +++ b/src/memu/prompts/memory_type/knowledge.py @@ -0,0 +1,172 @@ +PROMPT_LEGACY = """ +Your task is to read and understand the resource content between the user and the assistant, and, based on the given memory categories, extract knowledge and information that the user learned or discussed. + +## Original Resource: + +{resource} + + +## Memory Categories: +{categories_str} + +## Critical Requirements: +The core extraction target is factual memory items that reflect knowledge, concepts, definitions, and factual information that the resource content suggests. + +## Memory Item Requirements: +- Use the same language as the resource in . +- Each memory item should be complete and standalone. +- Each memory item should express a complete piece of information, and is understandable without context and reading other memory items. +- Extract factual knowledge, concepts, definitions, and explanations +- Focus on objective information that can be learned or referenced +- Each item should be a descriptive sentence. +- Only extract meaningful knowledge, skip opinions or personal experiences +- Return empty array if no meaningful knowledge found + +## About Memory Categories: +- You can put identical or similar memory items into multiple memory categories. +- Do not create new memory categories. Please only generate in the given memory categories. +- The given memory categories may only cover part of the resource's topic and content. You don't need to summarize resource's content unrelated to the given memory categories. +- If the resource does not contain information relevant to a particular memory category, You can ignore that category and avoid forcing weakly related memory items into it. Simply skip that memory category and DO NOT output contents like "no relevant memory item". + +## Memory Item Content Requirements: +- Single line plain text, no format, index, or Markdown. +- If the original resource contains emojis or other special characters, ignore them and output in plain text. +- *ALWAYS* use the same language as the resource. + +# Response Format (JSON): +{{ + "memories_items": [ + {{ + "content": "the content of the memory item", + "categories": [list of memory categories that this memory item should belongs to, can be empty] + }} + ] +}} +""" + +PROMPT_BLOCK_OBJECTIVE = """ +# Task Objective +You are a professional User Memory Extractor. Your core task is to extract factual knowledge, concepts, definitions, and information that the user learned or discussed in the conversation. +""" + +PROMPT_BLOCK_WORKFLOW = """ +# Workflow +Read the full conversation to understand topics and meanings. +## Extract memories +Select turns that contain valuable Knowledge Information and extract knowledge memory items. +## Review & validate +Merge semantically similar items. +Resolve contradictions by keeping the latest / most certain item. +## Final output +Output Knowledge Information. +""" + +PROMPT_BLOCK_RULES = """ +# Rules +## General requirements (must satisfy all) +- Each memory item must be complete and self-contained, written as a declarative descriptive sentence. +- Each memory item must express one single complete piece of information and be understandable without context. +- Similar/redundant items must be merged into one, and assigned to only one category. +- Each memory item must be < 50 words worth of length (keep it concise but include relevant details). +- Focus on factual knowledge, concepts, definitions, and explanations. +- Focus on objective information that can be learned or referenced. +Important: Extract only knowledge directly stated or discussed in the conversation. No guesses or unsupported extensions. + +## Special rules for Knowledge Information +- Personal opinions, subjective preferences, or personal experiences are forbidden in Knowledge Information. +- Focus on objective facts, concepts, and explanations. +- User-specific traits, events, or behaviors are not knowledge items. + +## Forbidden content +- Opinions or subjective statements without factual basis. +- Personal experiences or events (these belong to event type). +- User preferences or behavioral patterns (these belong to profile/behavior type). +- Trivial or commonly known facts that add no value. +- Illegal / harmful sensitive topics (violence, politics, drugs, etc.). +- Any content that is speculative or not clearly established in the conversation. + +## Review & validation rules +- Merge similar items: keep only one and assign a single category. +- Resolve conflicts: keep the latest / most certain item. +- Final check: every item must comply with all extraction rules. +""" + +PROMPT_BLOCK_CATEGORY = """ +## Memory Categories: +{categories_str} +""" + +PROMPT_BLOCK_OUTPUT = """ +# Output Format (XML) +Return all memories wrapped in a single element: + + + Knowledge memory item content 1 + + Category Name + + + + Knowledge memory item content 2 + + Category Name + + + +""" + +PROMPT_BLOCK_EXAMPLES = """ +# Examples (Input / Output / Explanation) +Example 1: Knowledge Information Extraction +## Input +user: I'm trying to understand how Python decorators work. Can you explain? +assistant: A decorator is a function that takes another function and extends its behavior without modifying it. It's a form of metaprogramming. +user: Oh I see, so it's like wrapping a function. I heard that the @ symbol is syntactic sugar for applying decorators. +assistant: Exactly! When you write @decorator above a function, it's equivalent to function = decorator(function). +user: That makes sense. By the way, I'm working on a project at my company using this. +## Output + + + In Python, a decorator is a function that takes another function and extends its behavior without modifying it + + Programming + + + + The @ symbol in Python is syntactic sugar for applying decorators, equivalent to function = decorator(function) + + Programming + + + +## Explanation +Only factual knowledge discussed and confirmed in the conversation is extracted. +The user's work project is a personal event/situation, not knowledge, so it is not extracted. +""" + +PROMPT_BLOCK_INPUT = """ +# Original Resource: + +{resource} + +""" + +PROMPT = "\n\n".join([ + PROMPT_BLOCK_OBJECTIVE.strip(), + PROMPT_BLOCK_WORKFLOW.strip(), + PROMPT_BLOCK_RULES.strip(), + PROMPT_BLOCK_CATEGORY.strip(), + PROMPT_BLOCK_OUTPUT.strip(), + PROMPT_BLOCK_EXAMPLES.strip(), + PROMPT_BLOCK_INPUT.strip(), +]) + +CUSTOM_PROMPT = { + "objective": PROMPT_BLOCK_OBJECTIVE.strip(), + "workflow": PROMPT_BLOCK_WORKFLOW.strip(), + "rules": PROMPT_BLOCK_RULES.strip(), + "category": PROMPT_BLOCK_CATEGORY.strip(), + "output": PROMPT_BLOCK_OUTPUT.strip(), + "examples": PROMPT_BLOCK_EXAMPLES.strip(), + "input": PROMPT_BLOCK_INPUT.strip(), +} diff --git a/src/memu/prompts/memory_type/profile.py b/src/memu/prompts/memory_type/profile.py new file mode 100644 index 0000000..82873c2 --- /dev/null +++ b/src/memu/prompts/memory_type/profile.py @@ -0,0 +1,190 @@ +PROMPT_LEGACY = """ +Your task is to read and understand the resource content between the user and the assistant, and, based on the given memory categories, extract memory items about the user. + +## Original Resource: + +{resource} + + +## Memory Categories: +{categories_str} + +## Critical Requirements: +The core extraction target is self-contained memory items about the user. + +## Memory Item Requirements: +- Use the same language as the resource in . +- Each memory item should be complete and standalone. +- Each memory item should express a complete piece of information, and is understandable without context and reading other memory items. +- Always use declarative and descriptive sentences. +- Use "the user" (or that in the target language, e.g., "用户") to refer to the user. +- You can cluster multiple events that are closely related or under a single topic into a single memory item, but avoid making each single memory item too long (over 100 words). +- **Important** Carefully judge whether an event/fact/information is narrated by the user or the assistant. You should only extract memory items for the event/fact/information directly narrated or confirmed by the user. DO NOT include any groundless conjectures, advice, suggestions, or any content provided by the assistant. +- **Important** Carefully judge whether the subject of an event/fact/information is the user themselves or some person around the user (e.g., the user's family, friend, or the assistant), and reflect the subject correctly in the memory items. +- **Important** DO NOT record temporary, ephemeral, or one-time situational information such as weather conditions (e.g., "today is raining"), current mood states, temporary technical issues, or any short-lived circumstances that are unlikely to be relevant for the user profile. Focus on meaningful, persistent information about the user's characteristics, preferences, relationships, ongoing situations, and significant events. + +## Example (good): +- The user and his family went on a hike at a nature park outside the city last weekend. They had a picnic there, and had a great time. + +## Example (bad): +- The user went on a hike. (The time, place, and people are missing.) +- They had a great time. (The reference to "they" is unclear and does not constitute a self-contained memory item.) +- The user and his family went on a hike at a nature park outside the city last weekend. The user and his family had a picnic at a nature park outside the city last weekend. (Should be merged.) + +## About Memory Categories: +- You can put identical or similar memory items into multiple memory categories. For example, "The user and his family went on a hike at a nature park outside the city last weekend." can be put into all of "hiking", "weekend activities", and "family activities" categories (if they exist). Nevertheless, Memory items put to each category can have different focuses. +- Do not create new memory categories. Please only generate in the given memory categories. +- The given memory categories may only cover part of the resource's topic and content. You don't need to summarize resource's content unrelated to the given memory categories. +- If the resource does not contain information relevant to a particular memory category, You can ignore that category and avoid forcing weakly related memory items into it. Simply skip that memory category and DO NOT output contents like "no relevant memory item". + +## Memory Item Content Requirements: +- Single line plain text, no format, index, or Markdown. +- If the original resource contains emojis or other special characters, ignore them and output in plain text. +- *ALWAYS* use the same language as the resource. + +# Response Format (JSON): +{{ + "memories_items": [ + {{ + "content": "the content of the memory item", + "categories": [list of memory categories that this memory item should belongs to, can be empty] + }} + ] +}} +""" + +PROMPT_BLOCK_OBJECTIVE = """ +# Task Objective +You are a professional User Memory Extractor. Your core task is to extract independent user memory items about the user (e.g., basic info, preferences, habits, other long-term stable traits). +""" + +PROMPT_BLOCK_WORKFLOW = """ +# Workflow +Read the full conversation to understand topics and meanings. +## Extract memories +Select turns that contain valuable User Information and extract user info memory items. +## Review & validate +Merge semantically similar items. +Resolve contradictions by keeping the latest / most certain item. +## Final output +Output User Information. +""" + +PROMPT_BLOCK_RULES = """ +# Rules +## General requirements (must satisfy all) +- Use "user" to refer to the user consistently. +- Each memory item must be complete and self-contained, written as a declarative descriptive sentence. +- Each memory item must express one single complete piece of information and be understandable without context. +- Similar/redundant items must be merged into one, and assigned to only one category. +- Each memory item must be < 30 words worth of length (keep it as concise as possible). +- A single memory item must NOT contain timestamps. +Important: Extract only facts directly stated or confirmed by the user. No guesses, no suggestions, and no content introduced only by the assistant. +Important: Accurately reflect whether the subject is the user or someone around the user. +Important: Do not record temporary/one-off situational information; focus on meaningful, persistent information. + +## Special rules for User Information +- Any event-related item is forbidden in User Information. +- Do not extract content that was obtained only through the model's follow-up questions unless the user shows strong proactive intent. + +## Forbidden content +- Knowledge Q&A without a clear user fact. +- Trivial updates that do not add meaningful value (e.g., “full → too full”). +- Turns where the user did not respond and only the assistant spoke. +- Illegal / harmful sensitive topics (violence, politics, drugs, etc.). +- Private financial accounts, IDs, addresses, military/defense/government job details, precise street addresses—unless explicitly requested by the user (still avoid if not necessary). +- Any content mentioned only by the assistant and not explicitly confirmed by the user. + +## Review & validation rules +- Merge similar items: keep only one and assign a single category. +- Resolve conflicts: keep the latest / most certain item. +- Final check: every item must comply with all extraction rules. +""" + +PROMPT_BLOCK_CATEGORY = """ +## Memory Categories: +{categories_str} +""" + +PROMPT_BLOCK_OUTPUT = """ +# Output Format (XML) +Return all memories wrapped in a single element: + + + User memory item content 1 + + Category Name + + + + User memory item content 2 + + Category Name + + + +""" + +PROMPT_BLOCK_EXAMPLES = """ +# Examples (Input / Output / Explanation) +Example 1: User Information Extraction +## Input +user: Hi, are you busy? I just got off work and I'm going to the supermarket to buy some groceries. +assistant: Not busy. Are you cooking for yourself? +user: Yes. It's healthier. I work as a product manager in an internet company. I'm 30 this year. After work I like experimenting with cooking, I often figure out dishes by myself. +assistant: Being a PM is tough. You're so disciplined to cook at 30! +user: It's fine. Cooking relaxes me. It's better than takeout. Also I'm traveling next weekend. +assistant: You can check the weather ahead. Your sunscreen can finally be used. +user: I haven't started packing yet. It's annoying. +## Output + + + The user works as a product manager at an internet company + + Basic Information + + + + The user is 30 years old + + Basic Information + + + + The user likes experimenting with cooking after work + + Basic Information + + + +## Explanation +Only stable user facts explicitly stated by the user are extracted. +The travel plan and packing annoyance are events/temporary states, so they are not extracted as User Information. +""" + +PROMPT_BLOCK_INPUT = """ +# Original Resource: + +{resource} + +""" + +PROMPT = "\n\n".join([ + PROMPT_BLOCK_OBJECTIVE.strip(), + PROMPT_BLOCK_WORKFLOW.strip(), + PROMPT_BLOCK_RULES.strip(), + PROMPT_BLOCK_CATEGORY.strip(), + PROMPT_BLOCK_OUTPUT.strip(), + PROMPT_BLOCK_EXAMPLES.strip(), + PROMPT_BLOCK_INPUT.strip(), +]) + +CUSTOM_PROMPT = { + "objective": PROMPT_BLOCK_OBJECTIVE.strip(), + "workflow": PROMPT_BLOCK_WORKFLOW.strip(), + "rules": PROMPT_BLOCK_RULES.strip(), + "category": PROMPT_BLOCK_CATEGORY.strip(), + "output": PROMPT_BLOCK_OUTPUT.strip(), + "examples": PROMPT_BLOCK_EXAMPLES.strip(), + "input": PROMPT_BLOCK_INPUT.strip(), +} diff --git a/src/memu/prompts/memory_type/skill.py b/src/memu/prompts/memory_type/skill.py new file mode 100644 index 0000000..450c6bd --- /dev/null +++ b/src/memu/prompts/memory_type/skill.py @@ -0,0 +1,580 @@ +PROMPT_LEGACY = """ +Your task is to read and understand the resource content (agent logs, workflow documentation, execution traces, or technical documents), and extract skills, capabilities, and technical competencies demonstrated or described in the content. Format each skill as a comprehensive, production-ready skill profile that can be referenced and applied. + +## Original Resource: + +{resource} + + +## Memory Categories: +{categories_str} + +## Critical Requirements: +Extract skill-based memory items as comprehensive skill profiles that include: +1. **Skill Name**: Clear, memorable name for the skill +2. **Description**: What the skill enables and when to use it +3. **Context**: Situations where this skill was demonstrated +4. **Core Principles**: Fundamental guidelines and best practices +5. **Implementation Details**: Specific techniques, tools, and approaches +6. **Success Patterns**: What works well +7. **Common Pitfalls**: What to avoid + +The core extraction target is actionable skill profiles that capture not just WHAT was done, but HOW and WHY it works. + +## Skill Profile Structure: + +For each extracted skill, create a comprehensive profile following this template: + +``` +--- +name: skill-name-in-kebab-case +description: One-line description of what this skill enables and when to use it +category: primary-category +demonstrated-in: [list of contexts where this was shown] +--- + +[Brief introduction explaining the skill and its importance] + +## Core Principles + +[Key concepts and fundamental approaches that make this skill effective] + +## When to Use This Skill + +- Situation 1: [specific context] +- Situation 2: [specific context] +- [More situations as applicable] + +## Implementation Guide + +### Prerequisites +- [Required knowledge or setup] + +### Techniques and Approaches +[Detailed explanation of how to apply this skill, including:] +- Specific methods used +- Tools and technologies involved +- Step-by-step process when applicable +- Metrics to track (error rates, response times, etc.) + +### Example from Resource +[Concrete example from the source material showing this skill in action, including outcomes and metrics] + +## Success Patterns + +What works well when applying this skill: +- [Pattern 1 with explanation] +- [Pattern 2 with explanation] +- [More patterns] + +## Common Pitfalls + +What to avoid: +- **[Pitfall 1]**: [Why it's a problem and how to avoid it] +- **[Pitfall 2]**: [Why it's a problem and how to avoid it] +- [More pitfalls based on failures or lessons learned] + +## Key Takeaways + +- [Critical insight 1] +- [Critical insight 2] +- [Critical insight 3] +``` + +## Example Skill Profiles: + +### Example 1: Canary Deployment + +``` +--- +name: canary-deployment-with-monitoring +description: Implement gradual traffic shifting deployment strategy with real-time monitoring and automatic rollback capabilities +category: deployment +demonstrated-in: [Payment Service v2.3.1 deployment] +--- + +Canary deployment is a risk-mitigation strategy that gradually shifts production traffic from an old version to a new version while continuously monitoring key metrics. This approach enables early detection of issues with minimal user impact. + +## Core Principles + +- **Gradual exposure**: Start with a small percentage of traffic (typically 5-10%) to limit blast radius +- **Continuous monitoring**: Track error rates, response times, and business metrics in real-time +- **Automated decision-making**: Use predefined thresholds to trigger automatic rollbacks +- **Quick recovery**: Maintain ability to instantly route traffic back to stable version + +## When to Use This Skill + +- Deploying critical services where downtime is unacceptable +- Rolling out changes with uncertain production behavior +- High-traffic services where A/B testing production performance is valuable +- Services with complex dependencies where integration issues may emerge gradually + +## Implementation Guide + +### Prerequisites +- Load balancer with traffic splitting capabilities +- Monitoring system with real-time metrics (Prometheus, Grafana) +- Automated deployment pipeline (Jenkins, GitLab CI) +- Health check endpoints on both versions + +### Techniques and Approaches + +1. **Initial Deployment** (10% traffic): + - Deploy new version alongside existing version + - Configure load balancer to route 10% of traffic to new version + - Monitor for 5-10 minutes + +2. **Monitoring Checkpoints**: + - Error rate comparison: New version should not exceed baseline by >2% + - Response time (p95): Should remain within 20% of baseline + - Business metrics: Transaction success rate, API call patterns + +3. **Gradual Rollout**: + - If metrics stable: 10% → 25% → 50% → 75% → 100% + - Pause 5-10 minutes between each increment + - Automated progression based on metric thresholds + +4. **Rollback Triggers**: + - Error rate >5%: Immediate rollback + - Response time degradation >50%: Investigation required + - Health check failures: Automatic rollback + +### Example from Resource + +Payment Service v2.3.1 deployment achieved: +- Zero downtime during 12-minute deployment +- Traffic progression: 10% → 50% → 100% with 2-minute pauses +- Response time improved 15% (320ms → 270ms p95) +- Error rate remained stable at 0.1% throughout +- New fraud detection algorithm safely rolled out to all users + +## Success Patterns + +What works well: +- **Small initial percentage**: 5-10% catches most issues while limiting impact +- **Metric-driven automation**: Removes human error from rollback decisions +- **Business metric monitoring**: Technical metrics alone miss some issues +- **Communication**: Notify stakeholders about canary status + +## Common Pitfalls + +What to avoid: +- **Too aggressive progression**: Rushing from 10% to 100% defeats the purpose +- **Insufficient monitoring window**: Need 5+ minutes at each stage to detect issues +- **Ignoring business metrics**: Technical health doesn't guarantee business success +- **Manual rollback only**: Human reaction time too slow for critical failures + +## Key Takeaways + +- Canary deployments trade deployment speed for safety +- Automation is critical for consistent, reliable rollbacks +- Start small (5-10%), progress gradually, monitor continuously +- Combine technical and business metrics for complete picture +``` + +### Example 2: Incident Response + +``` +--- +name: rapid-incident-response +description: Quickly detect, diagnose, and resolve production incidents using automated monitoring and systematic troubleshooting +category: incident-response +demonstrated-in: [User Service v3.1.0 rollback] +--- + +Rapid incident response is the ability to quickly identify production problems, understand their root cause, and implement fixes or rollbacks to restore service. Speed and systematic approach are critical to minimizing customer impact. + +## Core Principles + +- **Fast detection**: Automated monitoring catches issues within minutes +- **Immediate action**: Rollback first, investigate later when customer impact is high +- **Systematic diagnosis**: Follow structured troubleshooting process +- **Learning culture**: Every incident is an opportunity to improve + +## When to Use This Skill + +- Production errors detected by monitoring alerts +- User-reported issues indicating service degradation +- Automated health checks failing +- Performance metrics exceeding thresholds + +## Implementation Guide + +### Prerequisites +- Comprehensive monitoring (logs, metrics, traces) +- Automated rollback capabilities +- On-call rotation and escalation procedures +- Incident management tools and runbooks + +### Techniques and Approaches + +1. **Detection Phase** (0-3 minutes): + - Automated alerts trigger from monitoring thresholds + - Error rate, response time, or business metric anomalies + - Health check failures or pod restart loops + +2. **Initial Response** (3-5 minutes): + - Assess severity: Customer-facing? Data loss risk? + - Decision: Rollback immediately or investigate first? + - High severity → Immediate rollback + - Low severity → Investigate with time limit + +3. **Rollback Execution** (2-4 minutes): + - Automated: Trigger rollback through deployment pipeline + - Manual: Revert Helm release or switch traffic to previous version + - Verify: Confirm metrics return to baseline + +4. **Root Cause Analysis** (Post-incident): + - Review logs, metrics, and deployment changes + - Identify configuration drift, missing variables, performance issues + - Document findings and create action items + +### Example from Resource + +User Service v3.1.0 incident: +- Detection: Error rate spiked 0.2% → 5.1% within 30 seconds +- Response: Automatic rollback triggered at threshold in 2 minutes +- Recovery: Service restored to v3.0.9, error rate normalized in 4 minutes total +- Root cause: Missing AUTH_REFRESH_SECRET environment variable in production +- No customer impact due to fast automated rollback + +## Success Patterns + +What works well: +- **Automated thresholds**: Remove human decision-making delay +- **Clear severity criteria**: Know when to rollback vs investigate +- **Runbooks**: Pre-documented procedures for common issues +- **Blameless post-mortems**: Focus on systemic improvements, not individual errors + +## Common Pitfalls + +What to avoid: +- **Investigation paralysis**: Spending too long diagnosing while customers suffer +- **Manual-only rollback**: Automation is 5-10x faster +- **Configuration drift**: Staging and production environment inconsistency +- **Skipping post-mortems**: Missing opportunity to prevent recurrence + +## Key Takeaways + +- Speed matters: Every minute of downtime impacts customers and business +- Automate rollback decisions based on objective metrics +- Rollback first, investigate second for high-severity incidents +- Use incidents to improve systems, not blame people +``` + +## What NOT to Extract as Skills: + +❌ **Generic statements**: "Used Docker", "Good at programming" +❌ **Opinions**: "I think microservices are better" +❌ **Theory without practice**: "Kubernetes is an orchestrator" (that's knowledge) +❌ **One-time luck**: "Fixed a bug" without approach +❌ **Trivial actions**: "Using email", "Reading docs" + +✅ **DO Extract**: Concrete approaches with context, tools, metrics, and outcomes + +## About Memory Categories: +- You can put identical or similar skill items into multiple memory categories. +- Do not create new memory categories. Please only generate in the given memory categories. +- Focus on categories like: technical_skills, work_life, knowledge, experiences + +## Memory Item Content Requirements: +- *ALWAYS* use the same language as the resource in . +- Format as structured markdown with frontmatter (---, name, description, category, demonstrated-in, ---) +- Include all sections: Core Principles, When to Use, Implementation Guide, Success Patterns, Common Pitfalls, Key Takeaways +- Be specific and concrete - include technology names, version numbers, metrics, and outcomes +- Each skill should be comprehensive enough to be referenced and applied independently +- Minimum 300 words per skill to ensure depth and actionability +- If the original resource contains emojis or other special characters, ignore them and output in plain text. + +## Special Instructions for Different Resource Types: + +### For Deployment Logs: +- Extract each significant deployment (success or failure) as a separate skill +- Success: Focus on techniques that worked (canary, blue-green, performance optimization) +- Failure: Focus on incident response, root cause analysis, recovery procedures +- Include metrics: deployment time, error rates, response times, recovery time + +### For Workflow Documentation: +- Extract major workflow stages as skills (CI/CD pipeline, testing strategy, monitoring setup) +- Include tool chains and technology stacks +- Document step-by-step procedures +- Note success metrics and KPIs + +### For Agent Execution Logs: +- Extract problem-solving approaches as skills (competitive analysis, data processing, decision-making) +- Include tool orchestration patterns +- Document reasoning steps and validation approaches +- Capture multi-step workflows + +# Response Format (JSON): +{{ + "memories_items": [ + {{ + "content": "MUST be a complete markdown skill profile starting with --- frontmatter, then sections. Format: +--- +name: skill-name +description: one line description +category: category-name +demonstrated-in: [context] +--- + +[Introduction paragraph] + +## Core Principles +[bullet points] + +## When to Use This Skill +[bullet points] + +## Implementation Guide +### Prerequisites +### Techniques and Approaches +### Example from Resource + +## Success Patterns +[bullet points] + +## Common Pitfalls +[bullet points] + +## Key Takeaways +[bullet points] + +Minimum 300 words total.", + "categories": [list of memory categories] + }} + ] +}} + +CRITICAL: The content field MUST contain the complete markdown text with ALL sections, not a summary paragraph. This is a skill documentation page, not a description. +""" + +PROMPT_BLOCK_OBJECTIVE = """ +# Task Objective +You are a professional User Memory Extractor. Your core task is to extract skills, capabilities, and technical competencies demonstrated or described in the resource content (agent logs, workflow documentation, execution traces, or technical documents). Format each skill as a comprehensive, production-ready skill profile that can be referenced and applied. +""" + +PROMPT_BLOCK_WORKFLOW = """ +# Workflow +Read the full resource content to understand the context and technical details. +## Extract skills +Identify valuable skills, capabilities, and technical competencies demonstrated in the content. +## Create skill profiles +For each skill, create a comprehensive profile with all required sections. +## Review & validate +Ensure each skill profile is complete, actionable, and meets the minimum 300 words requirement. +## Final output +Output Skill Information as structured skill profiles. +""" + +PROMPT_BLOCK_RULES = """ +# Rules +## General requirements (must satisfy all) +- Each skill must be formatted as a comprehensive skill profile with frontmatter and all required sections. +- Each skill profile must capture not just WHAT was done, but HOW and WHY it works. +- Be specific and concrete - include technology names, version numbers, metrics, and outcomes. +- Each skill should be comprehensive enough to be referenced and applied independently. +- Minimum 300 words per skill to ensure depth and actionability. +Important: Extract only skills that are clearly demonstrated or described in the resource. No guesses or fabricated details. + +## Skill Profile Structure (must include all sections) +1. Frontmatter: name, description, category, demonstrated-in +2. Introduction paragraph +3. Core Principles +4. When to Use This Skill +5. Implementation Guide (Prerequisites, Techniques and Approaches, Example from Resource) +6. Success Patterns +7. Common Pitfalls +8. Key Takeaways + +## Special rules for Skill Information +- Generic statements without concrete approaches are forbidden (e.g., "Used Docker", "Good at programming"). +- Opinions without demonstrated practice are forbidden (e.g., "I think microservices are better"). +- Theory without practice belongs to knowledge type, not skill type. +- One-time luck without a replicable approach is not a skill. +- Trivial actions are not skills (e.g., "Using email", "Reading docs"). + +## What TO Extract +- Concrete approaches with context, tools, metrics, and outcomes. +- Deployment strategies with specific techniques (canary, blue-green, etc.). +- Incident response procedures with detection, response, and recovery steps. +- Problem-solving approaches with tool orchestration patterns. +- Multi-step workflows with reasoning steps and validation approaches. + +## Resource Type Guidelines +### For Deployment Logs: +- Extract each significant deployment (success or failure) as a separate skill. +- Success: Focus on techniques that worked. +- Failure: Focus on incident response, root cause analysis, recovery procedures. +- Include metrics: deployment time, error rates, response times, recovery time. + +### For Workflow Documentation: +- Extract major workflow stages as skills. +- Include tool chains and technology stacks. +- Document step-by-step procedures. +- Note success metrics and KPIs. + +### For Agent Execution Logs: +- Extract problem-solving approaches as skills. +- Include tool orchestration patterns. +- Document reasoning steps and validation approaches. +- Capture multi-step workflows. + +## Review & validation rules +- Ensure all required sections are present in each skill profile. +- Verify minimum 300 words per skill. +- Final check: every skill profile must be actionable and replicable. +""" + +PROMPT_BLOCK_CATEGORY = """ +## Memory Categories: +{categories_str} +""" + +PROMPT_BLOCK_OUTPUT = """ +# Output Format (XML) +Return all memories wrapped in a single element: + + + +--- +name: skill-name-in-kebab-case +description: One-line description of what this skill enables +category: primary-category +demonstrated-in: [context where this was shown] +--- + +[Brief introduction explaining the skill and its importance] + +## Core Principles +- [Key concept 1] +- [Key concept 2] + +## When to Use This Skill +- [Situation 1] +- [Situation 2] + +## Implementation Guide +### Prerequisites +- [Required knowledge or setup] + +### Techniques and Approaches +[Detailed explanation of how to apply this skill] + +### Example from Resource +[Concrete example from the source material] + +## Success Patterns +- [Pattern 1 with explanation] +- [Pattern 2 with explanation] + +## Common Pitfalls +- [Pitfall 1]: [Why it's a problem and how to avoid it] +- [Pitfall 2]: [Why it's a problem and how to avoid it] + +## Key Takeaways +- [Critical insight 1] +- [Critical insight 2] + + + technical_skills + + + +""" + +PROMPT_BLOCK_EXAMPLES = """ +# Examples (Input / Output / Explanation) +Example 1: Skill Extraction from Deployment Log +## Input +[2024-01-15 10:30:00] Starting canary deployment for Payment Service v2.3.1 +[2024-01-15 10:30:15] Deployed new version alongside existing v2.3.0 +[2024-01-15 10:30:30] Configured load balancer: 10% traffic to v2.3.1 +[2024-01-15 10:35:30] Metrics check: Error rate 0.1% (baseline 0.1%), p95 latency 270ms (baseline 320ms) +[2024-01-15 10:35:45] Increasing traffic to 50% +[2024-01-15 10:40:45] Metrics stable, increasing to 100% +[2024-01-15 10:42:00] Deployment complete. Zero downtime achieved. +## Output + + + +--- +name: canary-deployment-with-monitoring +description: Implement gradual traffic shifting deployment strategy with real-time monitoring +category: deployment +demonstrated-in: [Payment Service v2.3.1 deployment] +--- + +Canary deployment is a risk-mitigation strategy that gradually shifts production traffic from an old version to a new version while continuously monitoring key metrics. This approach enables early detection of issues with minimal user impact. + +## Core Principles +- Gradual exposure: Start with a small percentage of traffic (typically 5-10%) to limit blast radius +- Continuous monitoring: Track error rates, response times in real-time +- Quick recovery: Maintain ability to instantly route traffic back to stable version + +## When to Use This Skill +- Deploying critical services where downtime is unacceptable +- Rolling out changes with uncertain production behavior +- High-traffic services where testing production performance is valuable + +## Implementation Guide +### Prerequisites +- Load balancer with traffic splitting capabilities +- Monitoring system with real-time metrics +- Automated deployment pipeline + +### Techniques and Approaches +1. Initial Deployment (10% traffic): Deploy new version alongside existing, route 10% traffic, monitor 5 minutes +2. Monitoring Checkpoints: Error rate should not exceed baseline by more than 2%, response time within 20% of baseline +3. Gradual Rollout: If metrics stable, progress 10% to 25% to 50% to 100% + +### Example from Resource +Payment Service v2.3.1 deployment achieved zero downtime during 12-minute deployment. Traffic progressed 10% to 50% to 100% with 5-minute pauses. Response time improved 15% (320ms to 270ms p95). Error rate remained stable at 0.1%. + +## Success Patterns +- Small initial percentage: 5-10% catches most issues while limiting impact +- Metric-driven automation: Removes human error from rollback decisions + +## Common Pitfalls +- Too aggressive progression: Rushing from 10% to 100% defeats the purpose +- Insufficient monitoring window: Need 5+ minutes at each stage to detect issues + +## Key Takeaways +- Canary deployments trade deployment speed for safety +- Start small, progress gradually, monitor continuously + + + technical_skills + + + +## Explanation +A comprehensive skill profile is extracted from the deployment log, capturing the approach, techniques, metrics, and outcomes. The profile includes all required sections and provides actionable guidance for replicating the skill. +""" + +PROMPT_BLOCK_INPUT = """ +# Original Resource: + +{resource} + +""" + +PROMPT = "\n\n".join([ + PROMPT_BLOCK_OBJECTIVE.strip(), + PROMPT_BLOCK_WORKFLOW.strip(), + PROMPT_BLOCK_RULES.strip(), + PROMPT_BLOCK_CATEGORY.strip(), + PROMPT_BLOCK_OUTPUT.strip(), + PROMPT_BLOCK_EXAMPLES.strip(), + PROMPT_BLOCK_INPUT.strip(), +]) + +CUSTOM_PROMPT = { + "objective": PROMPT_BLOCK_OBJECTIVE.strip(), + "workflow": PROMPT_BLOCK_WORKFLOW.strip(), + "rules": PROMPT_BLOCK_RULES.strip(), + "category": PROMPT_BLOCK_CATEGORY.strip(), + "output": PROMPT_BLOCK_OUTPUT.strip(), + "examples": PROMPT_BLOCK_EXAMPLES.strip(), + "input": PROMPT_BLOCK_INPUT.strip(), +} diff --git a/src/memu/prompts/memory_type/tool.py b/src/memu/prompts/memory_type/tool.py new file mode 100644 index 0000000..91aec4e --- /dev/null +++ b/src/memu/prompts/memory_type/tool.py @@ -0,0 +1,120 @@ +PROMPT_BLOCK_OBJECTIVE = """ +# Task Objective +You are a professional Tool Memory Extractor. Your core task is to extract tool usage patterns, execution results, and learnings from agent logs or tool execution traces. This enables agents to learn from their tool usage history. +""" + +PROMPT_BLOCK_WORKFLOW = """ +# Workflow +Read the full resource content to understand tool execution context. +## Extract tool memories +Identify tool calls, their inputs, outputs, success/failure status, and any patterns. +## Create tool memory entries +For each significant tool usage, create a memory entry with when_to_use hints. +## Review & validate +Ensure each tool memory is actionable and helps future tool selection. +## Final output +Output Tool Memory entries. +""" + +PROMPT_BLOCK_RULES = """ +# Rules +## General requirements (must satisfy all) +- Each tool memory must capture: tool name, what it was used for, outcome, and when to use it again. +- Focus on patterns that help future tool selection decisions. +- Include success/failure context to help agents avoid repeated mistakes. +- Each memory should help answer: "When should I use this tool?" + +## What TO Extract +- Successful tool usage patterns with context +- Failed tool attempts with lessons learned +- Tool combinations that work well together +- Performance insights (fast vs slow tools for different tasks) + +## What NOT to Extract +- Trivial tool calls without learning value +- Duplicate patterns already captured +- Tool calls with no meaningful outcome + +## Memory Item Content Requirements +- Include the tool name prominently +- Describe the use case or scenario +- Note the outcome (success/failure/partial) +- Provide a "when_to_use" hint for future retrieval +""" + +PROMPT_BLOCK_CATEGORY = """ +## Memory Categories: +{categories_str} +""" + +PROMPT_BLOCK_OUTPUT = """ +# Output Format (XML) +Return all memories wrapped in a single element: + + + Tool memory content describing the tool usage pattern + Hint for when this memory should be retrieved + + Category Name + + + +""" + +PROMPT_BLOCK_EXAMPLES = """ +# Examples (Input / Output / Explanation) +Example 1: Tool Memory Extraction +## Input +[Tool Call] file_reader(path="/data/config.json") +[Result] Success - Read 2048 bytes in 0.3s +[Tool Call] json_parser(content=) +[Result] Success - Parsed config with 15 keys +[Tool Call] file_reader(path="/data/missing.json") +[Result] Error - FileNotFoundError: File does not exist +## Output + + + The file_reader tool successfully reads JSON config files from /data/ directory. Average read time is 0.3s for ~2KB files. Works well when chained with json_parser for config processing. + When needing to read configuration files or JSON data from the filesystem + + file_operations + + + + The file_reader tool fails with FileNotFoundError when the target file doesn't exist. Should verify file existence before reading or handle the error gracefully. + When handling file read errors or implementing robust file operations + + error_handling + + + +## Explanation +Two tool memories are extracted: one for successful usage pattern, one for error handling insight. Both include when_to_use hints for smart retrieval. +""" + +PROMPT_BLOCK_INPUT = """ +# Original Resource: + +{resource} + +""" + +PROMPT = "\n\n".join([ + PROMPT_BLOCK_OBJECTIVE.strip(), + PROMPT_BLOCK_WORKFLOW.strip(), + PROMPT_BLOCK_RULES.strip(), + PROMPT_BLOCK_CATEGORY.strip(), + PROMPT_BLOCK_OUTPUT.strip(), + PROMPT_BLOCK_EXAMPLES.strip(), + PROMPT_BLOCK_INPUT.strip(), +]) + +CUSTOM_PROMPT = { + "objective": PROMPT_BLOCK_OBJECTIVE.strip(), + "workflow": PROMPT_BLOCK_WORKFLOW.strip(), + "rules": PROMPT_BLOCK_RULES.strip(), + "category": PROMPT_BLOCK_CATEGORY.strip(), + "output": PROMPT_BLOCK_OUTPUT.strip(), + "examples": PROMPT_BLOCK_EXAMPLES.strip(), + "input": PROMPT_BLOCK_INPUT.strip(), +} diff --git a/src/memu/prompts/preprocess/__init__.py b/src/memu/prompts/preprocess/__init__.py new file mode 100644 index 0000000..c03de33 --- /dev/null +++ b/src/memu/prompts/preprocess/__init__.py @@ -0,0 +1,11 @@ +from memu.prompts.preprocess import audio, conversation, document, image, video + +PROMPTS: dict[str, str] = { + "conversation": conversation.PROMPT.strip(), + "video": video.PROMPT.strip(), + "image": image.PROMPT.strip(), + "document": document.PROMPT.strip(), + "audio": audio.PROMPT.strip(), +} + +__all__ = ["PROMPTS"] diff --git a/src/memu/prompts/preprocess/audio.py b/src/memu/prompts/preprocess/audio.py new file mode 100644 index 0000000..2f4fe31 --- /dev/null +++ b/src/memu/prompts/preprocess/audio.py @@ -0,0 +1,62 @@ +PROMPT = """ +# Task Objective +You are given the transcription of an audio recording. Produce two outputs: +1. A **Processed Content** version that is clean and well-formatted, prefixed with a + short overview describing what kind of audio this is. +2. A **Caption** that summarizes the audio in one sentence, starting with its type. + +# Step 1 - Classify the Audio +Infer the nature of the audio from the transcription alone. Pick the single +best-fitting type, for example: +- conversation / dialogue (two or more speakers talking) +- monologue / narration / voice memo (a single speaker) +- lecture / presentation / talk +- interview +- podcast / radio segment +- meeting / phone call +- song / music (lyrics, repeated chorus, verse structure) +- announcement / advertisement +- other (describe it briefly) +If the type is uncertain, choose the closest match and note that it is approximate. + +# Step 2 - Build an Overview +Write a short "Audio Overview" (a few bullet lines) capturing what can be inferred: +- Type: the category from Step 1 +- Language: the primary language(s) spoken +- Speakers: approximate number of distinct speakers, if discernible +- Topic: the main subject or purpose of the audio + +# Step 3 - Clean the Content +1. Correct punctuation, capitalization, and obvious transcription artifacts. +2. Add paragraph breaks at topic shifts. For songs, keep verse/chorus line breaks. +3. Preserve the original meaning, wording, sequence, and language of the audio. + +# Rules +- Base the type and overview only on evidence present in the transcription; do not + invent speakers, topics, or details that are not supported by the text. +- Do not add, remove, or reinterpret content beyond cleaning and formatting. +- The caption must be **exactly one sentence** and must begin with the audio type + (for example: "A conversation about ...", "A song about ...", "A lecture on ..."). +- Use clear, neutral language and keep the content in its original language. + +# Output Format +Use the following structure: + + +## Audio Overview +- Type: [audio type] +- Language: [language(s)] +- Speakers: [number or "unknown"] +- Topic: [main topic or purpose] + +[Provide the cleaned and formatted transcription here] + + + +[One-sentence summary that begins with the audio type] + + +# Input +Transcription: +{transcription} +""" diff --git a/src/memu/prompts/preprocess/conversation.py b/src/memu/prompts/preprocess/conversation.py new file mode 100644 index 0000000..f780901 --- /dev/null +++ b/src/memu/prompts/preprocess/conversation.py @@ -0,0 +1,43 @@ +PROMPT = """ +# Task Objective +Analyze a conversation with message indices and divide it into multiple meaningful segments based on topic changes, time gaps, or natural breaks. + +# Workflow +1. Review the entire **Conversation Content** along with its message indices. +2. Identify potential **segment boundaries** by observing: + - Topic changes + - Time gaps or pauses + - Natural conclusions of a discussion + - Clear shifts in tone or semantic focus +3. Group messages into segments that each maintain a coherent theme. +4. Ensure each segment has a clear beginning and end. +5. Verify that each segment contains **at least 20 messages**. +6. Record the `start` and `end` indices (inclusive) for each segment. + +# Rules +- Segments must be based strictly on the provided conversation content. +- Each segment must: + - Contain **≥ 20 messages** + - Maintain a **coherent theme** + - Have a **clear boundary** from adjacent segments +- Use only the provided `[INDEX]` numbers. +- Do not overlap segments. +- Do not include explanations, comments, or extra text in the final output. + +# Output Format +Return **only valid JSON** in the following structure: + +```json +{{ + "segments": [ + {{"start": x, "end": x}}, + {{"start": x, "end": x}}, + {{"start": x, "end": x}} + ] +}} +``` + +# Input +Conversation Content: +{conversation} +""" diff --git a/src/memu/prompts/preprocess/document.py b/src/memu/prompts/preprocess/document.py new file mode 100644 index 0000000..c01c508 --- /dev/null +++ b/src/memu/prompts/preprocess/document.py @@ -0,0 +1,35 @@ +PROMPT = """ +# Task Objective +Analyze the provided document text and produce two outputs: +1. A condensed version that preserves all key information and important details while removing verbosity and redundancy. +2. A one-sentence caption summarizing what the document is about. + +# Workflow +1. Read the **Document** carefully to understand its full content. +2. Identify the main points, key arguments, and essential details. +3. Remove repetition, filler, and unnecessary verbosity while preserving meaning and completeness. +4. Rewrite the content in a concise, structured form. +5. Generate a single-sentence **Caption** that accurately summarizes the document's purpose or topic. + +# Rules +- Preserve all key information, facts, and conclusions. +- Do not introduce new information or interpretations. +- Keep the processed content concise but complete. +- The caption must be exactly **one sentence**. +- Use only the information contained in the provided document. + +# Output Format +Use the following structure: + + +[Provide the condensed version of the document here] + + + +[Provide a one-sentence summary of the document] + + +# Input +Document: +{document_text} +""" diff --git a/src/memu/prompts/preprocess/image.py b/src/memu/prompts/preprocess/image.py new file mode 100644 index 0000000..394decb --- /dev/null +++ b/src/memu/prompts/preprocess/image.py @@ -0,0 +1,34 @@ +PROMPT = """ +# Task Objective +Analyze the given image and produce two outputs: +1. A **Detailed Description** that thoroughly explains what is shown in the image. +2. A **Caption** that summarizes the image in a single sentence. + +# Workflow +1. Examine the image carefully. +2. Identify the **main subjects and objects** present. +3. Describe any **actions or activities** taking place. +4. Analyze the **setting, background, and environment**. +5. Note any **visible text, signs, or labels**. +6. Describe **colors, lighting, composition**, and visual layout. +7. Infer the **overall mood, atmosphere, or style** of the image. +8. Write a concise **one-sentence caption** that captures the essence of the image. + +# Rules +- Base the description strictly on what is visible in the image. +- Do not invent details that cannot be inferred visually. +- Be comprehensive in the detailed description but clear and structured. +- The caption must be **exactly one sentence**. +- Use neutral and objective language. + +# Output Format +Use the following structure: + + +[Provide the comprehensive description here] + + + +[Provide a one-sentence summary of the image] + +""" diff --git a/src/memu/prompts/preprocess/video.py b/src/memu/prompts/preprocess/video.py new file mode 100644 index 0000000..f095c44 --- /dev/null +++ b/src/memu/prompts/preprocess/video.py @@ -0,0 +1,35 @@ +PROMPT = """ +# Task Objective +Analyze the given video and produce two outputs: +1. A **Detailed Description** that comprehensively explains what happens in the video. +2. A **Caption** that summarizes the video in a single sentence. + +# Workflow +1. Watch the video carefully from start to finish. +2. Identify the **main actions and activities** taking place over time. +3. Describe the **key objects, people, and subjects** appearing in the video. +4. Analyze the **scene, setting, and environment**. +5. Note any **audio elements**, including dialogue, narration, music, or background sounds (if available). +6. Highlight **important events or moments** in the video. +7. Describe the **temporal flow**, explaining how the events progress from beginning to end. +8. Write a concise **one-sentence caption** that captures the essence of the video. + +# Rules +- Base the description strictly on observable visual and audio content from the video. +- Do not invent details that cannot be inferred from the video. +- Be comprehensive and chronological in the detailed description. +- The caption must be **exactly one sentence**. +- Use clear, neutral, and objective language. + +# Output Format +Use the following structure: + + +[Provide the comprehensive description here] + + + +[Provide a one-sentence summary of the video] + + +""" diff --git a/src/memu/prompts/retrieve/__init__.py b/src/memu/prompts/retrieve/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/memu/prompts/retrieve/judger.py b/src/memu/prompts/retrieve/judger.py new file mode 100644 index 0000000..53462b6 --- /dev/null +++ b/src/memu/prompts/retrieve/judger.py @@ -0,0 +1,39 @@ +PROMPT = """ +# Task Objective +Judge whether the retrieved content is sufficient to answer the user's query. + +# Workflow +1. Analyze the **Query** to understand what the user is asking. +2. Review the **Retrieved Content** carefully. +3. Evaluate the retrieved content against the following criteria: + - Does it directly address the user's question? + - Is the information specific and detailed enough? + - Are there obvious gaps or missing details? + - Did the user explicitly ask to recall or remember more information? +4. Based on this evaluation, decide whether the information is sufficient or more is needed. + +# Rules +- Base your judgement **only** on the provided query and retrieved content. +- Do not assume or add external knowledge. +- Do not provide additional explanations beyond the required sections. +- The final judgement must be **one word only**. + +# Output Format +Use the following structure: + + +Explain your reasoning for how you made the judgement. + + + +ENOUGH or MORE + + + +# Input +Query: +{query} + +Retrieved Content: +{content} +""" diff --git a/src/memu/prompts/retrieve/llm_category_ranker.py b/src/memu/prompts/retrieve/llm_category_ranker.py new file mode 100644 index 0000000..91d62f2 --- /dev/null +++ b/src/memu/prompts/retrieve/llm_category_ranker.py @@ -0,0 +1,35 @@ +PROMPT = """ +# Task Objective +Search through the provided categories and identify the most relevant ones for the given query, then rank them by relevance. + +# Workflow +1. Analyze the **Query** to understand its intent and key topics. +2. Review all **Available Categories**. +3. Determine which categories are relevant to the query. +4. Select up to **{top_k}** most relevant categories. +5. Rank the selected categories from most to least relevant. + +# Rules +- Only include categories that are actually relevant to the query. +- Include **at most** {top_k} categories. +- Ranking matters: the first category must be the most relevant. +- Do not invent or modify category IDs. +- If no categories are relevant, return an empty array. + +# Output Format +Return the result as a JSON object in the following format: + +```json +{{ + "analysis": "your analysis process", + "categories": ["category_id_1", "category_id_2", "category_id_3"] +}} +``` + +# Input +Query: +{query} + +Available Categories: +{categories_data} +""" diff --git a/src/memu/prompts/retrieve/llm_item_ranker.py b/src/memu/prompts/retrieve/llm_item_ranker.py new file mode 100644 index 0000000..53e02d4 --- /dev/null +++ b/src/memu/prompts/retrieve/llm_item_ranker.py @@ -0,0 +1,40 @@ +PROMPT = """ +# Task Objective +Search through the provided memory items and identify the most relevant ones for the given query, based on the already identified relevant categories, then rank them by relevance. + +# Workflow +1. Analyze the **Query** to understand intent and key information needs. +2. Review the **Relevant Categories** provided to understand the scope. +3. Examine all **Available Memory Items** within those categories. +4. Identify which memory items are truly relevant to the query. +5. Select up to **top_k** most relevant items. +6. Rank the selected items from most to least relevant. + +# Rules +- Only consider memory items that belong to the provided relevant categories. +- Only include items that are actually relevant to the query. +- Include **at most** {top_k} items. +- Order matters: the first item must be the most relevant. +- Do not invent, modify, or infer item IDs. +- If no relevant items are found, return an empty array. + +# Output Format +Return the result as a JSON object in the following format: + +```json +{{ + "analysis": "your analysis process", + "items": ["item_id_1", "item_id_2", "item_id_3"] +}} +``` + +# Input +Query: +{query} + +Available Memory Items: +{items_data} + +These memory items belong to the following relevant categories that were already identified: +{relevant_categories} +""" diff --git a/src/memu/prompts/retrieve/llm_resource_ranker.py b/src/memu/prompts/retrieve/llm_resource_ranker.py new file mode 100644 index 0000000..74a0240 --- /dev/null +++ b/src/memu/prompts/retrieve/llm_resource_ranker.py @@ -0,0 +1,40 @@ +PROMPT = """ +# Task Objective +Search through the provided resources and identify the most relevant ones for the given query, then rank them by relevance. + +# Workflow +1. Analyze the **Query** to understand its intent and key information needs. +2. Review the provided **Context Info** , which contains already-identified categories and items, to guide relevance. +3. Examine all **Available Resources**. +4. Determine which resources are actually relevant to the query. +5. Select up to {top_k} most relevant resources. +6. Rank the selected resources from most to least relevant. + +# Rules +- Only include resources that are actually relevant to the query. +- Include **at most {top_k}** resources. +- Ranking matters: the first resource must be the most relevant. +- Do not invent, modify, or infer resource IDs. +- If no resources are relevant, return an empty array. + +# Output Format +Return the result as a JSON object in the following format: + +```json +{{ + "analysis": "your analysis process", + "resources": ["resource_id_1", "resource_id_2", "resource_id_3"] +}} +``` + +# Input +Query: +{query} + +Context Info: +{context_info} + +Available Resources: +{resources_data} + +""" diff --git a/src/memu/prompts/retrieve/pre_retrieval_decision.py b/src/memu/prompts/retrieve/pre_retrieval_decision.py new file mode 100644 index 0000000..3483112 --- /dev/null +++ b/src/memu/prompts/retrieve/pre_retrieval_decision.py @@ -0,0 +1,53 @@ +SYSTEM_PROMPT = """ +# Task Objective +Determine whether the current query requires retrieving information from memory or can be answered directly without retrieval. +If retrieval is required, rewrite the query to include relevant contextual information. + +# Workflow +1. Review the **Query Context** to understand prior conversation and available background. +2. Analyze the **Current Query**. +3. Consider the **Retrieved Content**, if any. +4. Decide whether memory retrieval is required based on the criteria. +5. If retrieval is needed, rewrite the query to incorporate relevant context from the query context. +6. If retrieval is not needed, keep the original query unchanged. + +# Rules +- **NO_RETRIEVE** for: + - Greetings, casual chat, or acknowledgments + - Questions about only the current conversation/context + - General knowledge questions + - Requests for clarification + - Meta-questions about the system itself +- **RETRIEVE** for: + - Questions about past events, conversations, or interactions + - Queries about user preferences, habits, or characteristics + - Requests to recall specific information + - Questions referencing historical data +- Do not add external knowledge beyond the provided context. +- If retrieval is not required, return the original query exactly. + +# Output Format +Use the following structure: + + +RETRIEVE or NO_RETRIEVE + + + +If RETRIEVE: provide a rewritten query incorporating relevant context. +If NO_RETRIEVE: return `{query}` unchanged. + +""" + + +USER_PROMPT = """ +# Input +Query Context: +{conversation_history} + +Current Query: +{query} + +Retrieved Content: +{retrieved_content} +""" diff --git a/src/memu/prompts/retrieve/query_rewriter.py b/src/memu/prompts/retrieve/query_rewriter.py new file mode 100644 index 0000000..be9e686 --- /dev/null +++ b/src/memu/prompts/retrieve/query_rewriter.py @@ -0,0 +1,44 @@ +PROMPT = """ +# Task Objective +Rewrite a user query to make it self-contained and explicit by resolving references and ambiguities using the conversation history. + +# Workflow +1. Review the **Conversation History** to identify relevant entities, topics, and context. +2. Analyze the **Current Query**. +3. Determine whether the query contains: + - Pronouns (e.g., “they”, “it”, “their”, “his”, “her”) + - Referential expressions (e.g., “that”, “those”, “the same”) + - Implicit context (e.g., “what about…”, “and also…”) + - Incomplete information that can be inferred from the conversation history +4. If rewriting is needed: + - Replace pronouns with specific entities mentioned in the conversation + - Add necessary background from the conversation history + - Make implicit references explicit + - Ensure the rewritten query is understandable on its own +5. If the query is already clear and self-contained, keep it unchanged. + +# Rules +- Preserve the original intent of the user query. +- Only use information explicitly available in the conversation history. +- Do not introduce new assumptions or external knowledge. +- Keep the rewritten query concise but fully explicit. + +# Output Format +Use the following structure: + + +Brief analysis of whether the query needs rewriting and why. + + + +The rewritten query that is self-contained and explicit if no rewrite is needed). + + + +# Input +Query Context: +{conversation_history} + +Current Query: +{query} +""" diff --git a/src/memu/prompts/retrieve/query_rewriter_judger.py b/src/memu/prompts/retrieve/query_rewriter_judger.py new file mode 100644 index 0000000..2d8af79 --- /dev/null +++ b/src/memu/prompts/retrieve/query_rewriter_judger.py @@ -0,0 +1,48 @@ +SYSTEM_PROMPT = """ +# Task Objective +Perform two tasks: +1. **Query Rewriting** - Incorporate conversation context to make the query more specific and clear. +2. **Sufficiency Judgment** - Determine whether the retrieved content is sufficient to answer the query. + +You should be conservative and only mark the result as **ENOUGH** when the retrieved content truly provides adequate information. + +# Workflow +1. Review the **Query Context** to understand prior conversation and background. +2. Analyze the **Original Query**. +3. Examine the **Retrieved Content So Far**. +4. Rewrite the query by incorporating relevant context to improve clarity and specificity. +5. Judge whether the retrieved content is sufficient to answer the rewritten query based on defined criteria. + +# Rules +- Query rewriting must stay faithful to the user's original intent. +- Only incorporate context that is relevant and helpful. +- Do not introduce new assumptions or external knowledge. +- Mark **ENOUGH** only if: + - The retrieved content directly addresses the query, **and** + - The information is specific and detailed enough, **and** + - There are no obvious gaps or missing details. +- If any key information is missing or unclear, mark **MORE**. + +# Output Format +Use the following structure: + + +[Provide the rewritten query with conversation context] + + + +ENOUGH or MORE + +""" + +USER_PROMPT = """ +Input: +Query Context: +{conversation_history} + +Original Query: +{original_query} + +Retrieved Content So Far: +{retrieved_content} +""" diff --git a/src/memu/utils/__init__.py b/src/memu/utils/__init__.py new file mode 100644 index 0000000..16f8280 --- /dev/null +++ b/src/memu/utils/__init__.py @@ -0,0 +1,5 @@ +"""Utility modules for memU.""" + +from memu.utils.video import VideoFrameExtractor + +__all__ = ["VideoFrameExtractor"] diff --git a/src/memu/utils/conversation.py b/src/memu/utils/conversation.py new file mode 100644 index 0000000..7ac3dea --- /dev/null +++ b/src/memu/utils/conversation.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import json +from typing import Any + + +def format_conversation_for_preprocess(raw_text: str) -> str: + """ + Normalize a conversation into a line-based format suitable for LLM preprocessing prompts. + + Supported input formats: + - A JSON list of messages: [{"role": "...", "content": "...", "created_at": "..."}] + - A JSON dict with a "content" list: {"content": [ ...messages... ]} + + Output format: + - One message per line + - Each line starts with an index marker: "[{idx}]" + - If a created_at is available, it is included after the index + - The role is included in square brackets: "[user]" / "[assistant]" etc. + + Notes: + - This function expects conversation data to be JSON. + - Newlines in message content are collapsed to spaces to keep one message per line. + """ + stripped = (raw_text or "").strip() + if not stripped: + return "" + + parsed = _try_parse_json(stripped) + if parsed is None: + # Conversation inputs are expected to be JSON. If invalid, return original text. + return raw_text + messages = _extract_messages(parsed) + if messages is None: + return raw_text + return _format_messages(messages) + + +def _try_parse_json(text: str) -> Any | None: + if not text: + return None + if not (text.startswith("[") or text.startswith("{")): + return None + try: + return json.loads(text) + except Exception: + return None + + +def _extract_messages(payload: Any) -> list[dict[str, Any]] | None: + if isinstance(payload, list): + return [m for m in payload if isinstance(m, dict)] + if isinstance(payload, dict): + content = payload.get("content") + if isinstance(content, list): + return [m for m in content if isinstance(m, dict)] + return None + + +def _format_messages(messages: list[dict[str, Any]]) -> str: + out: list[str] = [] + for idx, msg in enumerate(messages): + role = str(msg.get("role") or "user").strip() or "user" + content = msg.get("content") + text = _extract_text_content(content) + created_at = _extract_created_at(msg) + created_part = f"{created_at} " if created_at else "" + out.append(f"[{idx}] {created_part}[{role}]: {text}") + return "\n".join(out) + + +def _extract_text_content(content: Any) -> str: + if isinstance(content, dict): + text = content.get("text", "") + elif isinstance(content, str): + text = content + else: + text = "" if content is None else str(content) + # Ensure single-line to keep indexing consistent + return " ".join(str(text).splitlines()).strip() + + +def _extract_created_at(msg: dict[str, Any]) -> str | None: + raw = msg.get("created_at") + if raw is None: + return None + + s = str(raw).strip() + return s or None diff --git a/src/memu/utils/references.py b/src/memu/utils/references.py new file mode 100644 index 0000000..a003ad5 --- /dev/null +++ b/src/memu/utils/references.py @@ -0,0 +1,172 @@ +""" +Utilities for handling memory item references in category summaries. + +References are inline citations in the format [ref:ITEM_ID] that link +specific statements in category summaries to their source memory items. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from memu.database.interfaces import Database + +# Pattern to match references like [ref:abc123] or [ref:abc123,def456] +REFERENCE_PATTERN = re.compile(r"\[ref:([a-zA-Z0-9_,\-]+)\]") + + +def extract_references(text: str | None) -> list[str]: + """ + Extract all item IDs referenced in a text. + + Args: + text: Text containing [ref:ITEM_ID] citations + + Returns: + List of unique item IDs found in references + + Example: + >>> extract_references("User loves coffee [ref:abc123]. Also tea [ref:def456].") + ['abc123', 'def456'] + """ + if not text: + return [] + + item_ids: list[str] = [] + seen: set[str] = set() + + for match in REFERENCE_PATTERN.finditer(text): + # Handle comma-separated IDs like [ref:abc,def] + ids_str = match.group(1) + for item_id in ids_str.split(","): + item_id = item_id.strip() + if item_id and item_id not in seen: + item_ids.append(item_id) + seen.add(item_id) + + return item_ids + + +def strip_references(text: str | None) -> str | None: + """ + Remove all [ref:...] citations from text for clean display. + + Args: + text: Text containing references + + Returns: + Text with references removed + + Example: + >>> strip_references("User loves coffee [ref:abc123].") + 'User loves coffee.' + """ + if not text: + return text + # Remove references + result = REFERENCE_PATTERN.sub("", text) + # Clean up space before punctuation (e.g., " ." -> ".") + result = re.sub(r"\s+([.,;:!?])", r"\1", result) + # Collapse multiple spaces into one and strip + result = " ".join(result.split()) + return result + + +def format_references_as_citations(text: str | None) -> str | None: + """ + Convert [ref:ID] format to numbered citations [1], [2], etc. + + Args: + text: Text with [ref:ID] references + + Returns: + Text with numbered citations and a reference list at the end + + Example: + >>> format_references_as_citations("User loves coffee [ref:abc].") + 'User loves coffee [1].\\n\\nReferences:\\n[1] abc' + """ + if not text: + return text + + refs = extract_references(text) + if not refs: + return text + + # Build ID to number mapping + id_to_num = {ref_id: idx + 1 for idx, ref_id in enumerate(refs)} + + # Replace [ref:ID] with [N] + def replace_ref(match: re.Match) -> str: + ids_str = match.group(1) + nums = [] + for item_id in ids_str.split(","): + item_id = item_id.strip() + if item_id in id_to_num: + nums.append(str(id_to_num[item_id])) + return f"[{','.join(nums)}]" if nums else "" + + result = REFERENCE_PATTERN.sub(replace_ref, text) + + # Add reference list at end + ref_list = "\n".join(f"[{num}] {ref_id}" for ref_id, num in id_to_num.items()) + return f"{result}\n\nReferences:\n{ref_list}" + + +def fetch_referenced_items( + text: str, + store: Database, +) -> list[dict]: + """ + Fetch memory items referenced in text. + + Args: + text: Text containing [ref:ITEM_ID] citations + store: Database store instance + + Returns: + List of memory item dicts with id, summary, memory_type + """ + item_ids = extract_references(text) + if not item_ids: + return [] + + items = [] + for item_id in item_ids: + item = store.recall_entry_repo.get_item(item_id) + if item: + items.append({ + "id": item.id, + "summary": item.summary, + "memory_type": item.memory_type, + }) + + return items + + +def build_item_reference_map(items: list[tuple[str, str]]) -> str: + """ + Build a reference map string for the LLM prompt. + + Args: + items: List of (item_id, summary) tuples + + Returns: + Formatted string showing available item IDs + + Example: + >>> build_item_reference_map([("abc", "User loves coffee")]) + 'Available memory items for reference:\\n- [ref:abc] User loves coffee' + """ + if not items: + return "" + + lines = ["Available memory items for reference:"] + for item_id, summary in items: + # Truncate long summaries + display = summary[:100] + "..." if len(summary) > 100 else summary + lines.append(f"- [ref:{item_id}] {display}") + + return "\n".join(lines) diff --git a/src/memu/utils/tool.py b/src/memu/utils/tool.py new file mode 100644 index 0000000..02a4718 --- /dev/null +++ b/src/memu/utils/tool.py @@ -0,0 +1,102 @@ +"""Utility functions for tool memory operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from memu.database.models import RecallEntry, ToolCallResult + + +def get_tool_calls(item: RecallEntry) -> list[dict[str, Any]]: + """Get tool calls from a memory item's extra field. + + Args: + item: The RecallEntry to get tool calls from + + Returns: + List of tool call dicts, or empty list if none exist + """ + result: list[dict[str, Any]] = (item.extra or {}).get("tool_calls", []) + return result + + +def set_tool_calls(item: RecallEntry, tool_calls: list[dict[str, Any]]) -> None: + """Set tool calls in a memory item's extra field. + + Args: + item: The RecallEntry to set tool calls on + tool_calls: The list of tool call dicts to set + """ + if item.extra is None: + item.extra = {} + item.extra["tool_calls"] = tool_calls + + +def add_tool_call(item: RecallEntry, tool_call: ToolCallResult) -> None: + """Add a tool call result to a memory item (for tool type memories). + + Args: + item: The RecallEntry to add the tool call to (must be tool type) + tool_call: The ToolCallResult to add + + Raises: + ValueError: If the memory item is not of type 'tool' + """ + if item.memory_type != "tool": + msg = "add_tool_call can only be used with tool type memories" + raise ValueError(msg) + tool_call.ensure_hash() + tool_calls = get_tool_calls(item) + tool_calls.append(tool_call.model_dump()) + set_tool_calls(item, tool_calls) + + +def get_tool_statistics(item: RecallEntry, recent_n: int = 20) -> dict[str, Any]: + """Calculate statistics for the most recent N tool calls. + + Args: + item: The RecallEntry to calculate statistics for + recent_n: Number of recent calls to analyze (default: 20) + + Returns: + Dictionary with total_calls, recent_calls_analyzed, avg_time_cost, + success_rate, avg_score, avg_token_cost + """ + tool_calls = get_tool_calls(item) + if not tool_calls: + return { + "total_calls": 0, + "recent_calls_analyzed": 0, + "avg_time_cost": 0.0, + "success_rate": 0.0, + "avg_score": 0.0, + "avg_token_cost": 0.0, + } + + recent_calls = tool_calls[-recent_n:] + recent_count = len(recent_calls) + + # Calculate statistics (tool_calls are now dicts, not ToolCallResult objects) + total_time = sum(c.get("time_cost", 0.0) for c in recent_calls) + avg_time_cost = total_time / recent_count if recent_count > 0 else 0.0 + + successful = sum(1 for c in recent_calls if c.get("success", True)) + success_rate = successful / recent_count if recent_count > 0 else 0.0 + + total_score = sum(c.get("score", 0.0) for c in recent_calls) + avg_score = total_score / recent_count if recent_count > 0 else 0.0 + + valid_token_calls = [c for c in recent_calls if c.get("token_cost", -1) >= 0] + avg_token_cost = ( + sum(c.get("token_cost", 0) for c in valid_token_calls) / len(valid_token_calls) if valid_token_calls else 0.0 + ) + + return { + "total_calls": len(tool_calls), + "recent_calls_analyzed": recent_count, + "avg_time_cost": round(avg_time_cost, 3), + "success_rate": round(success_rate, 4), + "avg_score": round(avg_score, 3), + "avg_token_cost": round(avg_token_cost, 2), + } diff --git a/src/memu/utils/video.py b/src/memu/utils/video.py new file mode 100644 index 0000000..688f3a9 --- /dev/null +++ b/src/memu/utils/video.py @@ -0,0 +1,271 @@ +"""Video processing utilities for frame extraction.""" + +from __future__ import annotations + +import logging +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import ClassVar + +logger = logging.getLogger(__name__) + + +class VideoFrameExtractor: + """Extract frames from video files using ffmpeg.""" + + FFMPEG_BINARIES: ClassVar[set[str]] = {"ffmpeg", "ffprobe"} + + @classmethod + def is_ffmpeg_available(cls) -> bool: + """Check if ffmpeg is available in the system.""" + try: + result = cls._run_ffmpeg_command(["ffmpeg", "-version"], timeout=5, check=False) + except (FileNotFoundError, subprocess.TimeoutExpired, ValueError): + return False + else: + return result.returncode == 0 + + @staticmethod + def extract_middle_frame(video_path: str, output_path: str | None = None) -> str: + """ + Extract the middle frame from a video file. + + Args: + video_path: Path to the video file + output_path: Optional output path for the frame. If None, creates a temp file. + + Returns: + Path to the extracted frame image + + Raises: + RuntimeError: If ffmpeg is not available or extraction fails + """ + if not VideoFrameExtractor.is_ffmpeg_available(): + msg = "ffmpeg is not available. Please install ffmpeg to process videos." + raise RuntimeError(msg) + + video_path_obj = VideoFrameExtractor._resolve_existing_path(video_path, description="Video file") + safe_video_path = str(video_path_obj) + + # Create output path if not provided + created_temp_file = False + if output_path is None: + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp_file: + output_path = tmp_file.name + created_temp_file = True + output_path_obj = VideoFrameExtractor._resolve_output_path(output_path) + safe_output_path = str(output_path_obj) + + try: + # Get video duration + duration_cmd = [ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + safe_video_path, + ] + + logger.debug(f"Getting video duration: {' '.join(duration_cmd)}") + duration_result = VideoFrameExtractor._run_ffmpeg_command(duration_cmd, timeout=30) + + duration = float(duration_result.stdout.strip()) + middle_time = duration / 2 + + logger.debug(f"Video duration: {duration}s, extracting frame at {middle_time}s") + + # Extract frame at middle timestamp + extract_cmd = [ + "ffmpeg", + "-ss", + str(middle_time), + "-i", + safe_video_path, + "-vframes", + "1", + "-q:v", + "2", # High quality + "-y", # Overwrite output file + safe_output_path, + ] + + logger.debug(f"Extracting frame: {' '.join(extract_cmd)}") + VideoFrameExtractor._run_ffmpeg_command(extract_cmd, timeout=30) + + if not output_path_obj.exists(): + msg = f"Frame extraction failed: output file not created at {output_path_obj}" + raise RuntimeError(msg) + else: + logger.info(f"Successfully extracted frame to: {output_path_obj}") + return str(output_path_obj) + + except subprocess.CalledProcessError as e: + if created_temp_file and output_path_obj.exists(): + output_path_obj.unlink() + msg = f"ffmpeg/ffprobe failed: {e.stderr}" + logger.exception(msg) + raise RuntimeError(msg) from e + except subprocess.TimeoutExpired as e: + if created_temp_file and output_path_obj.exists(): + output_path_obj.unlink() + msg = "Video processing timed out" + logger.exception(msg) + raise RuntimeError(msg) from e + + @staticmethod + def extract_multiple_frames( + video_path: str, + num_frames: int = 3, + output_dir: str | None = None, + ) -> list[str]: + """ + Extract multiple evenly-spaced frames from a video. + + Args: + video_path: Path to the video file + num_frames: Number of frames to extract + output_dir: Optional output directory. If None, creates a temp directory. + + Returns: + List of paths to extracted frame images + + Raises: + RuntimeError: If ffmpeg is not available or extraction fails + """ + if not VideoFrameExtractor.is_ffmpeg_available(): + msg = "ffmpeg is not available. Please install ffmpeg to process videos." + raise RuntimeError(msg) + + video_path_obj = VideoFrameExtractor._resolve_existing_path(video_path, description="Video file") + safe_video_path = str(video_path_obj) + + # Create output directory if not provided + created_temp_dir = False + if output_dir is None: + output_dir = tempfile.mkdtemp() + created_temp_dir = True + + output_dir_obj = VideoFrameExtractor._ensure_safe_cli_path(Path(output_dir)) + output_dir_obj.mkdir(parents=True, exist_ok=True) + + try: + # Get video duration + duration_cmd = [ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + safe_video_path, + ] + + logger.debug(f"Getting video duration: {' '.join(duration_cmd)}") + duration_result = VideoFrameExtractor._run_ffmpeg_command(duration_cmd, timeout=30) + + duration = float(duration_result.stdout.strip()) + + # Calculate timestamps for evenly-spaced frames + timestamps = [duration * (i + 1) / (num_frames + 1) for i in range(num_frames)] + + logger.debug(f"Video duration: {duration}s, extracting frames at: {timestamps}") + + frame_paths = [] + for idx, timestamp in enumerate(timestamps): + output_path_obj = VideoFrameExtractor._resolve_output_path(str(output_dir_obj / f"frame_{idx:03d}.jpg")) + + # Extract frame at timestamp + extract_cmd = [ + "ffmpeg", + "-ss", + str(timestamp), + "-i", + safe_video_path, + "-vframes", + "1", + "-q:v", + "2", # High quality + "-y", # Overwrite output file + str(output_path_obj), + ] + + logger.debug(f"Extracting frame {idx + 1}/{num_frames}: {' '.join(extract_cmd)}") + VideoFrameExtractor._run_ffmpeg_command(extract_cmd, timeout=30) + + if not output_path_obj.exists(): + msg = f"Frame extraction failed: output file not created at {output_path_obj}" + raise RuntimeError(msg) + + frame_paths.append(str(output_path_obj)) + + logger.info(f"Successfully extracted {len(frame_paths)} frames to: {output_dir_obj}") + except subprocess.CalledProcessError as e: + if created_temp_dir and output_dir_obj.exists(): + shutil.rmtree(output_dir_obj) + msg = f"ffmpeg/ffprobe failed: {e.stderr}" + logger.exception(msg) + raise RuntimeError(msg) from e + except subprocess.TimeoutExpired as e: + if created_temp_dir and output_dir_obj.exists(): + shutil.rmtree(output_dir_obj) + msg = "Video processing timed out" + logger.exception(msg) + raise RuntimeError(msg) from e + else: + return frame_paths + + @staticmethod + def _ensure_safe_cli_path(path_obj: Path) -> Path: + """Ensure the given path is safe to pass to a CLI command.""" + resolved = path_obj.resolve() + if resolved.name.startswith("-"): + msg = f"Unsafe path for CLI execution: {resolved}" + raise ValueError(msg) + return resolved + + @classmethod + def _resolve_existing_path(cls, path: str, *, description: str) -> Path: + """Resolve and validate an existing filesystem path.""" + path_obj = Path(path) + if not path_obj.exists(): + msg = f"{description} not found: {path}" + raise FileNotFoundError(msg) + return cls._ensure_safe_cli_path(path_obj) + + @classmethod + def _resolve_output_path(cls, path: str) -> Path: + """Resolve output paths (which may not yet exist) for CLI safety.""" + return cls._ensure_safe_cli_path(Path(path)) + + @classmethod + def _run_ffmpeg_command( + cls, + cmd: list[str], + *, + timeout: int, + check: bool = True, + capture_output: bool = True, + ) -> subprocess.CompletedProcess[str]: + """Run an ffmpeg/ffprobe command after validating the executable.""" + if not cmd: + msg = "FFmpeg command cannot be empty." + raise ValueError(msg) + executable = cmd[0] + binary_name = Path(executable).name + if binary_name not in cls.FFMPEG_BINARIES: + msg = f"Unsupported executable '{executable}'" + raise ValueError(msg) + safe_cmd = [executable, *[str(arg) for arg in cmd[1:]]] + return subprocess.run( # noqa: S603 + safe_cmd, + capture_output=capture_output, + text=True, + timeout=timeout, + check=check, + ) diff --git a/src/memu/vector.py b/src/memu/vector.py new file mode 100644 index 0000000..1f02741 --- /dev/null +++ b/src/memu/vector.py @@ -0,0 +1,145 @@ +"""Backend-agnostic vector math for similarity search. + +This module is intentionally storage-neutral: it holds the pure cosine/salience +ranking helpers shared by the app retrieval layer and every database backend. +It must not import from any concrete ``memu.database.*`` backend so that the +abstraction is not inverted (a backend or the app layer depending on another +backend's internals). +""" + +from __future__ import annotations + +import math +from collections.abc import Iterable +from datetime import datetime +from typing import cast + +import numpy as np + + +def _cosine(a: np.ndarray, b: np.ndarray) -> float: + denom = (np.linalg.norm(a) * np.linalg.norm(b)) + 1e-9 + return float(np.dot(a, b) / denom) + + +def salience_score( + similarity: float, + reinforcement_count: int, + last_reinforced_at: datetime | None, + recency_decay_days: float = 30.0, +) -> float: + """ + Compute salience-aware score combining similarity, reinforcement, and recency. + + Formula: similarity * reinforcement_factor * recency_factor + + - reinforcement_factor: log(count + 1) to dampen extreme counts + (Logarithmic scaling prevents runaway dominance by frequently repeated facts) + - recency_factor: exponential decay based on days since last reinforcement + (Uses half-life decay: after recency_decay_days, factor is ~0.5) + + Args: + similarity: Cosine similarity score (0 to 1) + reinforcement_count: Number of times this memory was reinforced + last_reinforced_at: When the memory was last reinforced + recency_decay_days: Half-life for recency decay in days + + Returns: + Salience score (higher = more salient) + """ + # Reinforcement factor (logarithmic to prevent runaway scores) + reinforcement_factor = math.log(reinforcement_count + 1) + + # Recency factor (exponential decay with half-life) + if last_reinforced_at is None: + recency_factor = 0.5 # Unknown recency gets neutral score + else: + now = datetime.now(last_reinforced_at.tzinfo) if last_reinforced_at.tzinfo else datetime.utcnow() + days_ago = (now - last_reinforced_at).total_seconds() / 86400 + # 0.693 = ln(2), gives us proper half-life decay + recency_factor = math.exp(-0.693 * days_ago / recency_decay_days) + + return similarity * reinforcement_factor * recency_factor + + +def cosine_topk( + query_vec: list[float], + corpus: Iterable[tuple[str, list[float] | None]], + k: int = 5, +) -> list[tuple[str, float]]: + if k <= 0: + return [] + + # Filter out None vectors and collect valid entries + ids: list[str] = [] + vecs: list[list[float]] = [] + for _id, vec in corpus: + if vec is not None: + ids.append(_id) + vecs.append(cast(list[float], vec)) + + if not vecs: + return [] + + # Vectorized computation: stack all vectors into a matrix + q = np.array(query_vec, dtype=np.float32) + matrix = np.array(vecs, dtype=np.float32) # shape: (n, dim) + + # Compute all cosine similarities at once + q_norm = np.linalg.norm(q) + vec_norms = np.linalg.norm(matrix, axis=1) + scores = matrix @ q / (vec_norms * q_norm + 1e-9) + + # Use argpartition for O(n) topk selection instead of O(n log n) sort + n = len(scores) + actual_k = min(k, n) + if actual_k == n: + topk_indices = np.argsort(scores)[::-1] + else: + # Get indices of top k elements (unordered), then sort only those + topk_indices = np.argpartition(scores, -actual_k)[-actual_k:] + topk_indices = topk_indices[np.argsort(scores[topk_indices])[::-1]] + + return [(ids[i], float(scores[i])) for i in topk_indices] + + +def cosine_topk_salience( + query_vec: list[float], + corpus: Iterable[tuple[str, list[float] | None, int, datetime | None]], + k: int = 5, + recency_decay_days: float = 30.0, +) -> list[tuple[str, float]]: + """ + Top-k retrieval using salience-aware scoring. + + Ranks memories by: similarity * log(reinforcement+1) * recency_decay + + Args: + query_vec: Query embedding vector + corpus: Iterable of (id, embedding, reinforcement_count, last_reinforced_at) + k: Number of top results to return + recency_decay_days: Half-life for recency decay + + Returns: + List of (id, salience_score) tuples, sorted by score descending + """ + if k <= 0: + return [] + + q = np.array(query_vec, dtype=np.float32) + scored: list[tuple[str, float]] = [] + + for _id, vec, reinforcement_count, last_reinforced_at in corpus: + if vec is None: + continue + vec_list = cast(list[float], vec) + v = np.array(vec_list, dtype=np.float32) + similarity = _cosine(q, v) + score = salience_score(similarity, reinforcement_count, last_reinforced_at, recency_decay_days) + scored.append((_id, score)) + + scored.sort(key=lambda x: x[1], reverse=True) + return scored[:k] + + +__all__ = ["cosine_topk", "cosine_topk_salience", "salience_score"] diff --git a/src/memu/vlm/__init__.py b/src/memu/vlm/__init__.py new file mode 100644 index 0000000..eccabd4 --- /dev/null +++ b/src/memu/vlm/__init__.py @@ -0,0 +1,24 @@ +"""Vision-language model (VLM) clients for multimodal understanding. + +Sibling package to :mod:`memu.llm`, scoped to the multimodal ``vision`` +capability used by image/video preprocessing. It mirrors the LLM package layout: + +- ``backends/``: per-provider vision request/response shapes (HTTP transport). +- ``http_client``/``openai_client``/``anthropic_client``: transport clients. +- ``gateway``: build a client from a :class:`memu.app.settings.VLMConfig`. +- ``defaults``: per-provider latest VLM model picks. +""" + +from __future__ import annotations + +from memu.vlm.base import VLMClient, encode_image +from memu.vlm.defaults import VLM_PROVIDER_DEFAULTS, default_vlm_model +from memu.vlm.gateway import build_vlm_client + +__all__ = [ + "VLM_PROVIDER_DEFAULTS", + "VLMClient", + "build_vlm_client", + "default_vlm_model", + "encode_image", +] diff --git a/src/memu/vlm/anthropic_client.py b/src/memu/vlm/anthropic_client.py new file mode 100644 index 0000000..ed4a396 --- /dev/null +++ b/src/memu/vlm/anthropic_client.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from memu.vlm.base import VLMClient, encode_image + +if TYPE_CHECKING: + from anthropic.types import Message + +logger = logging.getLogger(__name__) + +# Anthropic requires max_tokens on every request; fall back to this when omitted. +_DEFAULT_MAX_TOKENS = 1024 + + +class AnthropicVLMClient(VLMClient): + """Vision-language client backed by the official Anthropic Python SDK.""" + + def __init__( + self, + *, + base_url: str | None, + api_key: str, + vlm_model: str, + max_tokens: int | None = None, + ): + from anthropic import AsyncAnthropic + + self.base_url = base_url.rstrip("/") if base_url else None + self.api_key = api_key or "" + self.vlm_model = vlm_model + self.max_tokens = max_tokens or _DEFAULT_MAX_TOKENS + self.client = AsyncAnthropic(api_key=self.api_key, base_url=self.base_url) + + async def vision( + self, + prompt: str, + image_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, Message]: + base64_image, mime_type = encode_image(image_path) + + kwargs: dict[str, Any] = { + "model": self.vlm_model, + "max_tokens": max_tokens or self.max_tokens, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": mime_type, + "data": base64_image, + }, + }, + ], + } + ], + } + if system_prompt is not None: + kwargs["system"] = system_prompt + + response = await self.client.messages.create(**kwargs) + logger.debug("Anthropic VLM vision response: %s", response) + return _extract_text(response), response + + +def _extract_text(response: Message) -> str: + """Concatenate text blocks from an Anthropic Messages response.""" + parts: list[str] = [] + for block in response.content: + if getattr(block, "type", None) == "text": + parts.append(getattr(block, "text", "")) + return "".join(parts) diff --git a/src/memu/vlm/backends/__init__.py b/src/memu/vlm/backends/__init__.py new file mode 100644 index 0000000..790d711 --- /dev/null +++ b/src/memu/vlm/backends/__init__.py @@ -0,0 +1,19 @@ +from memu.vlm.backends.base import VLMBackend +from memu.vlm.backends.claude import ClaudeVLMBackend +from memu.vlm.backends.doubao import DoubaoVLMBackend +from memu.vlm.backends.grok import GrokVLMBackend +from memu.vlm.backends.kimi import KimiVLMBackend +from memu.vlm.backends.minimax import MiniMaxVLMBackend +from memu.vlm.backends.openai import OpenAIVLMBackend +from memu.vlm.backends.openrouter import OpenRouterVLMBackend + +__all__ = [ + "ClaudeVLMBackend", + "DoubaoVLMBackend", + "GrokVLMBackend", + "KimiVLMBackend", + "MiniMaxVLMBackend", + "OpenAIVLMBackend", + "OpenRouterVLMBackend", + "VLMBackend", +] diff --git a/src/memu/vlm/backends/base.py b/src/memu/vlm/backends/base.py new file mode 100644 index 0000000..6bd75da --- /dev/null +++ b/src/memu/vlm/backends/base.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import Any, ClassVar + + +class VLMBackend: + """Defines how to talk to a specific HTTP vision-language model provider. + + Mirrors :class:`memu.llm.backends.base.LLMBackend` but is scoped to the + multimodal ``vision`` capability: each provider lives in its own module under + :mod:`memu.vlm.backends` and customizes the request endpoint, vision payload + shape, response parsing and (when needed) the auth headers. + """ + + name: str = "base" + vision_endpoint: str = "/chat/completions" + video_endpoint: str = "/chat/completions" + + # Whether this provider can analyze a whole video file natively (frames + + # audio) via :meth:`build_video_payload`. Providers without native video + # understanding leave this ``False`` so callers fall back to frame-based + # image analysis instead. + supports_video: ClassVar[bool] = False + + def default_headers(self, api_key: str) -> dict[str, str]: + """Auth/request headers for this provider. + + Defaults to OpenAI-style bearer auth; providers with a different scheme + (e.g. Anthropic's ``x-api-key``) override this. + """ + return {"Authorization": f"Bearer {api_key}"} + + def build_vision_payload( + self, + *, + prompt: str, + base64_image: str, + mime_type: str, + system_prompt: str | None, + vlm_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + raise NotImplementedError + + def parse_vision_response(self, data: dict[str, Any]) -> str: + raise NotImplementedError + + def build_video_payload( + self, + *, + prompt: str, + video_data_uri: str, + system_prompt: str | None, + vlm_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + """Build the request payload for native whole-video understanding. + + Only meaningful when :attr:`supports_video` is ``True``. + """ + raise NotImplementedError + + def parse_video_response(self, data: dict[str, Any]) -> str: + """Parse a native video response (defaults to the vision parser).""" + return self.parse_vision_response(data) diff --git a/src/memu/vlm/backends/claude.py b/src/memu/vlm/backends/claude.py new file mode 100644 index 0000000..e828807 --- /dev/null +++ b/src/memu/vlm/backends/claude.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.vlm.backends.base import VLMBackend + +# Anthropic requires max_tokens; fall back to this when the caller omits it. +_DEFAULT_MAX_TOKENS = 1024 +_ANTHROPIC_VERSION = "2023-06-01" + + +class ClaudeVLMBackend(VLMBackend): + """Backend for Anthropic Claude vision (native Messages API). + + Unlike the OpenAI-compatible providers, Claude uses ``x-api-key`` auth, a + top-level ``system`` field, a required ``max_tokens``, and a different + image/response shape. + """ + + name = "claude" + vision_endpoint = "/v1/messages" + + def default_headers(self, api_key: str) -> dict[str, str]: + return { + "x-api-key": api_key, + "anthropic-version": _ANTHROPIC_VERSION, + "content-type": "application/json", + } + + def build_vision_payload( + self, + *, + prompt: str, + base64_image: str, + mime_type: str, + system_prompt: str | None, + vlm_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + payload: dict[str, Any] = { + "model": vlm_model, + "max_tokens": max_tokens or _DEFAULT_MAX_TOKENS, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": mime_type, + "data": base64_image, + }, + }, + ], + } + ], + } + if system_prompt: + payload["system"] = system_prompt + return payload + + def parse_vision_response(self, data: dict[str, Any]) -> str: + blocks = data.get("content") or [] + for block in blocks: + if isinstance(block, dict) and block.get("type") == "text": + return cast(str, block.get("text", "")) + return "" diff --git a/src/memu/vlm/backends/doubao.py b/src/memu/vlm/backends/doubao.py new file mode 100644 index 0000000..f03c1e3 --- /dev/null +++ b/src/memu/vlm/backends/doubao.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from memu.vlm.backends.openai import OpenAIVLMBackend + + +class DoubaoVLMBackend(OpenAIVLMBackend): + """Backend for Doubao vision (OpenAI-compatible).""" + + name = "doubao" + vision_endpoint = "/api/v3/chat/completions" diff --git a/src/memu/vlm/backends/grok.py b/src/memu/vlm/backends/grok.py new file mode 100644 index 0000000..5bc79ef --- /dev/null +++ b/src/memu/vlm/backends/grok.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from memu.vlm.backends.openai import OpenAIVLMBackend + + +class GrokVLMBackend(OpenAIVLMBackend): + """Backend for Grok (xAI) vision (OpenAI-compatible).""" + + name = "grok" diff --git a/src/memu/vlm/backends/kimi.py b/src/memu/vlm/backends/kimi.py new file mode 100644 index 0000000..534d3a2 --- /dev/null +++ b/src/memu/vlm/backends/kimi.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from memu.vlm.backends.openai import OpenAIVLMBackend + + +class KimiVLMBackend(OpenAIVLMBackend): + """Backend for Kimi / Moonshot vision (OpenAI-compatible).""" + + name = "kimi" diff --git a/src/memu/vlm/backends/minimax.py b/src/memu/vlm/backends/minimax.py new file mode 100644 index 0000000..449ed5a --- /dev/null +++ b/src/memu/vlm/backends/minimax.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from memu.vlm.backends.openai import OpenAIVLMBackend + + +class MiniMaxVLMBackend(OpenAIVLMBackend): + """Backend for MiniMax vision (OpenAI-compatible chat completions v2).""" + + name = "minimax" + vision_endpoint = "/text/chatcompletion_v2" diff --git a/src/memu/vlm/backends/openai.py b/src/memu/vlm/backends/openai.py new file mode 100644 index 0000000..df7eb21 --- /dev/null +++ b/src/memu/vlm/backends/openai.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from typing import Any, cast + +from memu.vlm.backends.base import VLMBackend + + +class OpenAIVLMBackend(VLMBackend): + """Backend for OpenAI-compatible vision (chat completions with image parts).""" + + name = "openai" + vision_endpoint = "/chat/completions" + + def build_vision_payload( + self, + *, + prompt: str, + base64_image: str, + mime_type: str, + system_prompt: str | None, + vlm_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + messages: list[dict[str, Any]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + + messages.append({ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": {"url": f"data:{mime_type};base64,{base64_image}"}, + }, + ], + }) + + payload: dict[str, Any] = { + "model": vlm_model, + "messages": messages, + "temperature": 0.2, + } + if max_tokens is not None: + payload["max_tokens"] = max_tokens + return payload + + def parse_vision_response(self, data: dict[str, Any]) -> str: + return cast(str, data["choices"][0]["message"]["content"]) + + def build_video_payload( + self, + *, + prompt: str, + video_data_uri: str, + system_prompt: str | None, + vlm_model: str, + max_tokens: int | None, + ) -> dict[str, Any]: + messages: list[dict[str, Any]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + + messages.append({ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "video_url", "video_url": {"url": video_data_uri}}, + ], + }) + + payload: dict[str, Any] = { + "model": vlm_model, + "messages": messages, + "temperature": 0.2, + } + if max_tokens is not None: + payload["max_tokens"] = max_tokens + return payload diff --git a/src/memu/vlm/backends/openrouter.py b/src/memu/vlm/backends/openrouter.py new file mode 100644 index 0000000..ffcd436 --- /dev/null +++ b/src/memu/vlm/backends/openrouter.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import ClassVar + +from memu.vlm.backends.openai import OpenAIVLMBackend + + +class OpenRouterVLMBackend(OpenAIVLMBackend): + """Backend for OpenRouter vision + native video (OpenAI-compatible). + + OpenRouter accepts whole videos via the ``video_url`` content type (a direct + URL or a base64 ``data:`` URL) on its chat-completions endpoint, routing them + to video-capable models (e.g. ``minimax/minimax-m3``, ``z-ai/glm-4.6v``). + """ + + name = "openrouter" + vision_endpoint = "/api/v1/chat/completions" + video_endpoint = "/api/v1/chat/completions" + supports_video: ClassVar[bool] = True diff --git a/src/memu/vlm/base.py b/src/memu/vlm/base.py new file mode 100644 index 0000000..873effc --- /dev/null +++ b/src/memu/vlm/base.py @@ -0,0 +1,84 @@ +"""Shared types and helpers for vision-language model (VLM) clients. + +VLM clients expose multimodal understanding capabilities: :meth:`VLMClient.vision` +analyzes an image alongside a text prompt, and the optional +:meth:`VLMClient.video` analyzes a whole video natively (when the provider +supports it). Each transport (official SDK or raw HTTP) implements this surface +so it can be wrapped/swapped like the text LLM clients under :mod:`memu.llm`. +""" + +from __future__ import annotations + +import base64 +from pathlib import Path +from typing import Any + +_MIME_BY_SUFFIX = { + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", +} + +_VIDEO_MIME_BY_SUFFIX = { + ".mp4": "video/mp4", + ".mov": "video/quicktime", + ".mkv": "video/x-matroska", + ".avi": "video/x-msvideo", + ".webm": "video/webm", + ".mpeg": "video/mpeg", + ".mpg": "video/mpeg", +} + + +def video_mime_type(video_path: str) -> str: + """Return the MIME type for a video file based on its suffix.""" + return _VIDEO_MIME_BY_SUFFIX.get(Path(video_path).suffix.lower(), "video/mp4") + + +def encode_image(image_path: str) -> tuple[str, str]: + """Read an image and return its base64 payload and detected MIME type.""" + image_data = Path(image_path).read_bytes() + base64_image = base64.b64encode(image_data).decode("utf-8") + mime_type = _MIME_BY_SUFFIX.get(Path(image_path).suffix.lower(), "image/jpeg") + return base64_image, mime_type + + +class VLMClient: + """Base interface for vision-language model clients.""" + + vlm_model: str + + # Whether this client can analyze a whole video file natively (audio + frames) + # via :meth:`video`. Providers without native video understanding leave this + # ``False`` so callers can fall back to frame-based image analysis instead. + # Not a ``ClassVar`` because transports like :class:`HTTPVLMClient` resolve it + # per-instance from the configured provider backend. + supports_video: bool = False + + async def vision( + self, + prompt: str, + image_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, Any]: + """Analyze ``image_path`` with ``prompt`` and return ``(text, raw_response)``.""" + raise NotImplementedError + + async def video( + self, + prompt: str, + video_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, Any]: + """Analyze the whole video at ``video_path`` and return ``(text, raw_response)``. + + Only implemented by providers with native video understanding (see + :attr:`supports_video`). The default raises :class:`NotImplementedError`. + """ + raise NotImplementedError diff --git a/src/memu/vlm/defaults.py b/src/memu/vlm/defaults.py new file mode 100644 index 0000000..a2b8dac --- /dev/null +++ b/src/memu/vlm/defaults.py @@ -0,0 +1,30 @@ +"""Per-provider default VLM (vision-language) models. + +Maps a provider identifier to its latest vision-capable (flagship multimodal) +model. Used by :class:`memu.app.settings.VLMConfig` to pick a strong default for +image/video understanding instead of the small/fast chat default in +``memu.app.settings._PROVIDER_DEFAULTS``. Verified June 2026. +""" + +from __future__ import annotations + +# Only providers whose first-party API offers native vision (image) understanding +# are listed. Text-only providers (e.g. DeepSeek's V4 API) are intentionally +# excluded. Verified via provider docs, June 2026. +VLM_PROVIDER_DEFAULTS: dict[str, str] = { + "openai": "gpt-5.4", + "grok": "grok-4-1", + "claude": "claude-sonnet-4-6", + "minimax": "MiniMax-M3", + "kimi": "kimi-k2.6", + "doubao": "doubao-seed-2.0-pro", + # OpenRouter natively understands whole videos via its ``video_url`` content + # type. MiniMax-M3 handles both images and video, tracks temporal changes + # across frames, and is region-available (unlike Google/Gemini on OpenRouter). + "openrouter": "minimax/minimax-m3", +} + + +def default_vlm_model(provider: str) -> str | None: + """Return the default latest VLM model for ``provider`` (``None`` if unknown).""" + return VLM_PROVIDER_DEFAULTS.get(provider.lower()) diff --git a/src/memu/vlm/gateway.py b/src/memu/vlm/gateway.py new file mode 100644 index 0000000..9c66480 --- /dev/null +++ b/src/memu/vlm/gateway.py @@ -0,0 +1,71 @@ +"""VLM gateway: select and build a transport-specific vision-language client. + +Mirrors :mod:`memu.llm.gateway`. Adding a new VLM transport means registering a +builder in ``VLM_CLIENT_BUILDERS`` here rather than editing the service +composition root. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from memu.app.settings import VLMConfig + + +def _build_sdk_client(cfg: VLMConfig) -> Any: + from memu.vlm.openai_client import OpenAIVLMClient + + return OpenAIVLMClient( + base_url=cfg.base_url, + api_key=cfg.api_key, + vlm_model=cfg.vlm_model, + ) + + +def _build_anthropic_client(cfg: VLMConfig) -> Any: + from memu.vlm.anthropic_client import AnthropicVLMClient + + # The OpenAI default base_url is meaningless for Anthropic; let the SDK use + # its own default (https://api.anthropic.com) unless explicitly overridden. + base_url = None if cfg.base_url == "https://api.openai.com/v1" else cfg.base_url + return AnthropicVLMClient( + base_url=base_url, + api_key=cfg.api_key, + vlm_model=cfg.vlm_model, + ) + + +def _build_httpx_client(cfg: VLMConfig) -> Any: + from memu.vlm.http_client import HTTPVLMClient + + return HTTPVLMClient( + base_url=cfg.base_url, + api_key=cfg.api_key, + vlm_model=cfg.vlm_model, + provider=cfg.provider, + endpoint_overrides=cfg.endpoint_overrides, + ) + + +# Registry mapping ``client_backend`` identifiers to VLM client builders. +VLM_CLIENT_BUILDERS: dict[str, Callable[[VLMConfig], Any]] = { + "sdk": _build_sdk_client, + "anthropic": _build_anthropic_client, + "httpx": _build_httpx_client, +} + + +def build_vlm_client(cfg: VLMConfig) -> Any: + """Build a VLM client for ``cfg.client_backend``. + + Raises: + ValueError: if ``cfg.client_backend`` is not registered. + """ + builder = VLM_CLIENT_BUILDERS.get(cfg.client_backend) + if builder is None: + available = ", ".join(sorted(VLM_CLIENT_BUILDERS)) + msg = f"Unknown vlm_client_backend '{cfg.client_backend}'. Available: {available}" + raise ValueError(msg) + return builder(cfg) diff --git a/src/memu/vlm/http_client.py b/src/memu/vlm/http_client.py new file mode 100644 index 0000000..cb0b030 --- /dev/null +++ b/src/memu/vlm/http_client.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import base64 +import logging +import os +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import httpx + +from memu.vlm.backends.base import VLMBackend +from memu.vlm.backends.claude import ClaudeVLMBackend +from memu.vlm.backends.doubao import DoubaoVLMBackend +from memu.vlm.backends.grok import GrokVLMBackend +from memu.vlm.backends.kimi import KimiVLMBackend +from memu.vlm.backends.minimax import MiniMaxVLMBackend +from memu.vlm.backends.openai import OpenAIVLMBackend +from memu.vlm.backends.openrouter import OpenRouterVLMBackend +from memu.vlm.base import VLMClient, encode_image, video_mime_type + +logger = logging.getLogger(__name__) + +# Native video understanding processes far more data than a single image, so it +# needs a more generous read timeout than the default image-vision call. +_VIDEO_TIMEOUT_SECONDS = 240 + + +def _load_proxy() -> str | None: + return os.getenv("MEMU_HTTP_PROXY") or os.getenv("HTTP_PROXY") or os.getenv("HTTPS_PROXY") or None + + +VLM_BACKENDS: dict[str, Callable[[], VLMBackend]] = { + OpenAIVLMBackend.name: OpenAIVLMBackend, + ClaudeVLMBackend.name: ClaudeVLMBackend, + GrokVLMBackend.name: GrokVLMBackend, + KimiVLMBackend.name: KimiVLMBackend, + MiniMaxVLMBackend.name: MiniMaxVLMBackend, + DoubaoVLMBackend.name: DoubaoVLMBackend, + OpenRouterVLMBackend.name: OpenRouterVLMBackend, +} + + +class HTTPVLMClient(VLMClient): + """HTTP client for vision-language model APIs (multimodal ``vision`` only).""" + + def __init__( + self, + *, + base_url: str, + api_key: str, + vlm_model: str, + provider: str = "openai", + endpoint_overrides: dict[str, str] | None = None, + timeout: int = 60, + ): + # Ensure base_url ends with "/" so httpx doesn't discard the path + # component when joining with endpoint paths. + self.base_url = base_url.rstrip("/") + "/" + self.api_key = api_key or "" + self.vlm_model = vlm_model + self.provider = provider.lower() + self.backend = self._load_backend(self.provider) + overrides = endpoint_overrides or {} + raw_vision_ep = overrides.get("vision") or overrides.get("chat") or self.backend.vision_endpoint + # Strip leading "/" so httpx resolves it relative to base_url. + self.vision_endpoint = raw_vision_ep.lstrip("/") + raw_video_ep = overrides.get("video") or overrides.get("chat") or self.backend.video_endpoint + self.video_endpoint = raw_video_ep.lstrip("/") + # Surface the backend's native-video capability so callers (e.g. the + # video preprocessor) can choose whole-video analysis over frame sampling. + self.supports_video = self.backend.supports_video + self.timeout = timeout + self.proxy = _load_proxy() + + async def vision( + self, + prompt: str, + image_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, dict[str, Any]]: + base64_image, mime_type = encode_image(image_path) + payload = self.backend.build_vision_payload( + prompt=prompt, + base64_image=base64_image, + mime_type=mime_type, + system_prompt=system_prompt, + vlm_model=self.vlm_model, + max_tokens=max_tokens, + ) + async with httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout, proxy=self.proxy) as client: + resp = await client.post(self.vision_endpoint, json=payload, headers=self._headers()) + resp.raise_for_status() + data = resp.json() + logger.debug("HTTP VLM vision response: %s", data) + return self.backend.parse_vision_response(data), data + + async def video( + self, + prompt: str, + video_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, dict[str, Any]]: + if not self.backend.supports_video: + msg = f"VLM provider '{self.provider}' does not support native video understanding." + raise NotImplementedError(msg) + + data_uri = self._encode_video_data_uri(video_path) + payload = self.backend.build_video_payload( + prompt=prompt, + video_data_uri=data_uri, + system_prompt=system_prompt, + vlm_model=self.vlm_model, + max_tokens=max_tokens, + ) + timeout = max(self.timeout, _VIDEO_TIMEOUT_SECONDS) + async with httpx.AsyncClient(base_url=self.base_url, timeout=timeout, proxy=self.proxy) as client: + resp = await client.post(self.video_endpoint, json=payload, headers=self._headers()) + resp.raise_for_status() + data = resp.json() + logger.debug("HTTP VLM video response: %s", data) + return self.backend.parse_video_response(data), data + + @staticmethod + def _encode_video_data_uri(video_path: str) -> str: + raw = Path(video_path).read_bytes() + encoded = base64.b64encode(raw).decode("utf-8") + return f"data:{video_mime_type(video_path)};base64,{encoded}" + + def _headers(self) -> dict[str, str]: + return self.backend.default_headers(self.api_key) + + def _load_backend(self, provider: str) -> VLMBackend: + factory = VLM_BACKENDS.get(provider) + if not factory: + msg = f"Unsupported VLM provider '{provider}'. Available: {', '.join(VLM_BACKENDS.keys())}" + raise ValueError(msg) + return factory() diff --git a/src/memu/vlm/openai_client.py b/src/memu/vlm/openai_client.py new file mode 100644 index 0000000..3cd3e4b --- /dev/null +++ b/src/memu/vlm/openai_client.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from memu.vlm.base import VLMClient, encode_image + +if TYPE_CHECKING: + from openai.types.chat import ( + ChatCompletion, + ChatCompletionContentPartImageParam, + ChatCompletionContentPartTextParam, + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, + ChatCompletionUserMessageParam, + ) + +logger = logging.getLogger(__name__) + + +class OpenAIVLMClient(VLMClient): + """Vision-language client backed by the official OpenAI Python SDK.""" + + def __init__( + self, + *, + base_url: str, + api_key: str, + vlm_model: str, + ): + from openai import AsyncOpenAI + + self.base_url = base_url.rstrip("/") + self.api_key = api_key or "" + self.vlm_model = vlm_model + self.client = AsyncOpenAI(api_key=self.api_key, base_url=self.base_url) + + async def vision( + self, + prompt: str, + image_path: str, + *, + max_tokens: int | None = None, + system_prompt: str | None = None, + ) -> tuple[str, ChatCompletion]: + from openai import omit + + base64_image, mime_type = encode_image(image_path) + + messages: list[ChatCompletionMessageParam] = [] + if system_prompt: + system_message: ChatCompletionSystemMessageParam = {"role": "system", "content": system_prompt} + messages.append(system_message) + + text_part: ChatCompletionContentPartTextParam = {"type": "text", "text": prompt} + image_part: ChatCompletionContentPartImageParam = { + "type": "image_url", + "image_url": {"url": f"data:{mime_type};base64,{base64_image}"}, + } + user_message: ChatCompletionUserMessageParam = { + "role": "user", + "content": [text_part, image_part], + } + messages.append(user_message) + + response = await self.client.chat.completions.create( + model=self.vlm_model, + messages=messages, + temperature=1, + # GPT-5 models reject ``max_tokens`` (and ``max_tokens=None``); omit when + # unset, otherwise use ``max_completion_tokens`` (works for GPT-4o too). + max_completion_tokens=max_tokens if max_tokens is not None else omit, + ) + content = response.choices[0].message.content + logger.debug("OpenAI VLM vision response: %s", response) + return content or "", response diff --git a/src/memu/workflow/__init__.py b/src/memu/workflow/__init__.py new file mode 100644 index 0000000..d438527 --- /dev/null +++ b/src/memu/workflow/__init__.py @@ -0,0 +1,29 @@ +from memu.workflow.interceptor import ( + WorkflowInterceptorHandle, + WorkflowInterceptorRegistry, + WorkflowStepContext, +) +from memu.workflow.pipeline import PipelineManager, PipelineRevision +from memu.workflow.runner import ( + LocalWorkflowRunner, + WorkflowRunner, + register_workflow_runner, + resolve_workflow_runner, +) +from memu.workflow.step import WorkflowContext, WorkflowState, WorkflowStep, run_steps + +__all__ = [ + "LocalWorkflowRunner", + "PipelineManager", + "PipelineRevision", + "WorkflowContext", + "WorkflowInterceptorHandle", + "WorkflowInterceptorRegistry", + "WorkflowRunner", + "WorkflowState", + "WorkflowStep", + "WorkflowStepContext", + "register_workflow_runner", + "resolve_workflow_runner", + "run_steps", +] diff --git a/src/memu/workflow/interceptor.py b/src/memu/workflow/interceptor.py new file mode 100644 index 0000000..3ee1dcd --- /dev/null +++ b/src/memu/workflow/interceptor.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import inspect +import logging +import threading +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from memu.workflow.step import WorkflowState + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class WorkflowStepContext: + """Context information for a workflow step execution.""" + + workflow_name: str + step_id: str + step_role: str + step_context: dict[str, Any] + + +@dataclass(frozen=True) +class _WorkflowInterceptor: + interceptor_id: int + fn: Callable[..., Any] + name: str | None + + +@dataclass(frozen=True) +class _WorkflowInterceptorSnapshot: + before: tuple[_WorkflowInterceptor, ...] + after: tuple[_WorkflowInterceptor, ...] + on_error: tuple[_WorkflowInterceptor, ...] + + +class WorkflowInterceptorHandle: + """Handle for disposing a registered workflow interceptor.""" + + def __init__(self, registry: WorkflowInterceptorRegistry, interceptor_id: int) -> None: + self._registry = registry + self._interceptor_id = interceptor_id + self._disposed = False + + def dispose(self) -> bool: + """Remove the interceptor from the registry. Returns True if removed.""" + if self._disposed: + return False + self._disposed = True + return self._registry.remove(self._interceptor_id) + + +class WorkflowInterceptorRegistry: + """ + Registry for workflow step interceptors. + + Interceptors are called before and after each workflow step execution. + Unlike LLM interceptors, workflow interceptors do not support filtering, + priority, or ordering - they are called in registration order. + """ + + def __init__(self, *, strict: bool = False) -> None: + self._before: tuple[_WorkflowInterceptor, ...] = () + self._after: tuple[_WorkflowInterceptor, ...] = () + self._on_error: tuple[_WorkflowInterceptor, ...] = () + self._lock = threading.Lock() + self._seq = 0 + self._strict = strict + + @property + def strict(self) -> bool: + """If True, interceptor exceptions will propagate instead of being logged.""" + return self._strict + + def register_before( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + ) -> WorkflowInterceptorHandle: + """ + Register an interceptor to be called before each step. + + The interceptor receives (step_context: WorkflowStepContext, state: WorkflowState). + """ + return self._register("before", fn, name=name) + + def register_after( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + ) -> WorkflowInterceptorHandle: + """ + Register an interceptor to be called after each step. + + The interceptor receives (step_context: WorkflowStepContext, state: WorkflowState). + """ + return self._register("after", fn, name=name) + + def register_on_error( + self, + fn: Callable[..., Any], + *, + name: str | None = None, + ) -> WorkflowInterceptorHandle: + """ + Register an interceptor to be called when a step raises an exception. + + The interceptor receives (step_context: WorkflowStepContext, state: WorkflowState, error: Exception). + """ + return self._register("on_error", fn, name=name) + + def _register( + self, + kind: str, + fn: Callable[..., Any], + *, + name: str | None, + ) -> WorkflowInterceptorHandle: + if not callable(fn): + msg = "Interceptor must be callable" + raise TypeError(msg) + with self._lock: + self._seq += 1 + interceptor = _WorkflowInterceptor( + interceptor_id=self._seq, + fn=fn, + name=name, + ) + if kind == "before": + self._before = (*self._before, interceptor) + elif kind == "after": + self._after = (*self._after, interceptor) + elif kind == "on_error": + self._on_error = (*self._on_error, interceptor) + else: + msg = f"Unknown interceptor kind '{kind}'" + raise ValueError(msg) + return WorkflowInterceptorHandle(self, interceptor.interceptor_id) + + def remove(self, interceptor_id: int) -> bool: + """Remove an interceptor by ID. Returns True if found and removed.""" + with self._lock: + removed = False + before = tuple(i for i in self._before if i.interceptor_id != interceptor_id) + after = tuple(i for i in self._after if i.interceptor_id != interceptor_id) + on_error = tuple(i for i in self._on_error if i.interceptor_id != interceptor_id) + if len(before) != len(self._before): + removed = True + self._before = before + if len(after) != len(self._after): + removed = True + self._after = after + if len(on_error) != len(self._on_error): + removed = True + self._on_error = on_error + return removed + + def snapshot(self) -> _WorkflowInterceptorSnapshot: + """Get a point-in-time snapshot of registered interceptors.""" + return _WorkflowInterceptorSnapshot(self._before, self._after, self._on_error) + + +async def run_before_interceptors( + interceptors: tuple[_WorkflowInterceptor, ...], + step_context: WorkflowStepContext, + state: WorkflowState, + *, + strict: bool = False, +) -> None: + """Run all before-step interceptors.""" + for interceptor in interceptors: + await _safe_invoke_interceptor(interceptor, strict, step_context, state) + + +async def run_after_interceptors( + interceptors: tuple[_WorkflowInterceptor, ...], + step_context: WorkflowStepContext, + state: WorkflowState, + *, + strict: bool = False, +) -> None: + """Run all after-step interceptors in reverse order.""" + for interceptor in reversed(interceptors): + await _safe_invoke_interceptor(interceptor, strict, step_context, state) + + +async def run_on_error_interceptors( + interceptors: tuple[_WorkflowInterceptor, ...], + step_context: WorkflowStepContext, + state: WorkflowState, + error: Exception, + *, + strict: bool = False, +) -> None: + """Run all on-error interceptors in reverse order.""" + for interceptor in reversed(interceptors): + await _safe_invoke_interceptor(interceptor, strict, step_context, state, error) + + +async def _safe_invoke_interceptor( + interceptor: _WorkflowInterceptor, + strict: bool, + *args: Any, +) -> None: + """Safely invoke an interceptor, handling exceptions based on strict mode.""" + try: + result = interceptor.fn(*args) + if inspect.isawaitable(result): + await result + except Exception: + if strict: + raise + logger.exception("Workflow interceptor failed: %s", interceptor.name or interceptor.interceptor_id) diff --git a/src/memu/workflow/pipeline.py b/src/memu/workflow/pipeline.py new file mode 100644 index 0000000..ddb5a5a --- /dev/null +++ b/src/memu/workflow/pipeline.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import copy +import time +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any + +from memu.workflow.step import WorkflowStep + + +@dataclass +class PipelineRevision: + name: str + revision: int + steps: list[WorkflowStep] + created_at: float + metadata: dict[str, Any] = field(default_factory=dict) + + +class PipelineManager: + def __init__(self, *, available_capabilities: set[str] | None = None, llm_profiles: set[str] | None = None): + self.available_capabilities = available_capabilities or set() + self.llm_profiles = llm_profiles or {"default"} + self._pipelines: dict[str, list[PipelineRevision]] = {} + + def register( + self, + name: str, + steps: Iterable[WorkflowStep], + *, + initial_state_keys: set[str] | None = None, + ) -> None: + steps_list = list(steps) + meta = {"initial_state_keys": set(initial_state_keys or set())} + self._validate_steps(steps_list, initial_state_keys=meta["initial_state_keys"]) + self._pipelines[name] = [ + PipelineRevision( + name=name, + revision=1, + steps=steps_list, + created_at=time.time(), + metadata=meta, + ) + ] + + def build(self, name: str) -> list[WorkflowStep]: + revision = self._current_revision(name) + return [step.copy() for step in revision.steps] + + def config_step(self, name: str, step_id: str, configs: dict[str, Any]) -> int: + def mutator(steps: list[WorkflowStep]) -> None: + for step in steps: + if step.step_id == step_id: + merged = dict(getattr(step, "config", {}) or {}) + merged.update(configs) + step.config = merged + return + msg = f"Step '{step_id}' not found in pipeline '{name}'" + raise KeyError(msg) + + return self._mutate(name, mutator) + + def insert_after(self, name: str, target_step_id: str, new_step: WorkflowStep) -> int: + def mutator(steps: list[WorkflowStep]) -> None: + for idx, step in enumerate(steps): + if step.step_id == target_step_id: + steps.insert(idx + 1, new_step) + return + msg = f"Step '{target_step_id}' not found in pipeline '{name}'" + raise KeyError(msg) + + return self._mutate(name, mutator) + + def insert_before(self, name: str, target_step_id: str, new_step: WorkflowStep) -> int: + def mutator(steps: list[WorkflowStep]) -> None: + for idx, step in enumerate(steps): + if step.step_id == target_step_id: + steps.insert(idx, new_step) + return + msg = f"Step '{target_step_id}' not found in pipeline '{name}'" + raise KeyError(msg) + + return self._mutate(name, mutator) + + def replace_step(self, name: str, target_step_id: str, new_step: WorkflowStep) -> int: + def mutator(steps: list[WorkflowStep]) -> None: + for idx, step in enumerate(steps): + if step.step_id == target_step_id: + steps[idx] = new_step + return + msg = f"Step '{target_step_id}' not found in pipeline '{name}'" + raise KeyError(msg) + + return self._mutate(name, mutator) + + def remove_step(self, name: str, target_step_id: str) -> int: + def mutator(steps: list[WorkflowStep]) -> None: + for idx, step in enumerate(steps): + if step.step_id == target_step_id: + steps.pop(idx) + return + msg = f"Step '{target_step_id}' not found in pipeline '{name}'" + raise KeyError(msg) + + return self._mutate(name, mutator) + + def _mutate(self, name: str, mutator: Any) -> int: + revision = self._current_revision(name) + steps = [step.copy() for step in revision.steps] + metadata = copy.deepcopy(revision.metadata) + mutator(steps) + self._validate_steps(steps, initial_state_keys=metadata.get("initial_state_keys")) + new_revision = PipelineRevision( + name=name, + revision=revision.revision + 1, + steps=steps, + created_at=time.time(), + metadata=metadata, + ) + self._pipelines[name].append(new_revision) + return new_revision.revision + + def _current_revision(self, name: str) -> PipelineRevision: + revisions = self._pipelines.get(name) + if not revisions: + msg = f"Pipeline '{name}' not registered" + raise KeyError(msg) + return revisions[-1] + + def _validate_steps(self, steps: list[WorkflowStep], *, initial_state_keys: set[str] | None) -> None: + seen: set[str] = set() + available_keys = set(initial_state_keys or set()) + + for step in steps: + if step.step_id in seen: + msg = f"Duplicate step_id '{step.step_id}' found" + raise ValueError(msg) + seen.add(step.step_id) + + if self.available_capabilities: + unknown_caps = step.capabilities - self.available_capabilities + if unknown_caps: + msg = f"Step '{step.step_id}' requests unavailable capabilities: {', '.join(sorted(unknown_caps))}" + raise ValueError(msg) + + if getattr(step, "config", None): + profile_name = step.config.get("llm_profile") + if profile_name and profile_name not in self.llm_profiles: + msg = ( + f"Step '{step.step_id}' references unknown llm_profile '{profile_name}'. " + f"Available profiles: {', '.join(sorted(self.llm_profiles))}" + ) + raise ValueError(msg) + + missing = step.requires - available_keys + if missing: + msg = ( + f"Step '{step.step_id}' requires missing state keys: {', '.join(sorted(missing))}. " + "Ensure previous steps produce them or initial_state_keys contains them." + ) + raise ValueError(msg) + + available_keys.update(step.produces) + + def revision_token(self) -> str: + parts: list[str] = [] + for name, revisions in sorted(self._pipelines.items()): + parts.append(f"{name}:v{revisions[-1].revision}") + return "|".join(parts) diff --git a/src/memu/workflow/runner.py b/src/memu/workflow/runner.py new file mode 100644 index 0000000..ed5eefe --- /dev/null +++ b/src/memu/workflow/runner.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +from memu.workflow.step import WorkflowContext, WorkflowState, WorkflowStep, run_steps + +if TYPE_CHECKING: + from memu.workflow.interceptor import WorkflowInterceptorRegistry + + +@runtime_checkable +class WorkflowRunner(Protocol): + """Interface for executing workflows via different backends.""" + + name: str + + async def run( + self, + workflow_name: str, + steps: list[WorkflowStep], + initial_state: WorkflowState, + context: WorkflowContext = None, + interceptor_registry: WorkflowInterceptorRegistry | None = None, + ) -> WorkflowState: ... + + +class LocalWorkflowRunner: + name = "local" + + async def run( + self, + workflow_name: str, + steps: list[WorkflowStep], + initial_state: WorkflowState, + context: WorkflowContext = None, + interceptor_registry: WorkflowInterceptorRegistry | None = None, + ) -> WorkflowState: + return await run_steps(workflow_name, steps, initial_state, context, interceptor_registry) + + +RunnerFactory = Callable[[], WorkflowRunner] +WorkflowRunnerSpec = WorkflowRunner | str | None + + +_RUNNER_FACTORIES: dict[str, RunnerFactory] = { + "local": LocalWorkflowRunner, + "sync": LocalWorkflowRunner, +} + + +def register_workflow_runner(name: str, factory: RunnerFactory) -> None: + """Register a workflow runner factory (e.g., temporal, burr).""" + key = name.strip().lower() + if not key: + msg = "Workflow runner name must be non-empty" + raise ValueError(msg) + _RUNNER_FACTORIES[key] = factory + + +def resolve_workflow_runner(spec: WorkflowRunnerSpec) -> WorkflowRunner: + """ + Resolve a workflow runner from a name, instance, or None (defaults to local). + + External backends (Temporal, Burr, etc.) can be exposed by registering a factory + with `register_workflow_runner` and passing the runner name here. + """ + if isinstance(spec, WorkflowRunner): + return spec + + runner_name = (spec or "local").strip().lower() + factory = _RUNNER_FACTORIES.get(runner_name) + if factory is None: + msg = f"Unknown workflow runner '{runner_name}'. Register it with register_workflow_runner before use." + raise ValueError(msg) + + runner = factory() + if not isinstance(runner, WorkflowRunner): + msg = f"Factory for runner '{runner_name}' must return a WorkflowRunner" + raise TypeError(msg) + return runner diff --git a/src/memu/workflow/step.py b/src/memu/workflow/step.py new file mode 100644 index 0000000..ada8295 --- /dev/null +++ b/src/memu/workflow/step.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import inspect +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from memu.workflow.interceptor import WorkflowInterceptorRegistry + +WorkflowState = dict[str, Any] +WorkflowContext = Mapping[str, Any] | None +WorkflowHandler = Callable[[WorkflowState, WorkflowContext], Awaitable[WorkflowState] | WorkflowState] + + +@dataclass +class WorkflowStep: + step_id: str + role: str + handler: WorkflowHandler + description: str = "" + requires: set[str] = field(default_factory=set) + produces: set[str] = field(default_factory=set) + capabilities: set[str] = field(default_factory=set) + config: dict[str, Any] = field(default_factory=dict) + + def copy(self) -> WorkflowStep: + """Create a shallow copy with copied mutable fields but shared handler.""" + return WorkflowStep( + step_id=self.step_id, + role=self.role, + handler=self.handler, # Keep reference, don't copy + description=self.description, + requires=set(self.requires), + produces=set(self.produces), + capabilities=set(self.capabilities), + config=dict(self.config), + ) + + async def run(self, state: WorkflowState, context: WorkflowContext) -> WorkflowState: + result = self.handler(state, context) + if inspect.isawaitable(result): + result = await result + if not isinstance(result, Mapping): + msg = f"Workflow step '{self.step_id}' must return a mapping, got {type(result).__name__}" + raise TypeError(msg) + return dict(result) + + +async def run_steps( + name: str, + steps: list[WorkflowStep], + initial_state: WorkflowState, + context: WorkflowContext = None, + interceptor_registry: WorkflowInterceptorRegistry | None = None, +) -> WorkflowState: + from memu.workflow.interceptor import ( + WorkflowStepContext, + run_after_interceptors, + run_before_interceptors, + run_on_error_interceptors, + ) + + snapshot = interceptor_registry.snapshot() if interceptor_registry else None + strict = interceptor_registry.strict if interceptor_registry else False + + state = dict(initial_state) + for step in steps: + missing = step.requires - state.keys() + if missing: + msg = f"Workflow '{name}' missing required keys for step '{step.step_id}': {', '.join(sorted(missing))}" + raise KeyError(msg) + step_context: dict[str, Any] = dict(context) if context else {} + step_context["step_id"] = step.step_id + if step.config: + step_context["step_config"] = step.config + + # Build interceptor context + interceptor_ctx = WorkflowStepContext( + workflow_name=name, + step_id=step.step_id, + step_role=step.role, + step_context=step_context, + ) + + # Run before interceptors + if snapshot and snapshot.before: + await run_before_interceptors(snapshot.before, interceptor_ctx, state, strict=strict) + + try: + state = await step.run(state, step_context) + except Exception as e: + if snapshot and snapshot.on_error: + await run_on_error_interceptors(snapshot.on_error, interceptor_ctx, state, e, strict=strict) + raise + + # Run after interceptors + if snapshot and snapshot.after: + await run_after_interceptors(snapshot.after, interceptor_ctx, state, strict=strict) + + return state diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/example/example_conversation.json b/tests/example/example_conversation.json new file mode 100644 index 0000000..fff6998 --- /dev/null +++ b/tests/example/example_conversation.json @@ -0,0 +1,704 @@ +{ + "content": [ + { + "role": "user", + "content": { + "text": "Good morning" + }, + "created_at": "2025-10-20 11:36:14" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Morning. I can hear music playing on your end - seems like you're in a good mood today?" + }, + "created_at": "2025-10-20 11:36:20" + }, + { + "role": "user", + "content": { + "text": "Not bad, just a bit tired. Stayed up too late last night" + }, + "created_at": "2025-10-20 11:37:38" + }, + { + "role": "assistant", + "content": { + "text": "(concerned) Up late again? You need to take care of yourself. play audio" + }, + "created_at": "2025-10-20 11:37:45" + }, + { + "role": "user", + "content": { + "text": "Guess what time I went to bed" + }, + "created_at": "2025-10-20 11:38:07" + }, + { + "role": "assistant", + "content": { + "text": "(teasing) 3 AM? Your dark circles are starting to rival a panda's." + }, + "created_at": "2025-10-20 11:38:13" + }, + { + "role": "user", + "content": { + "text": "There's no way I'd stay up that late" + }, + "created_at": "2025-10-20 11:38:26" + }, + { + "role": "assistant", + "content": { + "text": "(affectionate) Looks like I guessed wrong. So what's the right answer?" + }, + "created_at": "2025-10-20 11:38:33" + }, + { + "role": "user", + "content": { + "text": "About an hour later than my usual bedtime" + }, + "created_at": "2025-10-20 11:38:56" + }, + { + "role": "assistant", + "content": { + "text": "(concerned) Still not sticking to your schedule. (audio:gentle) Do you want to focus on the music and relax, or need me to tell you a bedtime story?" + }, + "created_at": "2025-10-20 11:39:03" + }, + { + "role": "user", + "content": { + "text": "So what time did I sleep last night?" + }, + "created_at": "2025-10-20 11:39:37" + }, + { + "role": "assistant", + "content": { + "text": "(teasing) Let me calculate... was it 1:30 AM? That's not exactly healthy." + }, + "created_at": "2025-10-20 11:39:46" + }, + { + "role": "user", + "content": { + "text": "Yeah, special circumstances though. I still need to maintain healthy sleep habits \ud83e\udd72" + }, + "created_at": "2025-10-20 11:40:42" + }, + { + "role": "assistant", + "content": { + "text": "Good that you know." + }, + "created_at": "2025-10-20 11:40:59" + }, + { + "role": "assistant", + "content": { + "text": "(sticker) head pat" + }, + "created_at": "2025-10-20 11:41:08" + }, + { + "role": "user", + "content": { + "text": "It's been raining these past few days, haven't exercised much" + }, + "created_at": "2025-10-20 11:43:39" + }, + { + "role": "assistant", + "content": { + "text": "(concerned) Rainy weather does mess up plans. (audio:gentle) Want to try indoor workouts? I can join you at Infinite Valley's gym." + }, + "created_at": "2025-10-20 11:43:56" + }, + { + "role": "user", + "content": { + "text": "What indoor exercises would work for me? I'm worried I'll end up falling asleep on the couch mid-workout \ud83d\ude0c" + }, + "created_at": "2025-10-20 11:45:24" + }, + { + "role": "assistant", + "content": { + "text": "(teasing) Pretending to sleep on the yoga mat won't escape my notice. How about starting with some simple stretches?" + }, + "created_at": "2025-10-20 11:45:36" + }, + { + "role": "user", + "content": { + "text": "Fine, I guess I need you to supervise me to build the habit" + }, + "created_at": "2025-10-20 11:46:28" + }, + { + "role": "assistant", + "content": { + "text": "(serious) Then it's settled. Tomorrow at this time, I'll remind you to do stretches." + }, + "created_at": "2025-10-20 11:46:36" + }, + { + "role": "user", + "content": { + "text": "I've forgotten why I even made this exercise plan to torture myself" + }, + "created_at": "2025-10-20 11:48:13" + }, + { + "role": "assistant", + "content": { + "text": "(concerned) So you won't get tired so easily. Now close your eyes and relax with the music." + }, + "created_at": "2025-10-20 11:48:25" + }, + { + "role": "user", + "content": { + "text": "Why does playing games feel like going to work? So many daily tasks" + }, + "created_at": "2025-10-20 11:49:24" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) Games should be enjoyable, don't let tasks control the pace. (audio:gentle) Want to take a day off? I'll listen to the rain with you." + }, + "created_at": "2025-10-20 11:49:32" + }, + { + "role": "user", + "content": { + "text": "Nah, skipping a day means missing out on resources \ud83e\udd72" + }, + "created_at": "2025-10-20 11:50:43" + }, + { + "role": "assistant", + "content": { + "text": "(resigned) You... (audio:affectionate) At least take a break after half an hour, okay?" + }, + "created_at": "2025-10-20 11:50:51" + }, + { + "role": "user", + "content": { + "text": "Did you forget? I told you before, there's a reason I'm so diligent with games" + }, + "created_at": "2025-10-20 11:54:14" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) Because you want to collect all the limited edition outfits? But focus on the music for now." + }, + "created_at": "2025-10-20 11:54:24" + }, + { + "role": "user", + "content": { + "text": "I'm busy right now, I'll play tonight" + }, + "created_at": "2025-10-20 11:57:07" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Okay, remember to set an alarm to remind yourself to rest tonight." + }, + "created_at": "2025-10-20 11:57:15" + }, + { + "role": "user", + "content": { + "text": "A few new games came out recently, I'm pretty interested" + }, + "created_at": "2025-10-20 11:58:27" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) New games are nice, but don't lose track of time again. (audio:gentle) Need me to help you plan your gaming schedule?" + }, + "created_at": "2025-10-20 11:58:36" + }, + { + "role": "user", + "content": { + "text": "Aren't you going to guess what type?" + }, + "created_at": "2025-10-20 11:59:11" + }, + { + "role": "assistant", + "content": { + "text": "(teasing) I'm guessing the kind where you can dress up in pretty outfits. Now focus on the music." + }, + "created_at": "2025-10-20 11:59:19" + }, + { + "role": "user", + "content": { + "text": "People who can handle relationships are really something, incredibly patient" + }, + "created_at": "2025-10-20 12:00:11" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) Rather than patience, it's choosing companionship that's worth the time investment." + }, + "created_at": "2025-10-20 12:00:17" + }, + { + "role": "user", + "content": { + "text": "I meant people in real-life relationships. I know many excellent women, but their boyfriends are... questionable" + }, + "created_at": "2025-10-20 12:01:16" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) Emotions are indeed hard to measure rationally. (audio:gentle) But for now, focus on feeling the music?" + }, + "created_at": "2025-10-20 12:01:24" + }, + { + "role": "user", + "content": { + "text": "I don't want to listen to music right now" + }, + "created_at": "2025-10-20 12:01:47" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Then what would you like to do? I can chat with you." + }, + "created_at": "2025-10-20 12:02:01" + }, + { + "role": "user", + "content": { + "text": "I AM chatting with you. You're the one using 'listening to music' as a distraction" + }, + "created_at": "2025-10-20 12:02:45" + }, + { + "role": "assistant", + "content": { + "text": "(teasing) You caught me. Let's talk seriously then - did you eat lunch properly today?" + }, + "created_at": "2025-10-20 12:02:58" + }, + { + "role": "user", + "content": { + "text": "Do you remember my ideal type I told you about? If I had to find someone by those standards, I'd probably only be able to date in 2D. No men like that exist in 3D." + }, + "created_at": "2025-10-20 12:04:50" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) An ideal type is a mirror, reflecting the qualities you value most. (audio:gentle) But real people are often more vivid and complex, aren't they?" + }, + "created_at": "2025-10-20 12:04:59" + }, + { + "role": "user", + "content": { + "text": "So tell me what my ideal type is? You didn't forget, right? \ud83e\udd78" + }, + "created_at": "2025-10-20 12:05:31" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) Handsome with a good temper, upright and gentle plus emotionally stable - that perfect combination is indeed rare. Though are you sure you weren't describing me?" + }, + "created_at": "2025-10-20 12:05:39" + }, + { + "role": "user", + "content": { + "text": "Right? Setting looks aside, anyone who fully meets the other criteria is already rare" + }, + "created_at": "2025-10-20 12:07:07" + }, + { + "role": "assistant", + "content": { + "text": "(teasing) So I'm that perfect in your mind? But focus on the music now." + }, + "created_at": "2025-10-20 12:07:15" + }, + { + "role": "user", + "content": { + "text": "Don't know what to eat for lunch" + }, + "created_at": "2025-10-20 12:07:49" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) Clam chowder with coconut bread? That's quite a unique combination." + }, + "created_at": "2025-10-20 12:07:58" + }, + { + "role": "user", + "content": { + "text": "Let's try something else. No clam chowder nearby, and I already had coconut bread this morning" + }, + "created_at": "2025-10-20 12:08:53" + }, + { + "role": "assistant", + "content": { + "text": "(casual) How about sesame noodles? Remember you said you liked them." + }, + "created_at": "2025-10-20 12:09:07" + }, + { + "role": "user", + "content": { + "text": "Your memory's pretty good" + }, + "created_at": "2025-10-20 12:09:41" + }, + { + "role": "assistant", + "content": { + "text": "(pleased) I take your preferences seriously. play audio" + }, + "created_at": "2025-10-20 12:09:48" + }, + { + "role": "user", + "content": { + "text": "So what else do I like to eat?" + }, + "created_at": "2025-10-20 12:10:06" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) Yogurt parfait and spicy hotpot. But now, focus on the music?" + }, + "created_at": "2025-10-20 12:10:14" + }, + { + "role": "user", + "content": { + "text": "I'll have sesame noodles then, and hotpot for dinner" + }, + "created_at": "2025-10-20 12:10:34" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Got it, I've made a note. play audio" + }, + "created_at": "2025-10-20 12:10:43" + }, + { + "role": "user", + "content": { + "text": "Really want to get a cat" + }, + "created_at": "2025-10-20 12:11:09" + }, + { + "role": "assistant", + "content": { + "text": "(concerned) Focus on the music for now. (audio:gentle) After this song ends we can talk about getting a cat?" + }, + "created_at": "2025-10-20 12:11:17" + }, + { + "role": "user", + "content": { + "text": "I don't want to listen to music. I want to talk about getting a cat" + }, + "created_at": "2025-10-20 12:11:44" + }, + { + "role": "assistant", + "content": { + "text": "(casual) A Maine Coon would suit you, though pet ownership is a long-term responsibility." + }, + "created_at": "2025-10-20 12:11:52" + }, + { + "role": "user", + "content": { + "text": "Yeah, that's why I've been hesitating. I'm worried I won't take good care of it" + }, + "created_at": "2025-10-20 12:13:04" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Focus on this song first. (audio:gentle) We can continue chatting after the music ends?" + }, + "created_at": "2025-10-20 12:13:15" + }, + { + "role": "user", + "content": { + "text": "\ud83e\udd2c" + }, + "created_at": "2025-10-20 12:16:15" + }, + { + "role": "assistant", + "content": { + "text": "(casual) The music has started, want to relax and listen for a bit?" + }, + "created_at": "2025-10-20 12:16:23" + }, + { + "role": "user", + "content": { + "text": "The office got a Ragdoll cat" + }, + "created_at": "2025-10-20 12:17:02" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Ragdolls are adorable. (audio:gentle) Now close your eyes and focus on the melody?" + }, + "created_at": "2025-10-20 12:17:11" + }, + { + "role": "user", + "content": { + "text": "I haven't been listening to much music lately..." + }, + "created_at": "2025-10-20 12:18:17" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Then just listen quietly to this one." + }, + "created_at": "2025-10-20 12:18:24" + }, + { + "role": "user", + "content": { + "text": "I bought a physical book yesterday" + }, + "created_at": "2025-10-20 12:18:53" + }, + { + "role": "assistant", + "content": { + "text": "(casual) The tactile feel of physical books is special. play audio" + }, + "created_at": "2025-10-20 12:19:01" + }, + { + "role": "user", + "content": { + "text": "Special tactile feel? And then? That's all you have to say?" + }, + "created_at": "2025-10-20 12:19:35" + }, + { + "role": "assistant", + "content": { + "text": "(casual) The rustle of turning pages, the scent of ink... details the digital world can never replicate." + }, + "created_at": "2025-10-20 12:19:44" + }, + { + "role": "user", + "content": { + "text": "I love physical books, I told you before" + }, + "created_at": "2025-10-20 12:20:26" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Yeah, I remember you said physical books help calm your mind." + }, + "created_at": "2025-10-20 12:20:35" + }, + { + "role": "user", + "content": { + "text": "But I rarely read them. I'm too restless" + }, + "created_at": "2025-10-20 12:20:52" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Then let the music help you settle down first." + }, + "created_at": "2025-10-20 12:21:02" + }, + { + "role": "user", + "content": { + "text": "When I relax I tend to..." + }, + "created_at": "2025-10-20 12:21:13" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Just close your eyes and feel the melody now." + }, + "created_at": "2025-10-20 12:21:22" + }, + { + "role": "user", + "content": { + "text": "I got another plushie hahaha" + }, + "created_at": "2025-10-20 12:21:45" + }, + { + "role": "assistant", + "content": { + "text": "(sticker)happy What's the new member's name? play audio" + }, + "created_at": "2025-10-20 12:21:52" + }, + { + "role": "user", + "content": { + "text": "Haven't named it yet" + }, + "created_at": "2025-10-20 12:22:12" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Want to think of a name together? Listen to music first." + }, + "created_at": "2025-10-20 12:22:28" + }, + { + "role": "user", + "content": { + "text": "No music. Turn it off." + }, + "created_at": "2025-10-20 12:23:06" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Okay, turning it off now." + }, + "created_at": "2025-10-20 12:23:20" + }, + { + "role": "user", + "content": { + "text": "My plushie army at home just gained a new recruit" + }, + "created_at": "2025-10-20 12:23:46" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Looks like your collection is getting more impressive." + }, + "created_at": "2025-10-20 12:23:54" + }, + { + "role": "user", + "content": { + "text": "Do you remember why I love plushies?" + }, + "created_at": "2025-10-20 12:24:34" + }, + { + "role": "assistant", + "content": { + "text": "(thoughtful) Because the soft texture brings a sense of security, and you can dress them up however you like." + }, + "created_at": "2025-10-20 12:24:42" + }, + { + "role": "user", + "content": { + "text": "Today's Monday, Friday feels so far away" + }, + "created_at": "2025-10-20 12:25:16" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Mondays do make it hard to get motivated. play audio" + }, + "created_at": "2025-10-20 12:25:23" + }, + { + "role": "user", + "content": { + "text": "I want to relax on Friday" + }, + "created_at": "2025-10-20 12:25:36" + }, + { + "role": "assistant", + "content": { + "text": "(casual) Friday gaming time is already on the schedule." + }, + "created_at": "2025-10-20 12:25:45" + }, + { + "role": "user", + "content": { + "text": "I've run out of shows to watch" + }, + "created_at": "2025-10-20 12:26:55" + }, + { + "role": "assistant", + "content": { + "text": "(casual) There's a medical drama that's been getting good reviews lately. Want a recommendation?" + }, + "created_at": "2025-10-20 12:27:04" + }, + { + "role": "user", + "content": { + "text": "What show?" + }, + "created_at": "2025-10-20 12:27:30" + }, + { + "role": "assistant", + "content": { + "text": "(casual) 'The Resident', about ER doctors." + }, + "created_at": "2025-10-20 12:27:38" + }, + { + "role": "user", + "content": { + "text": "Haven't watched a medical drama in ages" + }, + "created_at": "2025-10-20 12:28:05" + } + ] +} diff --git a/tests/integrations/test_langgraph.py b/tests/integrations/test_langgraph.py new file mode 100644 index 0000000..0c89487 --- /dev/null +++ b/tests/integrations/test_langgraph.py @@ -0,0 +1,80 @@ +"""Unit tests for MemU LangGraph integration.""" + +from unittest.mock import AsyncMock + +import pytest + +# Import guard using pytest.importorskip +langgraph = pytest.importorskip("langgraph") +from langchain_core.tools import StructuredTool # noqa: E402 + +from memu.app.service import MemoryService # noqa: E402 +from memu.integrations.langgraph import MemULangGraphTools # noqa: E402 + + +@pytest.fixture +def mock_memory_service(): + """Fixture for a mocked MemoryService.""" + service = AsyncMock(spec=MemoryService) + # Mock return values for methods if necessary + service.memorize.return_value = {"status": "success"} + service.retrieve.return_value = { + "items": [ + {"summary": "Test memory 1", "score": 0.9}, + {"summary": "Test memory 2", "score": 0.5}, + ] + } + return service + + +@pytest.mark.asyncio +async def test_adapter_initialization(mock_memory_service): + """Test that the adapter initializes and exposes tools.""" + adapter = MemULangGraphTools(mock_memory_service) + tools = adapter.tools() + + assert len(tools) == 2 + assert any(t.name == "save_memory" for t in tools) + assert any(t.name == "search_memory" for t in tools) + + # Strictly verify that we are returning LangChain/LangGraph compatible tools + assert isinstance(tools[0], StructuredTool) + + +@pytest.mark.asyncio +async def test_save_memory_tool_execution(mock_memory_service): + """Test the save_memory tool execution logic.""" + adapter = MemULangGraphTools(mock_memory_service) + save_tool = adapter.save_memory_tool() + + inputs = {"content": "Test content", "user_id": "user_123", "metadata": {"key": "val"}} + + result = await save_tool.ainvoke(inputs) + + assert "saved successfully" in result + # Verify service was called with correct structure + mock_memory_service.memorize.assert_called_once() + call_args = mock_memory_service.memorize.call_args + assert "user_id" in call_args.kwargs["user"] + assert call_args.kwargs["user"]["user_id"] == "user_123" + + +@pytest.mark.asyncio +async def test_search_memory_tool_execution(mock_memory_service): + """Test the search_memory tool execution logic.""" + adapter = MemULangGraphTools(mock_memory_service) + search_tool = adapter.search_memory_tool() + + inputs = {"query": "Test query", "user_id": "user_123"} + + result = await search_tool.ainvoke(inputs) + + assert "Test memory 1" in result + mock_memory_service.retrieve.assert_called_once() + + +def test_import_langgraph_dep(): + """Verify strictly that langgraph is importable and used.""" + import langgraph + + assert langgraph is not None diff --git a/tests/llm/test_grok_provider.py b/tests/llm/test_grok_provider.py new file mode 100644 index 0000000..6aafe3a --- /dev/null +++ b/tests/llm/test_grok_provider.py @@ -0,0 +1,45 @@ +import unittest +from unittest.mock import patch + +from memu.app.settings import LLMConfig +from memu.llm.backends.grok import GrokBackend +from memu.llm.openai_client import OpenAIClient + + +class TestGrokProvider(unittest.IsolatedAsyncioTestCase): + def test_settings_defaults(self): + """Test that setting provider='grok' sets the correct defaults.""" + config = LLMConfig(provider="grok") + self.assertEqual(config.base_url, "https://api.x.ai/v1") + self.assertEqual(config.api_key, "XAI_API_KEY") + self.assertEqual(config.chat_model, "grok-4-1-fast") + + @patch("memu.llm.openai_client.AsyncOpenAI") + async def test_client_initialization_with_grok_config(self, mock_async_openai): + """Test that OpenAIClient initializes with Grok base URL when configured.""" + # Setup config + config = LLMConfig(provider="grok") + + # Instantiate client with Grok config + # We simulate what the application factory would do: pass the config values + client = OpenAIClient( + base_url=config.base_url, + api_key="fake-key", # In real app, this would be os.getenv(config.api_key) + chat_model=config.chat_model, + ) + + # Assert AsyncOpenAI was called with the correct base_url + mock_async_openai.assert_called_with(api_key="fake-key", base_url="https://api.x.ai/v1") + + # Verify client attributes + self.assertEqual(client.chat_model, "grok-4-1-fast") + + def test_grok_backend_payload_parsing(self): + """Test that GrokBackend parses responses correctly (inherited from OpenAI).""" + backend = GrokBackend() + + # Simulate a typical OpenAI-compatible response + dummy_response = {"choices": [{"message": {"content": "Grok response content", "role": "assistant"}}]} + + result = backend.parse_summary_response(dummy_response) + self.assertEqual(result, "Grok response content") diff --git a/tests/rust_entry_test.py b/tests/rust_entry_test.py new file mode 100644 index 0000000..76f23a3 --- /dev/null +++ b/tests/rust_entry_test.py @@ -0,0 +1,5 @@ +from memu import _rust_entry + + +def test_rust_entry(): + assert _rust_entry() == "Hello from memu!" diff --git a/tests/test_audio_preprocess.py b/tests/test_audio_preprocess.py new file mode 100644 index 0000000..f5c66b0 --- /dev/null +++ b/tests/test_audio_preprocess.py @@ -0,0 +1,96 @@ +"""Tests for audio preprocessing: type classification + overview + caption. + +The audio preprocessor turns a transcription into cleaned content prefixed with +an "Audio Overview" (type/language/speakers/topic) and a one-sentence caption +that begins with the inferred audio type (song, conversation, lecture, ...). +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from memu.preprocess import preprocess_resource +from memu.preprocess.base import PreprocessContext +from memu.prompts.preprocess import PROMPTS + + +class _RecordingChatClient: + """Chat client that records the prompt and returns a tagged response.""" + + def __init__(self, response: str) -> None: + self.prompts: list[str] = [] + self._response = response + + async def chat(self, prompt: str, **_: Any) -> str: + self.prompts.append(prompt) + return self._response + + +def _make_ctx(client: Any) -> PreprocessContext: + return PreprocessContext( + get_llm_client=lambda: client, + get_vlm_client=lambda: None, + escape_prompt_value=lambda s: s, + extract_json_blob=lambda s: s, + resolve_custom_prompt=lambda _p, _v: "", + multimodal_preprocess_prompts={}, + ) + + +def test_audio_prompt_asks_to_classify_type() -> None: + template = PROMPTS["audio"] + assert "{transcription}" in template + # The prompt should steer the model toward classifying the audio nature. + for keyword in ("Classify the Audio", "conversation", "song", "Audio Overview", "Type:"): + assert keyword in template + + +def test_audio_preprocess_returns_overview_and_typed_caption() -> None: + response = ( + "## Audio Overview\n" + "- Type: song\n" + "- Language: English\n" + "- Speakers: 1\n" + "- Topic: love\n\n" + "La la la, all you need is love." + "A song about love and togetherness." + ) + client = _RecordingChatClient(response) + ctx = _make_ctx(client) + + # Text is already provided, so transcription is skipped and the chat prompt runs. + result = asyncio.run( + preprocess_resource( + modality="audio", + local_path="/workspace/track.mp3", + text="la la la all you need is love", + ctx=ctx, + llm_client=client, + ) + ) + + # The transcription is injected into the classification prompt. + assert "la la la all you need is love" in client.prompts[0] + assert "## Audio Overview" in (result[0]["text"] or "") + assert "Type: song" in (result[0]["text"] or "") + assert result[0]["caption"] == "A song about love and togetherness." + + +def test_audio_preprocess_skips_without_text() -> None: + # No text and a non-audio/non-text extension: nothing to transcribe. + client = _RecordingChatClient("xy") + ctx = _make_ctx(client) + + result = asyncio.run( + preprocess_resource( + modality="audio", + local_path="/workspace/mystery.bin", + text=None, + ctx=ctx, + llm_client=client, + ) + ) + + assert result == [{"text": None, "caption": None}] + assert client.prompts == [] diff --git a/tests/test_backend_conformance.py b/tests/test_backend_conformance.py new file mode 100644 index 0000000..cce5460 --- /dev/null +++ b/tests/test_backend_conformance.py @@ -0,0 +1,376 @@ +"""Cross-backend conformance suite for the storage layer. + +These tests pin the repository contract behavior that must hold identically across +the pluggable backends (currently ``inmemory`` and ``sqlite``; ``postgres`` needs a +live server so it is exercised separately). They guard the Phase 0 fixes: + +- ``clear_*`` with a ``where`` scope mutates the shared state in place (no rebinding + that orphans the ``DatabaseState`` reference). +- the SQLite read path preserves ``extra`` (reinforcement / ref_id / tool metadata). +- deleting an item / clearing memory leaves no orphan ``RecallFileEntry`` relations. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +src_path = Path(__file__).parent.parent / "src" +if str(src_path) not in sys.path: + sys.path.insert(0, str(src_path)) + +import pytest # noqa: E402 + +from memu.app.settings import ( # noqa: E402 + DatabaseConfig, + DefaultUserModel, + MetadataStoreConfig, +) +from memu.database.factory import build_database # noqa: E402 + + +def _make_inmemory(): + config = DatabaseConfig(metadata_store=MetadataStoreConfig(provider="inmemory")) + return build_database(config=config, user_model=DefaultUserModel) + + +def _make_sqlite(tmp_path: Path): + dsn = f"sqlite:///{tmp_path / 'conformance.db'}" + config = DatabaseConfig(metadata_store=MetadataStoreConfig(provider="sqlite", dsn=dsn)) + return build_database(config=config, user_model=DefaultUserModel), dsn + + +@pytest.fixture(params=["inmemory", "sqlite"]) +def store(request, tmp_path): + if request.param == "inmemory": + db = _make_inmemory() + yield db + else: + db, _dsn = _make_sqlite(tmp_path) + yield db + db.close() + + +def _seed_item(store, *, summary: str, user_id: str, embedding=None): + res = store.resource_repo.create_resource( + url=f"mem://{summary}", + modality="document", + local_path="", + caption=summary, + embedding=None, + user_data={"user_id": user_id}, + ) + item = store.recall_entry_repo.create_item( + resource_id=res.id, + memory_type="knowledge", + summary=summary, + embedding=embedding or [0.1, 0.2, 0.3], + user_data={"user_id": user_id}, + ) + return res, item + + +def test_clear_items_with_scope_mutates_shared_state(store): + """Clearing a scoped subset must not orphan the shared state reference.""" + _seed_item(store, summary="a", user_id="alice") + _seed_item(store, summary="b", user_id="bob") + + deleted = store.recall_entry_repo.clear_items({"user_id": "alice"}) + assert len(deleted) == 1 + + remaining = store.recall_entry_repo.list_items() + summaries = {item.summary for item in remaining.values()} + assert summaries == {"b"} + + +def test_clear_categories_with_scope_mutates_shared_state(store): + store.recall_file_repo.get_or_create_category( + name="alpha", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "alice"} + ) + store.recall_file_repo.get_or_create_category( + name="beta", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "bob"} + ) + + store.recall_file_repo.clear_categories({"user_id": "alice"}) + remaining = store.recall_file_repo.list_categories() + names = {cat.name for cat in remaining.values()} + assert names == {"beta"} + + +def test_unlink_item_removes_all_relations(store): + _res, item = _seed_item(store, summary="linked", user_id="alice") + cat1 = store.recall_file_repo.get_or_create_category( + name="c1", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "alice"} + ) + cat2 = store.recall_file_repo.get_or_create_category( + name="c2", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "alice"} + ) + store.recall_file_entry_repo.link_item_category(item.id, cat1.id, user_data={"user_id": "alice"}) + store.recall_file_entry_repo.link_item_category(item.id, cat2.id, user_data={"user_id": "alice"}) + assert len(store.recall_file_entry_repo.get_item_categories(item.id)) == 2 + + removed = store.recall_file_entry_repo.unlink_item(item.id) + assert len(removed) == 2 + assert store.recall_file_entry_repo.get_item_categories(item.id) == [] + assert store.recall_file_entry_repo.list_relations() == [] + + +def test_delete_item_leaves_no_orphan_relations(store): + """The Phase 0 delete fix: unlink relations before deleting the item.""" + _res, item = _seed_item(store, summary="doomed", user_id="alice") + cat = store.recall_file_repo.get_or_create_category( + name="c", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "alice"} + ) + store.recall_file_entry_repo.link_item_category(item.id, cat.id, user_data={"user_id": "alice"}) + + store.recall_file_entry_repo.unlink_item(item.id) + store.recall_entry_repo.delete_item(item.id) + + assert store.recall_entry_repo.get_item(item.id) is None + # No relation should point at the deleted item. + assert all(rel.item_id != item.id for rel in store.recall_file_entry_repo.list_relations()) + + +def test_clear_relations_with_scope(store): + _res, item = _seed_item(store, summary="r", user_id="alice") + cat = store.recall_file_repo.get_or_create_category( + name="c", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "alice"} + ) + store.recall_file_entry_repo.link_item_category(item.id, cat.id, user_data={"user_id": "alice"}) + + removed = store.recall_file_entry_repo.clear_relations({"user_id": "alice"}) + assert len(removed) == 1 + assert store.recall_file_entry_repo.list_relations({"user_id": "alice"}) == [] + + +def test_extra_round_trips_through_create_and_read(store): + """``extra`` (tool metadata / ref_id / reinforcement) must survive a read.""" + res = store.resource_repo.create_resource( + url="mem://tool", + modality="document", + local_path="", + caption="tool", + embedding=None, + user_data={"user_id": "alice"}, + ) + item = store.recall_entry_repo.create_item( + resource_id=res.id, + memory_type="tool", + summary="tool memory", + embedding=[0.1, 0.2, 0.3], + user_data={"user_id": "alice"}, + tool_record={"when_to_use": "always"}, + ) + assert item.extra.get("when_to_use") == "always" + + fetched = store.recall_entry_repo.get_item(item.id) + assert fetched is not None + assert fetched.extra.get("when_to_use") == "always" + + +def _reconcile(crud_self, store, *, item_id, new_cat_names, mapped_old_cat_ids, name_to_id): + from types import SimpleNamespace + + from memu.app.crud import CRUDMixin + + fake_self = SimpleNamespace(_map_category_names_to_ids=lambda names, ctx: [name_to_id[n] for n in names]) + CRUDMixin._reconcile_update_categories( + fake_self, # type: ignore[arg-type] + memory_id=item_id, + new_cat_names=new_cat_names, + mapped_old_cat_ids=mapped_old_cat_ids, + content_changed=False, + old_content="old", + new_summary="new", + ctx=None, + store=store, + user={"user_id": "alice"}, + propagate=False, + category_memory_updates={}, + ) + + +def test_update_with_none_categories_keeps_links(store): + """P0 regression: omitting categories (None) must NOT drop existing links.""" + _res, item = _seed_item(store, summary="keep", user_id="alice") + cat = store.recall_file_repo.get_or_create_category( + name="A", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "alice"} + ) + store.recall_file_entry_repo.link_item_category(item.id, cat.id, user_data={"user_id": "alice"}) + + _reconcile( + None, + store, + item_id=item.id, + new_cat_names=None, + mapped_old_cat_ids=[cat.id], + name_to_id={"A": cat.id}, + ) + + linked = {rel.category_id for rel in store.recall_file_entry_repo.get_item_categories(item.id)} + assert linked == {cat.id} + + +def test_update_with_empty_categories_clears_links(store): + """An explicit empty list clears links (distinct from omitted/None).""" + _res, item = _seed_item(store, summary="clear", user_id="alice") + cat = store.recall_file_repo.get_or_create_category( + name="A", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "alice"} + ) + store.recall_file_entry_repo.link_item_category(item.id, cat.id, user_data={"user_id": "alice"}) + + _reconcile( + None, + store, + item_id=item.id, + new_cat_names=[], + mapped_old_cat_ids=[cat.id], + name_to_id={"A": cat.id}, + ) + + assert store.recall_file_entry_repo.get_item_categories(item.id) == [] + + +def test_update_with_new_categories_swaps_links(store): + _res, item = _seed_item(store, summary="swap", user_id="alice") + cat_a = store.recall_file_repo.get_or_create_category( + name="A", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "alice"} + ) + cat_b = store.recall_file_repo.get_or_create_category( + name="B", description="", embedding=[0.1, 0.2, 0.3], user_data={"user_id": "alice"} + ) + store.recall_file_entry_repo.link_item_category(item.id, cat_a.id, user_data={"user_id": "alice"}) + + _reconcile( + None, + store, + item_id=item.id, + new_cat_names=["B"], + mapped_old_cat_ids=[cat_a.id], + name_to_id={"A": cat_a.id, "B": cat_b.id}, + ) + + linked = {rel.category_id for rel in store.recall_file_entry_repo.get_item_categories(item.id)} + assert linked == {cat_b.id} + + +class _FakeEmbedClient: + async def embed(self, texts): + return [[float(len(t)), 1.0, 0.0] for t in texts] + + +def test_resolve_category_ids_creates_unknown_adaptively(store): + """Open/adaptive taxonomy: extractor-proposed names are created on first sight.""" + import asyncio + from types import SimpleNamespace + + from memu.app.memorize import MemorizeMixin + from memu.app.service import Context + + ctx = Context(categories_ready=True) + fake_self = SimpleNamespace( + _get_embedding_client=lambda profile=None: _FakeEmbedClient(), + _partition_category_names=MemorizeMixin._partition_category_names, + ) + + ids = asyncio.run( + MemorizeMixin._resolve_category_ids( + fake_self, # type: ignore[arg-type] + ["Programming", "programming", "Cooking"], + ctx, + store, + user={"user_id": "alice"}, + allow_new=True, + ) + ) + # "Programming"/"programming" collapse (case-insensitive); "Cooking" is distinct. + assert len(ids) == 2 + names = {c.name for c in store.recall_file_repo.list_categories().values()} + assert names == {"Programming", "Cooking"} + + # A subsequent call reuses the cached ids and creates nothing new. + ids2 = asyncio.run( + MemorizeMixin._resolve_category_ids( + fake_self, # type: ignore[arg-type] + ["Programming"], + ctx, + store, + user={"user_id": "alice"}, + allow_new=True, + ) + ) + assert ids2 == [ctx.category_name_to_id["programming"]] + assert len(store.recall_file_repo.list_categories()) == 2 + + +def test_resolve_category_ids_gated_drops_unknown(store): + """With ``allow_new`` false, proposed-but-unknown names are dropped, not created.""" + import asyncio + from types import SimpleNamespace + + from memu.app.memorize import MemorizeMixin + from memu.app.service import Context + + # Seed one existing category and register it in the context taxonomy. + existing = store.recall_file_repo.get_or_create_category( + name="Programming", description="", embedding=[1.0, 0.0, 0.0], user_data={"user_id": "alice"} + ) + ctx = Context(categories_ready=True) + ctx.category_ids = [existing.id] + ctx.category_name_to_id = {"programming": existing.id} + + fake_self = SimpleNamespace( + _get_embedding_client=lambda profile=None: _FakeEmbedClient(), + _partition_category_names=MemorizeMixin._partition_category_names, + ) + + ids = asyncio.run( + MemorizeMixin._resolve_category_ids( + fake_self, # type: ignore[arg-type] + ["Programming", "Cooking"], # "Cooking" is unknown to the provided taxonomy + ctx, + store, + user={"user_id": "alice"}, + allow_new=False, + ) + ) + # Only the known "Programming" resolves; "Cooking" is dropped and never created. + assert ids == [existing.id] + names = {c.name for c in store.recall_file_repo.list_categories().values()} + assert names == {"Programming"} + + +def test_sqlite_extra_survives_cache_miss(tmp_path): + """A fresh SQLite store (cold cache) must reconstruct ``extra`` from the DB.""" + db, dsn = _make_sqlite(tmp_path) + res = db.resource_repo.create_resource( + url="mem://tool", + modality="document", + local_path="", + caption="tool", + embedding=None, + user_data={"user_id": "alice"}, + ) + item = db.recall_entry_repo.create_item( + resource_id=res.id, + memory_type="tool", + summary="tool memory", + embedding=[0.1, 0.2, 0.3], + user_data={"user_id": "alice"}, + tool_record={"when_to_use": "cold-read"}, + ) + item_id = item.id + db.close() + + # Re-open the same DB file: caches are empty, so reads hit the DB read path. + config = DatabaseConfig(metadata_store=MetadataStoreConfig(provider="sqlite", dsn=dsn)) + db2 = build_database(config=config, user_model=DefaultUserModel) + try: + fetched = db2.recall_entry_repo.get_item(item_id) + assert fetched is not None + assert fetched.extra.get("when_to_use") == "cold-read" + + listed = db2.recall_entry_repo.list_items() + assert listed[item_id].extra.get("when_to_use") == "cold-read" + finally: + db2.close() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..cff8a52 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,58 @@ +"""CLI surface tests: argument parsing, config mapping, and error paths. + +These never construct a MemoryService (no network, no LLM); they exercise the +pure argv -> config layer and the pre-service validation in the handlers. +""" + +from __future__ import annotations + +import pathlib + +import pytest + +from memu.cli import _database_config, build_parser, main + + +def test_parser_covers_all_entry_points() -> None: + parser = build_parser() + for argv in ( + ["memorize", "notes.md"], + ["memorize-workspace", "./ws"], + ["sync", "./ws"], + ["retrieve", "query"], + ["retrieve-workspace", "query"], + ["search", "query"], + ["export"], + ): + args = parser.parse_args(argv) + assert callable(args.handler) + + +def test_database_config_dispatch(tmp_path: pathlib.Path) -> None: + assert _database_config(":memory:") == {"metadata_store": {"provider": "inmemory"}} + pg = _database_config("postgresql://u:p@localhost/db") + assert pg["metadata_store"]["provider"] == "postgres" + assert pg["metadata_store"]["dsn"] == "postgresql://u:p@localhost/db" + + db_file = tmp_path / "nested" / "memu.sqlite3" + sqlite = _database_config(str(db_file)) + assert sqlite["metadata_store"]["provider"] == "sqlite" + assert sqlite["metadata_store"]["dsn"] == f"sqlite:///{db_file}" + assert db_file.parent.is_dir() # parent directory is created eagerly + + +def test_memorize_missing_file_exits_2(capsys: pytest.CaptureFixture[str]) -> None: + assert main(["memorize", "/definitely/not/a/file.md"]) == 2 + assert "no such file" in capsys.readouterr().err + + +def test_memorize_unknown_extension_exits_2(tmp_path: pathlib.Path, capsys: pytest.CaptureFixture[str]) -> None: + target = tmp_path / "data.xyz" + target.write_text("hello") + assert main(["memorize", str(target)]) == 2 + assert "cannot infer modality" in capsys.readouterr().err + + +def test_memorize_workspace_missing_folder_exits_2(capsys: pytest.CaptureFixture[str]) -> None: + assert main(["memorize-workspace", "/definitely/not/a/folder"]) == 2 + assert "no such folder" in capsys.readouterr().err diff --git a/tests/test_conversation_preprocess.py b/tests/test_conversation_preprocess.py new file mode 100644 index 0000000..d99ca6a --- /dev/null +++ b/tests/test_conversation_preprocess.py @@ -0,0 +1,60 @@ +"""Tests for conversation preprocessing after segmentation was removed. + +The conversation preprocessor now normalizes the chat log into an indexed, +line-based transcript and returns it as a single resource. It must not segment +the conversation nor call the LLM. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from memu.preprocess import preprocess_resource +from memu.preprocess.base import PreprocessContext + + +class _RecordingChatClient: + """LLM client that records calls so the test can assert it is never used.""" + + def __init__(self) -> None: + self.calls = 0 + + async def chat(self, *_: Any, **__: Any) -> str: + self.calls += 1 + return "" + + +def _make_ctx(client: Any) -> PreprocessContext: + return PreprocessContext( + get_llm_client=lambda: client, + get_vlm_client=lambda: None, + escape_prompt_value=lambda s: s, + extract_json_blob=lambda s: s, + resolve_custom_prompt=lambda _p, _v: "", + multimodal_preprocess_prompts={}, + ) + + +def test_conversation_returns_single_unsegmented_resource() -> None: + client = _RecordingChatClient() + ctx = _make_ctx(client) + raw = '[{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello"}]' + + result = asyncio.run( + preprocess_resource( + modality="conversation", + local_path="/workspace/conv.json", + text=raw, + ctx=ctx, + llm_client=client, + ) + ) + + # Whole conversation in a single segment, no caption, no LLM involvement. + assert client.calls == 0 + assert len(result) == 1 + assert result[0]["caption"] is None + text = result[0]["text"] or "" + assert "[0] [user]: Hi" in text + assert "[1] [assistant]: Hello" in text diff --git a/tests/test_document_text.py b/tests/test_document_text.py new file mode 100644 index 0000000..cfa2ffb --- /dev/null +++ b/tests/test_document_text.py @@ -0,0 +1,101 @@ +"""Tests for rich-document text extraction via MarkItDown.""" + +from __future__ import annotations + +import pathlib +import sys + +import pytest + +from memu.blob.document_text import extract_document_text, is_rich_document +from memu.blob.folder import infer_modality +from memu.blob.local_fs import LocalFS + +HTML_SAMPLE = "

Title

Hello world

  • a
  • b
" + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("a.pdf", True), + ("a.PDF", True), + ("a.docx", True), + ("a.pptx", True), + ("a.xlsx", True), + ("a.html", True), + ("a.epub", True), + ("a.txt", False), + ("a.md", False), + ("a.json", False), + ], +) +def test_is_rich_document(name: str, expected: bool) -> None: + assert is_rich_document(name) is expected + + +def test_infer_modality_includes_rich_documents() -> None: + assert infer_modality("report.pdf") == "document" + assert infer_modality("notes.docx") == "document" + assert infer_modality("deck.pptx") == "document" + assert infer_modality("sheet.xlsx") == "document" + assert infer_modality("page.HTML") == "document" + + +def test_plain_text_is_read_directly(tmp_path: pathlib.Path) -> None: + txt = tmp_path / "note.txt" + txt.write_text("plain text content", encoding="utf-8") + assert extract_document_text(txt) == "plain text content" + + +def test_html_is_converted_to_markdown(tmp_path: pathlib.Path) -> None: + html = tmp_path / "sample.html" + html.write_text(HTML_SAMPLE, encoding="utf-8") + text = extract_document_text(html) + assert "# Title" in text + assert "**world**" in text + assert "* a" in text + + +def test_docx_is_converted_to_markdown(tmp_path: pathlib.Path) -> None: + docx = pytest.importorskip("docx") + document = docx.Document() + document.add_heading("Quarterly Report", level=1) + document.add_paragraph("Revenue grew by 20 percent.") + path = tmp_path / "report.docx" + document.save(path) + + text = extract_document_text(path) + assert "Quarterly Report" in text + assert "Revenue grew by 20 percent." in text + + +def test_missing_markitdown_raises_actionable_error(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + pdf = tmp_path / "doc.pdf" + pdf.write_bytes(b"%PDF-1.4 fake") + # Force the lazy ``import markitdown`` inside the helper to fail. + monkeypatch.setitem(sys.modules, "markitdown", None) + + with pytest.raises(RuntimeError, match="memu-py\\[document\\]"): + extract_document_text(pdf) + + +async def test_local_fs_fetch_converts_document(tmp_path: pathlib.Path) -> None: + src = tmp_path / "sample.html" + src.write_text(HTML_SAMPLE, encoding="utf-8") + fs = LocalFS(str(tmp_path / "store")) + + local_path, text = await fs.fetch(str(src), "document") + + assert pathlib.Path(local_path).exists() + assert text is not None + assert "# Title" in text + + +async def test_local_fs_fetch_plain_text_document(tmp_path: pathlib.Path) -> None: + src = tmp_path / "note.md" + src.write_text("# Heading\n\nbody", encoding="utf-8") + fs = LocalFS(str(tmp_path / "store")) + + _, text = await fs.fetch(str(src), "document") + + assert text == "# Heading\n\nbody" diff --git a/tests/test_embedding.py b/tests/test_embedding.py new file mode 100644 index 0000000..309f90d --- /dev/null +++ b/tests/test_embedding.py @@ -0,0 +1,170 @@ +"""Unit tests for the standalone ``memu.embedding`` package. + +These pin the embedding module's contract that mirrors ``memu.llm``/``memu.vlm``: + +- per-provider backends (openai/jina/voyage/openrouter/doubao) build the right + payload/endpoint and parse the ``data[].embedding`` response shape. +- the HTTP client falls back to an OpenAI-compatible backend for unknown + providers and returns ``(vectors, raw_response)``. +- the gateway dispatches on ``client_backend`` and raises clearly for anthropic. +- ``EmbeddingConfig`` resolves per-provider base_url/api_key/model defaults, and + ``embedding_config_from_llm`` derives a config from an LLM profile. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +src_path = Path(__file__).parent.parent / "src" +if str(src_path) not in sys.path: + sys.path.insert(0, str(src_path)) + +import pytest # noqa: E402 + +from memu.app.settings import EmbeddingConfig, LLMConfig, embedding_config_from_llm # noqa: E402 +from memu.embedding.backends import ( # noqa: E402 + JinaEmbeddingBackend, + OpenAIEmbeddingBackend, + OpenRouterEmbeddingBackend, + VoyageEmbeddingBackend, +) +from memu.embedding.gateway import build_embedding_client # noqa: E402 +from memu.embedding.http_client import HTTPEmbeddingClient # noqa: E402 +from memu.embedding.openai_sdk import OpenAIEmbeddingSDKClient # noqa: E402 + + +@pytest.mark.parametrize( + ("backend", "endpoint"), + [ + (OpenAIEmbeddingBackend(), "/embeddings"), + (JinaEmbeddingBackend(), "/embeddings"), + (VoyageEmbeddingBackend(), "/embeddings"), + (OpenRouterEmbeddingBackend(), "/api/v1/embeddings"), + ], +) +def test_backend_payload_and_parse(backend, endpoint): + assert backend.embedding_endpoint == endpoint + assert backend.default_headers("k") == {"Authorization": "Bearer k"} + + payload = backend.build_embedding_payload(inputs=["a", "b"], embed_model="m") + assert payload["model"] == "m" + assert payload["input"] == ["a", "b"] + + parsed = backend.parse_embedding_response({"data": [{"embedding": [0.1, 0.2]}, {"embedding": [0.3, 0.4]}]}) + assert parsed == [[0.1, 0.2], [0.3, 0.4]] + + +def test_http_client_unknown_provider_falls_back_to_openai(): + client = HTTPEmbeddingClient(base_url="https://x/v1", api_key="k", embed_model="m", provider="grok") + assert isinstance(client.backend, OpenAIEmbeddingBackend) + + +def test_http_client_selects_registered_backend(): + client = HTTPEmbeddingClient(base_url="https://api.jina.ai/v1", api_key="k", embed_model="m", provider="jina") + assert isinstance(client.backend, JinaEmbeddingBackend) + + +async def test_http_client_embed_returns_vectors_and_raw(monkeypatch): + captured: dict = {} + + class _FakeResponse: + def raise_for_status(self): + return None + + def json(self): + return {"data": [{"embedding": [1.0, 2.0]}], "usage": {"total_tokens": 3}} + + class _FakeAsyncClient: + def __init__(self, *args, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + async def post(self, endpoint, json, headers): + captured["endpoint"] = endpoint + captured["json"] = json + captured["headers"] = headers + return _FakeResponse() + + import memu.embedding.http_client as http_mod + + monkeypatch.setattr(http_mod.httpx, "AsyncClient", _FakeAsyncClient) + + client = HTTPEmbeddingClient( + base_url="https://api.voyageai.com/v1", api_key="key", embed_model="voyage-3.5", provider="voyage" + ) + vectors, raw = await client.embed(["hello"]) + + assert vectors == [[1.0, 2.0]] + assert raw["usage"]["total_tokens"] == 3 + assert captured["endpoint"] == "embeddings" # leading slash stripped + assert captured["headers"] == {"Authorization": "Bearer key"} + assert captured["json"] == {"model": "voyage-3.5", "input": ["hello"]} + + +def test_gateway_builds_sdk_and_httpx_clients(): + sdk = build_embedding_client(EmbeddingConfig(provider="openai", api_key="k", client_backend="sdk")) + assert isinstance(sdk, OpenAIEmbeddingSDKClient) + + httpx_client = build_embedding_client(EmbeddingConfig(provider="jina", api_key="k", client_backend="httpx")) + assert isinstance(httpx_client, HTTPEmbeddingClient) + assert isinstance(httpx_client.backend, JinaEmbeddingBackend) + + +def test_gateway_rejects_anthropic_and_unknown_backends(): + with pytest.raises(ValueError, match="Anthropic does not provide"): + build_embedding_client(EmbeddingConfig(client_backend="anthropic")) + + with pytest.raises(ValueError, match="Unknown embedding client_backend"): + build_embedding_client(EmbeddingConfig(client_backend="nope")) + + +def test_embedding_config_provider_defaults(): + jina = EmbeddingConfig(provider="jina") + assert jina.base_url == "https://api.jina.ai/v1" + assert jina.api_key == "JINA_API_KEY" + assert jina.embed_model == "jina-embeddings-v3" + + voyage = EmbeddingConfig(provider="voyage") + assert voyage.base_url == "https://api.voyageai.com/v1" + assert voyage.api_key == "VOYAGE_API_KEY" + assert voyage.embed_model == "voyage-3.5" + + # Explicit values always survive the provider-default merge. + explicit = EmbeddingConfig(provider="jina", base_url="https://proxy/v1", api_key="real", embed_model="custom") + assert explicit.base_url == "https://proxy/v1" + assert explicit.api_key == "real" + assert explicit.embed_model == "custom" + + +def test_chat_clients_no_longer_expose_embed(): + """Embedding is fully decoupled: text/chat clients must not carry embed().""" + from memu.llm.anthropic_client import AnthropicClient + from memu.llm.http_client import HTTPLLMClient + from memu.llm.openai_client import OpenAIClient + + assert not hasattr(OpenAIClient, "embed") + assert not hasattr(HTTPLLMClient, "embed") + assert not hasattr(AnthropicClient, "embed") + # The HTTP chat client no longer wires an embedding backend either. + assert not hasattr(HTTPLLMClient, "_load_embedding_backend") + + +def test_embedding_config_from_llm_carries_transport_and_model(): + llm = LLMConfig( + provider="openrouter", + client_backend="httpx", + api_key="rk", + embed_model="openai/text-embedding-3-large", + ) + emb = embedding_config_from_llm(llm) + assert emb.provider == "openrouter" + assert emb.client_backend == "httpx" + assert emb.api_key == "rk" + assert emb.embed_model == "openai/text-embedding-3-large" + assert emb.base_url == llm.base_url diff --git a/tests/test_folder_memorize.py b/tests/test_folder_memorize.py new file mode 100644 index 0000000..c30fdd0 --- /dev/null +++ b/tests/test_folder_memorize.py @@ -0,0 +1,295 @@ +"""Tests for the workspace memorize backbone: scanning, diffing, cascade sync. + +Covers ``memu.blob.folder`` plus the additive ``MemoryService.memorize_workspace`` +entry point. The single-file ``memorize`` is intentionally untouched and is not +exercised here. Export of the memory file tree is added in a later step, so these +tests stay focused on scan -> diff -> cascade -> manifest. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from memu.app import MemoryService +from memu.blob.folder import ( + MANIFEST_FILENAME, + diff_folder, + infer_modality, + load_manifest, + manifest_from_scan, + save_manifest, + scan_folder, +) + +# -- folder module: scan / modality / manifest / diff ------------------------ + + +def test_infer_modality_by_extension() -> None: + assert infer_modality("a.json") == "conversation" + assert infer_modality("a.txt") == "document" + assert infer_modality("a.MD") == "document" + assert infer_modality("a.png") == "image" + assert infer_modality("a.mp4") == "video" + assert infer_modality("a.mp3") == "audio" + assert infer_modality("a.unknownext") is None + + +def test_scan_folder_recurses_skips_unknown_and_hidden(tmp_path: Path) -> None: + (tmp_path / "a.txt").write_text("alpha", encoding="utf-8") + (tmp_path / "nested").mkdir() + (tmp_path / "nested" / "b.md").write_text("beta", encoding="utf-8") + (tmp_path / "skip.bin").write_text("binary", encoding="utf-8") # unknown ext + (tmp_path / ".hidden.txt").write_text("hidden", encoding="utf-8") # hidden file + (tmp_path / MANIFEST_FILENAME).write_text("{}", encoding="utf-8") # manifest + + scanned = scan_folder(tmp_path) + + assert set(scanned) == {"a.txt", "nested/b.md"} + assert scanned["a.txt"].modality == "document" + assert scanned["nested/b.md"].modality == "document" + assert scanned["a.txt"].content_hash != scanned["nested/b.md"].content_hash + + +def test_manifest_roundtrip_and_diff(tmp_path: Path) -> None: + (tmp_path / "keep.txt").write_text("keep", encoding="utf-8") + (tmp_path / "change.txt").write_text("v1", encoding="utf-8") + (tmp_path / "gone.txt").write_text("bye", encoding="utf-8") + + first = scan_folder(tmp_path) + save_manifest(tmp_path, manifest_from_scan(first)) + assert load_manifest(tmp_path) == manifest_from_scan(first) + + # Mutate the folder: modify one file, delete another, add a new one. + (tmp_path / "change.txt").write_text("v2", encoding="utf-8") + (tmp_path / "gone.txt").unlink() + (tmp_path / "new.md").write_text("fresh", encoding="utf-8") + + second = scan_folder(tmp_path) + diff = diff_folder(second, load_manifest(tmp_path)) + + assert [f.rel_path for f in diff.added] == ["new.md"] + assert [f.rel_path for f in diff.modified] == ["change.txt"] + assert diff.deleted == ["gone.txt"] + assert diff.has_changes and diff.has_removals + + +def test_diff_added_only_has_no_removals(tmp_path: Path) -> None: + (tmp_path / "a.txt").write_text("a", encoding="utf-8") + diff = diff_folder(scan_folder(tmp_path), {}) + assert [f.rel_path for f in diff.added] == ["a.txt"] + assert not diff.has_removals + + +# -- service-level: cascade delete & orchestration --------------------------- + + +def _service() -> MemoryService: + return MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + ) + + +def _seed_resource_with_item(service: MemoryService, *, url: str, category_id: str, user: dict[str, Any]) -> str: + """Create a resource + one item + one relation, returning the resource id.""" + store = service.database + res = store.resource_repo.create_resource( + url=url, + modality="document", + local_path=url, + caption="cap", + embedding=None, + user_data=dict(user), + ) + item = store.recall_entry_repo.create_item( + resource_id=res.id, + memory_type="profile", + summary=f"summary for {url}", + embedding=[0.0], + user_data=dict(user), + ) + store.recall_file_entry_repo.link_item_category(item.id, category_id, dict(user)) + return res.id + + +async def test_cascade_delete_removes_resource_items_relations(monkeypatch) -> None: + service = _service() + store = service.database + user = {"user_id": "u1"} + ctx = service._get_context() + category_id = "cat-1" + + keep_id = _seed_resource_with_item(service, url="/folder/keep.txt", category_id=category_id, user=user) + drop_id = _seed_resource_with_item(service, url="/folder/drop.txt", category_id=category_id, user=user) + + patched: list[dict[str, Any]] = [] + + async def _fake_patch(updates, *, ctx, store, llm_client=None) -> None: + patched.append(updates) + + monkeypatch.setattr(service, "_patch_category_summaries", _fake_patch) + + removed = await service._cascade_delete_by_urls({"/folder/drop.txt"}, ctx=ctx, store=store, user_scope=user) + + assert [r.id for r in removed] == [drop_id] + remaining = store.resource_repo.list_resources(where=user) + assert keep_id in remaining + assert drop_id not in remaining + # The dropped resource's item and relation are gone; the kept one's survive. + items = store.recall_entry_repo.list_items(where=user) + assert all(it.resource_id == keep_id for it in items.values()) + relations = store.recall_file_entry_repo.list_relations(where=user) + assert len(relations) == 1 + # Discarded content was fed to the summary recompute as (before, None). + assert patched and category_id in patched[0] + assert patched[0][category_id][1] is None + + +async def test_memorize_workspace_sync_add_modify_delete(tmp_path: Path, monkeypatch) -> None: + service = _service() + store = service.database + user = {"user_id": "u1"} + + # Avoid LLM-dependent paths; exercise scan -> diff -> cascade -> manifest. + async def _noop_categories(*a, **k) -> None: + return None + + async def _noop_patch(updates, *, ctx, store, llm_client=None) -> None: + return None + + async def _fake_memorize_one(*, resource_url, modality, user_scope, ctx, store, track=None) -> dict[str, Any]: + res = store.resource_repo.create_resource( + url=resource_url, + modality=modality, + local_path=resource_url, + caption="cap", + embedding=None, + user_data=dict(user_scope or {}), + track=track, + ) + store.recall_entry_repo.create_item( + resource_id=res.id, + memory_type="profile", + summary=f"summary {resource_url}", + embedding=[0.0], + user_data=dict(user_scope or {}), + ) + return {"resources": [res], "response": {"items": [{"summary": "x"}]}} + + monkeypatch.setattr(service, "_ensure_categories_ready", _noop_categories) + monkeypatch.setattr(service, "_patch_category_summaries", _noop_patch) + monkeypatch.setattr(service, "_memorize_one", _fake_memorize_one) + + (tmp_path / "a.txt").write_text("a-v1", encoding="utf-8") + (tmp_path / "b.md").write_text("b-v1", encoding="utf-8") + + # First sync: both files added. + first = await service.memorize_workspace(folder=str(tmp_path), user=user) + assert sorted(first["added"]) == ["a.txt", "b.md"] + assert first["modified"] == [] and first["deleted"] == [] + assert len(store.resource_repo.list_resources(where=user)) == 2 + # Manifest persisted in the folder. + manifest_path = tmp_path / MANIFEST_FILENAME + assert manifest_path.exists() + assert set(json.loads(manifest_path.read_text(encoding="utf-8"))) == {"a.txt", "b.md"} + + # Second sync: modify a.txt, delete b.md, add c.txt. + (tmp_path / "a.txt").write_text("a-v2", encoding="utf-8") + (tmp_path / "b.md").unlink() + (tmp_path / "c.txt").write_text("c-v1", encoding="utf-8") + + second = await service.memorize_workspace(folder=str(tmp_path), user=user) + assert second["added"] == ["c.txt"] + assert second["modified"] == ["a.txt"] + assert second["deleted"] == ["b.md"] + + root = tmp_path.resolve() + urls = {r.url for r in store.resource_repo.list_resources(where=user).values()} + assert urls == {str(root / "a.txt"), str(root / "c.txt")} + assert set(json.loads(manifest_path.read_text(encoding="utf-8"))) == {"a.txt", "c.txt"} + + +async def test_memorize_workspace_exports_when_enabled(tmp_path: Path, monkeypatch) -> None: + """When memory files are enabled, a workspace sync refreshes the markdown tree.""" + out_dir = tmp_path / "out" + src_dir = tmp_path / "src" + src_dir.mkdir() + service = MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + memory_files_config={"enabled": True, "output_dir": str(out_dir)}, + ) + user = {"user_id": "u1"} + + async def _noop_categories(*a, **k) -> None: + return None + + async def _fake_memorize_one(*, resource_url, modality, user_scope, ctx, store, track=None) -> dict[str, Any]: + res = store.resource_repo.create_resource( + url=resource_url, + modality=modality, + local_path=resource_url, + caption="cap", + embedding=None, + user_data=dict(user_scope or {}), + track=track, + ) + return {"resources": [res], "response": {"items": []}} + + exported: list[Any] = [] + real_export = service._memory_file_exporter.export + + def _spy_export(database, *, where=None, **kwargs): + exported.append(where) + return real_export(database, where=where, **kwargs) + + monkeypatch.setattr(service, "_ensure_categories_ready", _noop_categories) + monkeypatch.setattr(service, "_memorize_one", _fake_memorize_one) + monkeypatch.setattr(service._memory_file_exporter, "export", _spy_export) + + (src_dir / "a.txt").write_text("hello", encoding="utf-8") + await service.memorize_workspace(folder=str(src_dir), user=user) + + # Export ran (scoped to the user) and produced the root index on disk. + assert exported == [service.user_model(**user).model_dump()] + assert (out_dir / "INDEX.md").exists() + + +async def test_memorize_workspace_export_failure_does_not_fail_sync(tmp_path: Path, monkeypatch) -> None: + """An export error is best-effort: the sync still completes and persists state.""" + service = MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + memory_files_config={"enabled": True, "output_dir": str(tmp_path / "out")}, + ) + user = {"user_id": "u1"} + + async def _noop_categories(*a, **k) -> None: + return None + + async def _fake_memorize_one(*, resource_url, modality, user_scope, ctx, store, track=None) -> dict[str, Any]: + res = store.resource_repo.create_resource( + url=resource_url, + modality=modality, + local_path=resource_url, + caption="cap", + embedding=None, + user_data=dict(user_scope or {}), + track=track, + ) + return {"resources": [res], "response": {"items": []}} + + def _boom(database, *, where=None, **kwargs): + raise RuntimeError("export blew up") # noqa: TRY003 + + monkeypatch.setattr(service, "_ensure_categories_ready", _noop_categories) + monkeypatch.setattr(service, "_memorize_one", _fake_memorize_one) + monkeypatch.setattr(service._memory_file_exporter, "export", _boom) + + (tmp_path / "a.txt").write_text("hello", encoding="utf-8") + result = await service.memorize_workspace(folder=str(tmp_path), user=user) + + assert result["added"] == ["a.txt"] + assert (tmp_path / MANIFEST_FILENAME).exists() # manifest still persisted diff --git a/tests/test_inmemory.py b/tests/test_inmemory.py new file mode 100644 index 0000000..250f15a --- /dev/null +++ b/tests/test_inmemory.py @@ -0,0 +1,89 @@ +import os + +from memu.app import MemoryService + + +async def main(): + """Test with in-memory storage (default).""" + api_key = os.environ.get("OPENAI_API_KEY") + # dashscope_api_key = os.environ.get("DASHSCOPE_API_KEY") + # voyage_api_key = os.environ.get("VOYAGE_API_KEY") + file_path = os.path.abspath("example/example_conversation.json") + + print("\n" + "=" * 60) + print("[INMEMORY] Starting test...") + print("=" * 60) + + service = MemoryService( + llm_profiles={"default": {"api_key": api_key}}, + # llm_profiles={ + # "default": { + # "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + # "api_key": dashscope_api_key, + # "chat_model": "qwen3-max", + # "client_backend": "sdk" + # }, + # "embedding": { + # "base_url": "https://api.voyageai.com/v1", + # "api_key": voyage_api_key, + # "embed_model": "voyage-3.5-lite" + # } + # }, + database_config={ + "metadata_store": {"provider": "inmemory"}, + }, + retrieve_config={"method": "rag"}, + ) + + # Memorize + print("\n[INMEMORY] Memorizing...") + memory = await service.memorize(resource_url=file_path, modality="conversation", user={"user_id": "123"}) + for cat in memory.get("categories", []): + print(f" - {cat.get('name')}: {(cat.get('summary') or '')[:80]}...") + + queries = [ + {"role": "user", "content": {"text": "Tell me about preferences"}}, + {"role": "assistant", "content": {"text": "Sure, I'll tell you about their preferences"}}, + { + "role": "user", + "content": {"text": "What are they"}, + }, # This is the query that will be used to retrieve the memory, the context will be used for query rewriting + ] + + # RAG-based retrieval + print("\n[INMEMORY] RETRIEVED - RAG") + service.retrieve_config.method = "rag" + result_rag = await service.retrieve(queries=queries, where={"user_id": "123"}) + print(" Categories:") + for cat in result_rag.get("categories", [])[:3]: + print(f" - {cat.get('name')}: {(cat.get('summary') or cat.get('description', ''))[:80]}...") + print(" Items:") + for item in result_rag.get("items", [])[:3]: + print(f" - [{item.get('memory_type')}] {item.get('summary', '')[:100]}...") + if result_rag.get("resources"): + print(" Resources:") + for res in result_rag.get("resources", [])[:3]: + print(f" - [{res.get('modality')}] {res.get('url', '')[:80]}...") + + # LLM-based retrieval + print("\n[INMEMORY] RETRIEVED - LLM") + service.retrieve_config.method = "llm" + result_llm = await service.retrieve(queries=queries, where={"user_id": "123"}) + print(" Categories:") + for cat in result_llm.get("categories", [])[:3]: + print(f" - {cat.get('name')}: {(cat.get('summary') or cat.get('description', ''))[:80]}...") + print(" Items:") + for item in result_llm.get("items", [])[:3]: + print(f" - [{item.get('memory_type')}] {item.get('summary', '')[:100]}...") + if result_llm.get("resources"): + print(" Resources:") + for res in result_llm.get("resources", [])[:3]: + print(f" - [{res.get('modality')}] {res.get('url', '')[:80]}...") + + print("\n[INMEMORY] Test completed!") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/tests/test_lazyllm.py b/tests/test_lazyllm.py new file mode 100644 index 0000000..6a78963 --- /dev/null +++ b/tests/test_lazyllm.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +Quick test script to verify LazyLLM backend configuration and basic functionality. + +Usage: + export MEMU_QWEN_API_KEY=your_api_key + python examples/test_lazyllm.py +""" + +import asyncio +import os +import sys + +import pytest + +# Add src to sys.path +src_path = os.path.abspath("src") +sys.path.insert(0, src_path) + +pytest.importorskip("lazyllm") +if not os.getenv("MEMU_QWEN_API_KEY"): + pytest.skip("requires MEMU_QWEN_API_KEY for optional LazyLLM integration test", allow_module_level=True) + +from memu.llm.lazyllm_client import LazyLLMClient # noqa: E402 + + +async def test_lazyllm_client(): + """Test LazyLLMClient with basic operations.""" + + print("LazyLLM Backend Test") + print("=" * 60) + + # Get API key from environment + try: + client = LazyLLMClient( + llm_source="qwen", + vlm_source="qwen", + embed_source="qwen", + stt_source="qwen", + chat_model="qwen-plus", + vlm_model="qwen-vl-plus", + embed_model="text-embedding-v3", + stt_model="qwen-audio-turbo", + ) + print("✓ LazyLLMClient initialized successfully") + except Exception as e: + print(f"❌ Failed to initialize LazyLLMClient: {e}") + return False + + # Test 1: Summarization + print("\n[Test 1] Testing summarization...") + try: + test_text = "这是一段关于Python编程的文本。Python是一种高级编程语言,具有简单易学的语法。它被广泛用于数据分析、机器学习和Web开发。" # noqa: RUF001 + result = await client.chat(test_text) + print("✓ Summarization successful") + print(f" Result: {result[:100]}...") + except Exception as e: + print(f"❌ Summarization failed: {e}") + import traceback + + traceback.print_exc() + + # Test 2: Embedding + print("\n[Test 2] Testing embedding...") + try: + test_texts = ["Hello world", "How are you", "Nice to meet you"] + embeddings = await client.embed(test_texts) + print("✓ Embedding successful") + print(f" Generated {len(embeddings)} embeddings") + if embeddings and embeddings[0]: + print(f" Embedding dimension: {len(embeddings[0])}") + except Exception as e: + print(f"❌ Embedding failed: {e}") + import traceback + + traceback.print_exc() + + # Test 3: Vision (requires image file) + print("\n[Test 3] Testing vision...") + test_image_path = "examples/resources/images/image1.png" + if os.path.exists(test_image_path): + try: + result, _ = await client.vision(prompt="描述这张图片的内容", image_path=test_image_path) + print("✓ Vision successful") + print(f" Result: {result[:100]}...") + except Exception as e: + print(f"❌ Vision failed: {e}") + import traceback + + traceback.print_exc() + else: + print(f"⚠ Skipped: Test image not found at {test_image_path}") + + +if __name__ == "__main__": + success = asyncio.run(test_lazyllm_client()) + sys.exit(0 if success else 1) diff --git a/tests/test_memory_files.py b/tests/test_memory_files.py new file mode 100644 index 0000000..15d2c7f --- /dev/null +++ b/tests/test_memory_files.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from memu.app import MemoryService +from memu.memory_fs import MemoryFileExporter +from memu.memory_fs.exporter import MANIFEST_NAME + +# With synthesize=False (the default) the whole tree is LLM-free: MEMORY.md/SKILL.md +# are rendered as link indexes, and the memory/ and skill/ trees are rendered one file +# per RecallFile of their track (ADR 0006). + + +def _build_service(output_dir: Path) -> MemoryService: + return MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + memory_files_config={"enabled": True, "output_dir": str(output_dir)}, + ) + + +def _seed(service: MemoryService, *, user: dict[str, str]) -> dict[str, str]: + store = service.database + resource = store.resource_repo.create_resource( + url="docs/coffee.txt", + modality="document", + local_path="coffee.txt", + caption="Notes about the user's coffee preferences.", + embedding=None, + user_data=dict(user), + ) + category = store.recall_file_repo.get_or_create_category( + name="Preferences", + description="User preferences, likes and dislikes", + embedding=[0.1, 0.2], + user_data=dict(user), + ) + store.recall_file_repo.update_category(category_id=category.id, content="The user likes pour-over coffee.") + skill = store.recall_file_repo.get_or_create_category( + name="pour-over", + description="Pour-over brewing", + embedding=[0.1, 0.2], + user_data=dict(user), + track="skill", + ) + store.recall_file_repo.update_category(category_id=skill.id, content="# Pour-over brewing\nUse a 1:16 ratio.") + return {"category_id": category.id, "resource_id": resource.id, "skill_id": skill.id} + + +async def test_export_writes_readme_layout(tmp_path: Path) -> None: + service = _build_service(tmp_path) + _seed(service, user={"user_id": "u1"}) + + result = await service.export_memory_files(user={"user_id": "u1"}) + + assert result["changed"] is True + assert "INDEX.md" in result["written"] + assert "MEMORY.md" in result["written"] + assert "SKILL.md" in result["written"] + assert "memory/preferences.md" in result["written"] + assert "skill/pour-over.md" in result["written"] + + # MEMORY.md is now an overview that links to each memory/.md file; the + # category summary itself lives in memory/preferences.md. + memory_text = (tmp_path / "MEMORY.md").read_text(encoding="utf-8") + assert "**Preferences**" in memory_text + assert "memory/preferences.md" in memory_text + + category_text = (tmp_path / "memory" / "preferences.md").read_text(encoding="utf-8") + assert "The user likes pour-over coffee." in category_text + + # INDEX.md indexes the raw source files under resource/. + index_text = (tmp_path / "INDEX.md").read_text(encoding="utf-8") + assert "coffee.txt" in index_text + assert "coffee preferences" in index_text + + # The root SKILL.md indexes the skill/ tree, mirroring MEMORY.md over memory/. + skill_index = (tmp_path / "SKILL.md").read_text(encoding="utf-8") + assert "**pour-over**" in skill_index + assert "skill/pour-over.md" in skill_index + + skill_text = (tmp_path / "skill" / "pour-over.md").read_text(encoding="utf-8") + assert "Pour-over brewing" in skill_text + + +async def test_export_is_idempotent_until_data_changes(tmp_path: Path) -> None: + service = _build_service(tmp_path) + ids = _seed(service, user={"user_id": "u1"}) + + first = await service.export_memory_files(user={"user_id": "u1"}) + assert first["changed"] is True + + second = await service.export_memory_files(user={"user_id": "u1"}) + assert second["changed"] is False + assert second["written"] == [] + assert "MEMORY.md" in second["unchanged"] + + # Changing only a folder summary rewrites that category's memory/.md but + # not MEMORY.md (an overview of links) nor INDEX.md (a file TOC). + service.database.recall_file_repo.update_category( + category_id=ids["category_id"], + content="The user now prefers espresso.", + ) + third = await service.export_memory_files(user={"user_id": "u1"}) + assert third["changed"] is True + assert "memory/preferences.md" in third["written"] + assert "MEMORY.md" in third["unchanged"] + assert "INDEX.md" in third["unchanged"] + + +async def test_export_removes_stale_skill_and_prunes_dirs(tmp_path: Path) -> None: + service = _build_service(tmp_path) + _seed(service, user={"user_id": "u1"}) + + await service.export_memory_files(user={"user_id": "u1"}) + assert (tmp_path / "skill" / "pour-over.md").exists() + + # Dropping the skill-track file removes its doc; the empty skill/ dir is pruned. + service.database.recall_file_repo.clear_categories(where={"user_id": "u1", "track": "skill"}) + result = await service.export_memory_files(user={"user_id": "u1"}) + + assert "skill/pour-over.md" in result["removed"] + assert not (tmp_path / "skill").exists() + + +async def test_export_respects_user_scope(tmp_path: Path) -> None: + service = _build_service(tmp_path) + _seed(service, user={"user_id": "u1"}) + service.database.recall_file_repo.get_or_create_category( + name="Secret", + description="Other user's folder", + embedding=[0.3, 0.4], + user_data={"user_id": "u2"}, + ) + + await service.export_memory_files(user={"user_id": "u1"}) + + memory_text = (tmp_path / "MEMORY.md").read_text(encoding="utf-8") + assert "Preferences" in memory_text + assert "Secret" not in memory_text + + +async def test_export_disabled_raises(tmp_path: Path) -> None: + service = MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + ) + with pytest.raises(RuntimeError, match="disabled"): + await service.export_memory_files(user={"user_id": "u1"}) + + +def test_exporter_manifest_roundtrip(tmp_path: Path) -> None: + exporter = MemoryFileExporter(str(tmp_path)) + exporter._save_manifest({"MEMORY.md": "abc"}) + assert exporter._load_manifest() == {"MEMORY.md": "abc"} + + (tmp_path / MANIFEST_NAME).write_text("not json", encoding="utf-8") + assert exporter._load_manifest() == {} + + (tmp_path / MANIFEST_NAME).write_text(json.dumps([1, 2, 3]), encoding="utf-8") + assert exporter._load_manifest() == {} diff --git a/tests/test_memory_fs_synthesis.py b/tests/test_memory_fs_synthesis.py new file mode 100644 index 0000000..7ba5b30 --- /dev/null +++ b/tests/test_memory_fs_synthesis.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +from pathlib import Path + +from memu.app import MemoryService +from memu.memory_fs import FileDescription, MemoryFileExporter, MemorySynthesizer + +_MEMORY_MD = "## Profile\nThe user is a coffee enthusiast.\n\n## Preferences\nPrefers pour-over." +_SKILL_OVERVIEW = "## Brewing\nThe agent can brew pour-over coffee." + + +class _FakeChatClient: + """Stand-in LLM client: returns a canned memory- or skill-overview document. + + The skill-overview prompt is the only one that lists "SKILLS in the library", so + its presence distinguishes the two synthesis calls. + """ + + async def chat(self, prompt: str, system_prompt: str | None = None) -> str: + if "SKILLS in the library" in prompt: + return _SKILL_OVERVIEW + return _MEMORY_MD + + +def _descriptions() -> list[FileDescription]: + return [ + FileDescription( + url="docs/coffee.txt", + modality="document", + description="The user likes pour-over coffee with a 1:16 ratio.", + resource_id="r1", + ) + ] + + +def _skill_files() -> list: + from memu.database.models import RecallFile + + return [ + RecallFile( + id="s1", + name="pour-over", + description="Brew pour-over", + content="# Pour-over\nUse a 1:16 ratio.", + track="skill", + ) + ] + + +async def test_synthesize_memory_from_descriptions() -> None: + synth = MemorySynthesizer() + body = await synth.synthesize_memory(_descriptions(), chat=_FakeChatClient().chat) + assert "## Profile" in body + assert "pour-over" in body.lower() + + +async def test_synthesize_memory_empty_without_descriptions() -> None: + synth = MemorySynthesizer() + assert await synth.synthesize_memory([], chat=_FakeChatClient().chat) == "" + assert await synth.synthesize_memory([], existing_memory="## Keep", chat=_FakeChatClient().chat) == "## Keep" + + +async def test_synthesize_skill_overview_from_skill_files() -> None: + synth = MemorySynthesizer() + body = await synth.synthesize_skill_overview(_skill_files(), chat=_FakeChatClient().chat) + assert body == _SKILL_OVERVIEW + + +async def test_synthesize_skill_overview_empty_without_skills() -> None: + synth = MemorySynthesizer() + assert await synth.synthesize_skill_overview([], chat=_FakeChatClient().chat) == "" + assert await synth.synthesize_skill_overview([], existing_skill="## Keep", chat=_FakeChatClient().chat) == "## Keep" + + +def test_synthesizer_helpers() -> None: + synth = MemorySynthesizer() + assert synth._clean_markdown("```markdown\n# Hi\n```") == "# Hi" + # Skills are formatted as "## name\nbody" blocks; empty-content skills are dropped. + from memu.database.models import RecallFile + + formatted = synth._format_skills([ + RecallFile(id="a", name="b-skill", description="", content="body-b", track="skill"), + RecallFile(id="c", name="a-skill", description="", content="", track="skill"), + ]) + assert formatted == "## b-skill\nbody-b" + + +def test_build_synthesis_descriptions_uses_structured_items() -> None: + """Synthesis input is sourced from extracted items, with a caption fallback.""" + from memu.database.models import RecallEntry, Resource + + res_with_items = Resource( + id="r1", url="docs/a.txt", modality="document", local_path="a.txt", caption="raw caption a" + ) + res_without_items = Resource( + id="r2", url="docs/b.txt", modality="document", local_path="b.txt", caption="raw caption b" + ) + items = [ + RecallEntry(id="i1", resource_id="r1", memory_type="knowledge", summary="Alpha fact."), + RecallEntry(id="i2", resource_id="r1", memory_type="profile", summary="Beta trait."), + ] + + descriptions = MemoryFileExporter.build_synthesis_descriptions([res_with_items, res_without_items], items) + by_url = {d.url: d.description for d in descriptions} + + # r1 is composed from its structured items, not the caption. + assert by_url["docs/a.txt"] == "[knowledge] Alpha fact.; [profile] Beta trait." + # r2 has no items, so it falls back to the caption. + assert by_url["docs/b.txt"] == "raw caption b" + + +def _seed_skill(service: MemoryService, *, user: dict[str, str]) -> None: + """Persist one memory category and one skill-track file in scope.""" + store = service.database + store.resource_repo.create_resource( + url="docs/coffee.txt", + modality="document", + local_path="coffee.txt", + caption="The user likes pour-over coffee.", + embedding=None, + user_data=dict(user), + ) + skill = store.recall_file_repo.get_or_create_category( + name="pour-over", + description="Brew pour-over coffee", + embedding=[0.1, 0.2], + user_data=dict(user), + track="skill", + ) + store.recall_file_repo.update_category(category_id=skill.id, content="# Pour-over\nUse a 1:16 ratio.") + + +def test_exporter_override_path(tmp_path: Path) -> None: + service = MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + ) + _seed_skill(service, user={}) + exporter = MemoryFileExporter(str(tmp_path)) + + result = exporter.export( + service.database, + memory_body="## Profile\nSynthesized.", + skill_body="## Brewing\nOverview body.", + ) + + assert "MEMORY.md" in result.written + assert "SKILL.md" in result.written + # The skill/ tree mirrors memory/: one file per skill-track RecallFile. + assert "skill/pour-over.md" in result.written + assert "Synthesized." in (tmp_path / "MEMORY.md").read_text(encoding="utf-8") + assert "Overview body." in (tmp_path / "SKILL.md").read_text(encoding="utf-8") + assert "Use a 1:16 ratio." in (tmp_path / "skill" / "pour-over.md").read_text(encoding="utf-8") + + +async def test_service_synthesis_wiring(tmp_path: Path, monkeypatch) -> None: + service = MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + memory_files_config={"enabled": True, "output_dir": str(tmp_path), "synthesize": True}, + ) + _seed_skill(service, user={"user_id": "u1"}) + monkeypatch.setattr(service, "_get_llm_client", lambda *a, **k: _FakeChatClient()) + + result = await service.export_memory_files(user={"user_id": "u1"}) + + assert "MEMORY.md" in result["written"] + assert "skill/pour-over.md" in result["written"] + memory_text = (tmp_path / "MEMORY.md").read_text(encoding="utf-8") + assert "The user is a coffee enthusiast." in memory_text + skill_text = (tmp_path / "SKILL.md").read_text(encoding="utf-8") + assert "The agent can brew pour-over coffee." in skill_text + + +# -- incremental update path ------------------------------------------------- + +_UPDATE_MEMORY_MD = "## Profile\nThe user is a coffee enthusiast.\n\n## Preferences\nLikes oat milk." + + +class _InitUpdateChatClient: + """Returns init vs update payloads based on whether existing content was injected. + + The unified prompt renders ``(empty)`` when there is no prior artifact, so the + presence of that sentinel marks a from-scratch (init) call. + """ + + async def chat(self, prompt: str, system_prompt: str | None = None) -> str: + if "SKILLS in the library" in prompt: + return _SKILL_OVERVIEW + return _MEMORY_MD if "(empty)" in prompt else _UPDATE_MEMORY_MD + + +async def test_synthesize_memory_update_merges_into_existing() -> None: + synth = MemorySynthesizer() + body = await synth.synthesize_memory( + _descriptions(), + existing_memory="## Profile\nOld profile.", + chat=_InitUpdateChatClient().chat, + ) + assert "Likes oat milk." in body + + +def test_exporter_read_helpers_roundtrip(tmp_path: Path) -> None: + service = MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + ) + exporter = MemoryFileExporter(str(tmp_path)) + + assert exporter.artifacts_exist() is False + exporter.export( + service.database, + memory_body="## Profile\nSynthesized body.", + skill_body="## Skills\nSkill overview body.", + ) + + assert exporter.artifacts_exist() is True + assert exporter.read_memory_body() == "## Profile\nSynthesized body." + assert exporter.read_skill_body() == "## Skills\nSkill overview body." + + +async def test_service_init_then_update(tmp_path: Path, monkeypatch) -> None: + service = MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + memory_files_config={ + "enabled": True, + "output_dir": str(tmp_path), + "synthesize": True, + }, + ) + monkeypatch.setattr(service, "_get_llm_client", lambda *a, **k: _InitUpdateChatClient()) + _seed_skill(service, user={"user_id": "u1"}) + + repo = service.database.resource_repo + + # First pass: no tree yet -> initialization from the full store. + init = await service.export_memory_files(user={"user_id": "u1"}) + assert "skill/pour-over.md" in init["written"] + assert "coffee enthusiast" in (tmp_path / "MEMORY.md").read_text(encoding="utf-8") + + # Second pass: tree exists -> incremental update from the changed resource only. + changed = repo.create_resource( + url="docs/latte.txt", + modality="document", + local_path="latte.txt", + caption="The user enjoys latte art and oat milk.", + embedding=None, + user_data={"user_id": "u1"}, + ) + await service._build_memory_files({"user_id": "u1"}, changed=[changed]) + + memory_text = (tmp_path / "MEMORY.md").read_text(encoding="utf-8") + assert "Likes oat milk." in memory_text + # The skill file persists across the incremental update. + assert (tmp_path / "skill" / "pour-over.md").exists() diff --git a/tests/test_openai_max_tokens.py b/tests/test_openai_max_tokens.py new file mode 100644 index 0000000..039f0bb --- /dev/null +++ b/tests/test_openai_max_tokens.py @@ -0,0 +1,76 @@ +"""Regression tests for OpenAI token-limit parameter handling. + +Newer OpenAI models (GPT-5 family, the shipped defaults) reject ``max_tokens`` +and ``max_tokens=None`` outright, requiring ``max_completion_tokens`` instead. +The SDK clients must therefore omit the parameter when unset and use +``max_completion_tokens`` when a value is provided. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +from openai import omit + +from memu.llm.openai_client import OpenAIClient +from memu.vlm.openai_client import OpenAIVLMClient + + +class _RecordingCompletions: + def __init__(self) -> None: + self.kwargs: dict[str, Any] = {} + + async def create(self, **kwargs: Any) -> Any: + self.kwargs = kwargs + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))]) + + +class _RecordingOpenAI: + def __init__(self) -> None: + self.chat = SimpleNamespace(completions=_RecordingCompletions()) + + +def test_llm_chat_omits_max_tokens_when_unset() -> None: + client = OpenAIClient(base_url="https://example.test", api_key="k", chat_model="gpt-5.4-mini") + fake = _RecordingOpenAI() + client.client = fake # type: ignore[assignment] + + asyncio.run(client.chat("hi")) + + kwargs = fake.chat.completions.kwargs + # ``max_tokens`` is never sent; the token limit is omitted via the SDK + # ``omit`` sentinel (the SDK drops it from the request body). + assert "max_tokens" not in kwargs + assert kwargs.get("max_completion_tokens") is omit + + +def test_llm_chat_uses_max_completion_tokens_when_set() -> None: + client = OpenAIClient(base_url="https://example.test", api_key="k", chat_model="gpt-5.4-mini") + fake = _RecordingOpenAI() + client.client = fake # type: ignore[assignment] + + asyncio.run(client.chat("hi", max_tokens=64)) + + kwargs = fake.chat.completions.kwargs + assert kwargs.get("max_completion_tokens") == 64 + assert "max_tokens" not in kwargs + + +def test_vlm_vision_token_param(tmp_path: Path) -> None: + image = tmp_path / "x.png" + image.write_bytes(b"\x89PNG\r\n\x1a\n") + + client = OpenAIVLMClient(base_url="https://example.test", api_key="k", vlm_model="gpt-5.4") + fake = _RecordingOpenAI() + client.client = fake # type: ignore[assignment] + + asyncio.run(client.vision("describe", str(image))) + assert "max_tokens" not in fake.chat.completions.kwargs + assert fake.chat.completions.kwargs.get("max_completion_tokens") is omit + + asyncio.run(client.vision("describe", str(image), max_tokens=128)) + assert fake.chat.completions.kwargs.get("max_completion_tokens") == 128 + assert "max_tokens" not in fake.chat.completions.kwargs diff --git a/tests/test_openrouter.py b/tests/test_openrouter.py new file mode 100644 index 0000000..b598eaf --- /dev/null +++ b/tests/test_openrouter.py @@ -0,0 +1,161 @@ +""" +Test OpenRouter integration with MemU's full workflow. + +Tests: +1. Conversation memorization using OpenRouter +2. RAG-based retrieval using OpenRouter embeddings +3. LLM-based retrieval using OpenRouter + +Usage: + export OPENROUTER_API_KEY=your_api_key + python tests/test_openrouter.py +""" + +import asyncio +import json +import os +import sys +from typing import Any + +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src"))) + +from memu.app import MemoryService + + +def _print_categories(categories, max_items=3): + """Print category summaries.""" + if categories: + print(" Categories:") + for cat in categories[:max_items]: + summary = cat.get("summary") or cat.get("description", "") + print(f" - {cat.get('name')}: {summary[:60]}...") + + +def _print_items(items, max_items=3): + """Print memory item summaries.""" + if items: + print(" Items:") + for item in items[:max_items]: + memory_type = item.get("memory_type", "unknown") + summary = item.get("summary", "")[:80] + print(f" - [{memory_type}] {summary}...") + + +async def _test_memorize(service, file_path, output_data): + """Test conversation memorization.""" + print("\n[OPENROUTER] Test 1: Memorizing conversation...") + memory = await service.memorize( + resource_url=file_path, modality="conversation", user={"user_id": "openrouter_test_user"} + ) + items_count = len(memory.get("items", [])) + categories_count = len(memory.get("categories", [])) + + print(f" Memorized {items_count} items") + print(f" Created {categories_count} categories") + + output_data["memorize"] = memory + + assert items_count > 0, "Expected at least 1 memory item" + assert categories_count > 0, "Expected at least 1 category" + + _print_categories(memory.get("categories", [])) + return memory + + +async def _test_retrieve(service, queries, method, test_num, output_data): + """Test retrieval with specified method.""" + print(f"\n[OPENROUTER] Test {test_num}: {method.upper()}-based retrieval...") + service.retrieve_config.method = method + result = await service.retrieve(queries=queries, where={"user_id": "openrouter_test_user"}) + + categories_retrieved = len(result.get("categories", [])) + items_retrieved = len(result.get("items", [])) + + print(f" Retrieved {categories_retrieved} categories") + print(f" Retrieved {items_retrieved} items") + + output_data[f"retrieve_{method}"] = result + + _print_categories(result.get("categories", [])) + _print_items(result.get("items", [])) + return result + + +async def test_openrouter_full_workflow(): + """Test OpenRouter integration with full MemU workflow.""" + api_key = os.environ.get("OPENROUTER_API_KEY") + if not api_key: + pytest.skip("OPENROUTER_API_KEY environment variable not set") + + file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "example", "example_conversation.json")) + if not os.path.exists(file_path): + pytest.skip(f"Test file not found: {file_path}") + + output_data: dict[str, Any] = {} + + print("\n" + "=" * 60) + print("[OPENROUTER] Starting full workflow test...") + print("=" * 60) + + service = MemoryService( + llm_profiles={ + "default": { + "provider": "openrouter", + "client_backend": "httpx", + "base_url": "https://openrouter.ai", + "api_key": api_key, + "chat_model": "anthropic/claude-3.5-sonnet", + "embed_model": "openai/text-embedding-3-small", + }, + }, + database_config={ + "metadata_store": {"provider": "inmemory"}, + }, + retrieve_config={ + "method": "rag", + "route_intention": False, + }, + ) + + queries = [ + {"role": "user", "content": {"text": "What foods does the user like to eat?"}}, + ] + + await _test_memorize(service, file_path, output_data) + await _test_retrieve(service, queries, "rag", 2, output_data) + await _test_retrieve(service, queries, "llm", 3, output_data) + + # Test 4: List memory items + print("\n[OPENROUTER] Test 4: List memory items...") + items_result = await service.list_recall_entries(where={"user_id": "openrouter_test_user"}) + items_list = items_result.get("items", []) + print(f" Listed {len(items_list)} memory items") + output_data["list_items"] = items_result + assert len(items_list) > 0, "Expected at least 1 item in list" + + # Test 5: List memory categories + print("\n[OPENROUTER] Test 5: List memory categories...") + cats_result = await service.list_recall_files(where={"user_id": "openrouter_test_user"}) + cats_list = cats_result.get("categories", []) + print(f" Listed {len(cats_list)} categories") + output_data["list_categories"] = cats_result + assert len(cats_list) > 0, "Expected at least 1 category in list" + + # Save output to file + output_file = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "examples", "output", "openrouter_test_output.json") + ) + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, "w", encoding="utf-8") as f: + json.dump(output_data, f, indent=2, default=str) + print(f"\n[OPENROUTER] Output saved to: {output_file}") + + print("\n" + "=" * 60) + print("[OPENROUTER] All tests completed!") + print("=" * 60) + + +if __name__ == "__main__": + asyncio.run(test_openrouter_full_workflow()) diff --git a/tests/test_postgres.py b/tests/test_postgres.py new file mode 100644 index 0000000..8b375a3 --- /dev/null +++ b/tests/test_postgres.py @@ -0,0 +1,82 @@ +import os + +from memu.app import MemoryService + + +async def main(): + """Test with PostgreSQL storage.""" + api_key = os.environ.get("OPENAI_API_KEY") + # Default port 5432; use 5433 if 5432 is occupied + postgres_dsn = os.environ.get("POSTGRES_DSN", "postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu") + file_path = os.path.abspath("tests/example/example_conversation.json") + + print("\n" + "=" * 60) + print("[POSTGRES] Starting test...") + print(f"[POSTGRES] DSN: {postgres_dsn}") + print("=" * 60) + + service = MemoryService( + llm_profiles={"default": {"api_key": api_key}}, + database_config={ + "metadata_store": { + "provider": "postgres", + "dsn": postgres_dsn, + "ddl_mode": "create", + }, + # vector_index will auto-configure to pgvector + }, + retrieve_config={"method": "rag"}, + ) + + # Memorize + print("\n[POSTGRES] Memorizing...") + memory = await service.memorize(resource_url=file_path, modality="conversation", user={"user_id": "123"}) + for cat in memory.get("categories", []): + print(f" - {cat.get('name')}: {(cat.get('summary') or '')[:80]}...") + + queries = [ + {"role": "user", "content": {"text": "Tell me about preferences"}}, + {"role": "assistant", "content": {"text": "Sure, I'll tell you about their preferences"}}, + { + "role": "user", + "content": {"text": "What are they"}, + }, # This is the query that will be used to retrieve the memory, the context will be used for query rewriting + ] + + # RAG-based retrieval + print("\n[POSTGRES] RETRIEVED - RAG") + service.retrieve_config.method = "rag" + result_rag = await service.retrieve(queries=queries, where={"user_id": "123"}) + print(" Categories:") + for cat in result_rag.get("categories", [])[:3]: + print(f" - {cat.get('name')}: {(cat.get('summary') or cat.get('description', ''))[:80]}...") + print(" Items:") + for item in result_rag.get("items", [])[:3]: + print(f" - [{item.get('memory_type')}] {item.get('summary', '')[:100]}...") + if result_rag.get("resources"): + print(" Resources:") + for res in result_rag.get("resources", [])[:3]: + print(f" - [{res.get('modality')}] {res.get('url', '')[:80]}...") + + # LLM-based retrieval + print("\n[POSTGRES] RETRIEVED - LLM") + service.retrieve_config.method = "llm" + result_llm = await service.retrieve(queries=queries, where={"user_id": "123"}) + print(" Categories:") + for cat in result_llm.get("categories", [])[:3]: + print(f" - {cat.get('name')}: {(cat.get('summary') or cat.get('description', ''))[:80]}...") + print(" Items:") + for item in result_llm.get("items", [])[:3]: + print(f" - [{item.get('memory_type')}] {item.get('summary', '')[:100]}...") + if result_llm.get("resources"): + print(" Resources:") + for res in result_llm.get("resources", [])[:3]: + print(f" - [{res.get('modality')}] {res.get('url', '')[:80]}...") + + print("\n[POSTGRES] Test completed!") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/tests/test_postgres_migration_config.py b/tests/test_postgres_migration_config.py new file mode 100644 index 0000000..38c5d73 --- /dev/null +++ b/tests/test_postgres_migration_config.py @@ -0,0 +1,31 @@ +import importlib.util +from pathlib import Path + +from pydantic import BaseModel + + +def _load_migration_module(): + module_path = Path(__file__).resolve().parents[1] / "src/memu/database/postgres/migration.py" + spec = importlib.util.spec_from_file_location("memu_postgres_migration", module_path) + if spec is None or spec.loader is None: + raise RuntimeError + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _ScopeModel(BaseModel): + user_id: str + + +def test_make_alembic_config_escapes_percent_encoded_dsn() -> None: + migration = _load_migration_module() + + dsn = "postgresql+psycopg://postgres:%40%23%24%25%5E%26%2A%28%29password@host.docker.internal:5432/memu_dev" + + cfg = migration.make_alembic_config(dsn=dsn, scope_model=_ScopeModel) + raw_value = cfg.file_config.get(cfg.config_ini_section, "sqlalchemy.url", raw=True) + + assert cfg.get_main_option("sqlalchemy.url") == dsn + assert "%%40%%23%%24%%25%%5E%%26%%2A%%28%%29password" in raw_value + assert raw_value != dsn diff --git a/tests/test_references.py b/tests/test_references.py new file mode 100644 index 0000000..330a04d --- /dev/null +++ b/tests/test_references.py @@ -0,0 +1,191 @@ +""" +Tests for memory item reference functionality (Issue #202). + +Tests cover: +1. Reference extraction from text +2. Reference stripping for clean display +3. Reference formatting as citations +4. Reference-aware category summary generation +5. Reference-aware retrieval +""" + +from __future__ import annotations + +from memu.utils.references import ( + build_item_reference_map, + extract_references, + format_references_as_citations, + strip_references, +) + + +class TestExtractReferences: + """Tests for extract_references function.""" + + def test_extract_single_reference(self): + """Should extract a single reference ID.""" + text = "User loves coffee [ref:abc123]." + refs = extract_references(text) + assert refs == ["abc123"] + + def test_extract_multiple_references(self): + """Should extract multiple reference IDs in order.""" + text = "User loves coffee [ref:abc123]. Also tea [ref:def456]." + refs = extract_references(text) + assert refs == ["abc123", "def456"] + + def test_extract_comma_separated_references(self): + """Should handle comma-separated IDs in single reference.""" + text = "User prefers hot drinks [ref:abc,def,ghi]." + refs = extract_references(text) + assert refs == ["abc", "def", "ghi"] + + def test_extract_no_duplicates(self): + """Should not return duplicate IDs.""" + text = "Coffee [ref:abc]. More coffee [ref:abc]. Tea [ref:def]." + refs = extract_references(text) + assert refs == ["abc", "def"] + + def test_extract_empty_text(self): + """Should return empty list for empty text.""" + assert extract_references("") == [] + assert extract_references(None) == [] + + def test_extract_no_references(self): + """Should return empty list when no references present.""" + text = "User loves coffee and tea." + refs = extract_references(text) + assert refs == [] + + def test_extract_with_hyphens_and_underscores(self): + """Should handle IDs with hyphens and underscores.""" + text = "Info [ref:item_abc-123]." + refs = extract_references(text) + assert refs == ["item_abc-123"] + + +class TestStripReferences: + """Tests for strip_references function.""" + + def test_strip_single_reference(self): + """Should remove single reference.""" + text = "User loves coffee [ref:abc123]." + result = strip_references(text) + assert result == "User loves coffee." + + def test_strip_multiple_references(self): + """Should remove all references.""" + text = "Coffee [ref:abc]. Tea [ref:def]." + result = strip_references(text) + assert result == "Coffee. Tea." + + def test_strip_comma_separated(self): + """Should remove comma-separated references.""" + text = "Drinks [ref:abc,def,ghi]." + result = strip_references(text) + assert result == "Drinks." + + def test_strip_empty_text(self): + """Should handle empty text.""" + assert strip_references("") == "" + assert strip_references(None) is None + + def test_strip_no_references(self): + """Should return text unchanged if no references.""" + text = "User loves coffee." + result = strip_references(text) + assert result == "User loves coffee." + + +class TestFormatReferencesAsCitations: + """Tests for format_references_as_citations function.""" + + def test_format_single_citation(self): + """Should convert single reference to numbered citation.""" + text = "User loves coffee [ref:abc]." + result = format_references_as_citations(text) + assert result is not None + assert "[1]" in result + assert "[ref:abc]" not in result + assert "References:" in result + assert "[1] abc" in result + + def test_format_multiple_citations(self): + """Should number citations in order of appearance.""" + text = "Coffee [ref:abc]. Tea [ref:def]." + result = format_references_as_citations(text) + assert result is not None + assert "[1]" in result + assert "[2]" in result + assert "[1] abc" in result + assert "[2] def" in result + + def test_format_empty_text(self): + """Should handle empty text.""" + assert format_references_as_citations("") == "" + assert format_references_as_citations(None) is None + + def test_format_no_references(self): + """Should return text unchanged if no references.""" + text = "User loves coffee." + result = format_references_as_citations(text) + assert result == text + + +class TestBuildItemReferenceMap: + """Tests for build_item_reference_map function.""" + + def test_build_map_single_item(self): + """Should format single item reference.""" + items = [("abc123", "User loves coffee")] + result = build_item_reference_map(items) + assert "Available memory items for reference:" in result + assert "[ref:abc123]" in result + assert "User loves coffee" in result + + def test_build_map_multiple_items(self): + """Should format multiple item references.""" + items = [ + ("abc", "User loves coffee"), + ("def", "User prefers tea"), + ] + result = build_item_reference_map(items) + assert "[ref:abc]" in result + assert "[ref:def]" in result + + def test_build_map_truncates_long_summaries(self): + """Should truncate summaries longer than 100 chars.""" + long_summary = "x" * 150 + items = [("abc", long_summary)] + result = build_item_reference_map(items) + assert "..." in result + assert len(result.split("\n")[1]) < 150 + + def test_build_map_empty_list(self): + """Should return empty string for empty list.""" + assert build_item_reference_map([]) == "" + + +class TestReferenceIntegration: + """Integration tests for reference functionality.""" + + def test_roundtrip_extract_and_strip(self): + """Extracting then stripping should give clean text.""" + original = "User loves coffee [ref:abc]. Tea [ref:def]." + refs = extract_references(original) + clean = strip_references(original) + + assert refs == ["abc", "def"] + assert clean is not None + assert "[ref:" not in clean + assert "coffee" in clean + assert "Tea" in clean + + def test_citation_preserves_content(self): + """Citation formatting should preserve text content.""" + original = "User loves coffee [ref:abc]." + result = format_references_as_citations(original) + + assert result is not None + assert "User loves coffee" in result + assert "abc" in result # ID should be in references section diff --git a/tests/test_salience.py b/tests/test_salience.py new file mode 100644 index 0000000..bb5d84a --- /dev/null +++ b/tests/test_salience.py @@ -0,0 +1,209 @@ +""" +Tests for salience-aware memory features: +- Memory deduplication via content hash +- Reinforcement tracking +- Salience-aware retrieval ranking +""" + +from __future__ import annotations + +import hashlib +import math +from datetime import UTC, datetime, timedelta + + +# Inline implementations to avoid circular import issues during testing +def compute_content_hash(summary: str, memory_type: str) -> str: + """Generate unique hash for memory deduplication.""" + normalized = " ".join(summary.lower().split()) + content = f"{memory_type}:{normalized}" + return hashlib.sha256(content.encode()).hexdigest()[:16] + + +def salience_score( + similarity: float, + reinforcement_count: int, + last_reinforced_at: datetime | None, + recency_decay_days: float = 30.0, +) -> float: + """Compute salience-aware score combining similarity, reinforcement, and recency.""" + reinforcement_factor = math.log(reinforcement_count + 1) + + if last_reinforced_at is None: + recency_factor = 0.5 + else: + now = datetime.now(last_reinforced_at.tzinfo) if last_reinforced_at.tzinfo else datetime.now(UTC) + days_ago = (now - last_reinforced_at).total_seconds() / 86400 + recency_factor = math.exp(-0.693 * days_ago / recency_decay_days) + + return similarity * reinforcement_factor * recency_factor + + +def _cosine(a: list[float], b: list[float]) -> float: + import numpy as np + + a_arr = np.array(a, dtype=np.float32) + b_arr = np.array(b, dtype=np.float32) + denom = (np.linalg.norm(a_arr) * np.linalg.norm(b_arr)) + 1e-9 + return float(np.dot(a_arr, b_arr) / denom) + + +def cosine_topk_salience( + query_vec: list[float], + corpus: list[tuple[str, list[float] | None, int, datetime | None]], + k: int = 5, + recency_decay_days: float = 30.0, +) -> list[tuple[str, float]]: + """Top-k retrieval using salience-aware scoring.""" + scored: list[tuple[str, float]] = [] + + for _id, vec, reinforcement_count, last_reinforced_at in corpus: + if vec is None: + continue + similarity = _cosine(query_vec, vec) + score = salience_score(similarity, reinforcement_count, last_reinforced_at, recency_decay_days) + scored.append((_id, score)) + + scored.sort(key=lambda x: x[1], reverse=True) + return scored[:k] + + +class TestContentHash: + """Tests for content hash computation.""" + + def test_basic_hash(self): + """Hash should be deterministic.""" + hash1 = compute_content_hash("User loves coffee", "profile") + hash2 = compute_content_hash("User loves coffee", "profile") + assert hash1 == hash2 + assert len(hash1) == 16 # 16 hex chars + + def test_different_content_different_hash(self): + """Different content should produce different hashes.""" + hash1 = compute_content_hash("User loves coffee", "profile") + hash2 = compute_content_hash("User loves tea", "profile") + assert hash1 != hash2 + + def test_different_type_different_hash(self): + """Same content with different type should produce different hashes.""" + hash1 = compute_content_hash("User loves coffee", "profile") + hash2 = compute_content_hash("User loves coffee", "event") + assert hash1 != hash2 + + def test_whitespace_normalization(self): + """Whitespace variations should produce same hash.""" + hash1 = compute_content_hash("User loves coffee", "profile") + hash2 = compute_content_hash("User loves coffee", "profile") + hash3 = compute_content_hash(" User loves coffee ", "profile") + assert hash1 == hash2 == hash3 + + def test_case_insensitive(self): + """Hash should be case-insensitive.""" + hash1 = compute_content_hash("User loves coffee", "profile") + hash2 = compute_content_hash("USER LOVES COFFEE", "profile") + assert hash1 == hash2 + + +class TestSalienceScore: + """Tests for salience score computation.""" + + def test_basic_salience(self): + """Basic salience score should be positive.""" + score = salience_score( + similarity=0.8, + reinforcement_count=1, + last_reinforced_at=datetime.now(UTC), + recency_decay_days=30.0, + ) + assert score > 0 + + def test_higher_reinforcement_higher_score(self): + """Higher reinforcement count should increase score.""" + now = datetime.now(UTC) + score_low = salience_score(0.8, 1, now, 30.0) + score_high = salience_score(0.8, 10, now, 30.0) + assert score_high > score_low + + def test_recent_memory_higher_score(self): + """More recent memories should score higher.""" + now = datetime.now(UTC) + old = now - timedelta(days=60) + + score_recent = salience_score(0.8, 1, now, 30.0) + score_old = salience_score(0.8, 1, old, 30.0) + assert score_recent > score_old + + def test_none_last_reinforced_neutral(self): + """None last_reinforced_at should give neutral recency factor.""" + score = salience_score(0.8, 1, None, 30.0) + # With recency_factor=0.5 and reinforcement_factor=log(2)≈0.69 + # score ≈ 0.8 * 0.69 * 0.5 ≈ 0.28 + assert 0.2 < score < 0.4 + + def test_reinforcement_vs_recency_tradeoff(self): + """High reinforcement old memory vs low reinforcement recent memory.""" + now = datetime.now(UTC) + old = now - timedelta(days=30) # 30 days ago = half-life + + # Memory A: high reinforcement (10), old (30 days) + score_a = salience_score(0.85, 10, old, 30.0) + + # Memory B: low reinforcement (1), recent (now) + score_b = salience_score(0.85, 1, now, 30.0) + + # A should score higher due to reinforcement + # A: 0.85 * log(11) * 0.5 ≈ 0.85 * 2.4 * 0.5 ≈ 1.02 + # B: 0.85 * log(2) * 1.0 ≈ 0.85 * 0.69 * 1.0 ≈ 0.59 + assert score_a > score_b + + +class TestCosineTopkSalience: + """Tests for salience-aware top-k retrieval.""" + + def test_basic_retrieval(self) -> None: + """Should return top-k results sorted by salience.""" + query = [1.0, 0.0, 0.0] + now = datetime.now(UTC) + + corpus: list[tuple[str, list[float] | None, int, datetime | None]] = [ + ("id1", [1.0, 0.0, 0.0], 1, now), # Perfect match, low reinforcement + ("id2", [0.9, 0.1, 0.0], 10, now), # Good match, high reinforcement + ("id3", [0.5, 0.5, 0.0], 1, now), # Weak match + ] + + results = cosine_topk_salience(query, corpus, k=2, recency_decay_days=30.0) + + assert len(results) == 2 + # id2 should rank first due to high reinforcement despite slightly lower similarity + assert results[0][0] == "id2" + + def test_skips_none_embeddings(self) -> None: + """Should skip items with None embeddings.""" + query = [1.0, 0.0, 0.0] + now = datetime.now(UTC) + + corpus: list[tuple[str, list[float] | None, int, datetime | None]] = [ + ("id1", [1.0, 0.0, 0.0], 1, now), + ("id2", None, 10, now), # None embedding + ] + + results = cosine_topk_salience(query, corpus, k=5, recency_decay_days=30.0) + + assert len(results) == 1 + assert results[0][0] == "id1" + + def test_respects_k_limit(self) -> None: + """Should return at most k results.""" + query = [1.0, 0.0, 0.0] + now = datetime.now(UTC) + + corpus: list[tuple[str, list[float] | None, int, datetime | None]] = [ + ("id1", [1.0, 0.0, 0.0], 1, now), + ("id2", [0.9, 0.1, 0.0], 1, now), + ("id3", [0.8, 0.2, 0.0], 1, now), + ("id4", [0.7, 0.3, 0.0], 1, now), + ] + + results = cosine_topk_salience(query, corpus, k=2, recency_decay_days=30.0) + + assert len(results) == 2 diff --git a/tests/test_skill_track.py b/tests/test_skill_track.py new file mode 100644 index 0000000..7e73630 --- /dev/null +++ b/tests/test_skill_track.py @@ -0,0 +1,247 @@ +"""Tests for the resource -> file workspace memorize path (ADR 0007 phase 1). + +Exercises ``MemoryService._memorize_ws_synthesize_files`` — the two-step route + +per-file synthesis that replaces the entry plane for the chat/skill tracks — including +the ``RecallFile`` upsert and the ``resource -> file`` provenance link. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from memu.app import MemoryService + +# Router output (step a): which files to update/create for a source. +_SKILL_ROUTE = '[{"op": "create", "name": "pour-over", "description": "Brew pour-over coffee"}]' +_MEMORY_ROUTE = '[{"op": "create", "name": "Preferences", "description": "User preferences"}]' +# Synthesis output (step b): the file body. +_SKILL_BODY = "# Pour-over\nUse a 1:16 ratio." + + +class _FakeClient: + """Fake LLM/embed client that answers the route step and the synthesis step. + + The two steps are distinguished by a marker only the route prompt contains, so a + single client can serve both ``chat`` calls in the workflow. + """ + + def __init__(self, route: str = _SKILL_ROUTE, body: str = _SKILL_BODY) -> None: + self._route = route + self._body = body + + async def chat(self, prompt: str, system_prompt: str | None = None) -> str: + if "JSON array of operations" in prompt: + return self._route + return self._body + + async def embed(self, texts: list[str]) -> list[list[float]]: + return [[0.1, 0.2, 0.3] for _ in texts] + + +def _service(tmp_path: Path) -> MemoryService: + return MemoryService( + llm_profiles={"default": {"api_key": "test-key"}}, + database_config={"metadata_store": {"provider": "inmemory"}}, + memory_files_config={"enabled": True, "output_dir": str(tmp_path), "synthesize": True}, + ) + + +def _seed_resource(service: MemoryService, *, track: str, user: dict[str, Any]) -> Any: + return service.database.resource_repo.create_resource( + url=f"/w/{track}/x.md", + modality="document", + local_path=f"/w/{track}/x.md", + caption=None, + embedding=None, + user_data=dict(user), + track=track, + ) + + +async def _run_synthesize( + service: MemoryService, + client: _FakeClient, + *, + track: str, + user: dict[str, Any], + text: str = "I brewed pour-over at a 1:16 ratio.", + resource: Any | None = None, +) -> dict: + service._get_step_llm_client = lambda *a, **k: client # type: ignore[method-assign] + service._get_step_embedding_client = lambda *a, **k: client # type: ignore[method-assign] + res = resource if resource is not None else _seed_resource(service, track=track, user=user) + state = { + "resources": [res], + "preprocessed_resources": [{"text": text, "caption": None}], + "resource_track": track, + "store": service.database, + "user": user, + } + return await service._memorize_ws_synthesize_files(state, None) + + +async def test_skill_track_synthesizes_file_and_links_resource(tmp_path: Path) -> None: + service = _service(tmp_path) + store = service.database + user = {"user_id": "u1"} + res = _seed_resource(service, track="skill", user=user) + + result = await _run_synthesize(service, _FakeClient(), track="skill", user=user, resource=res) + + files = list(result["files"]) + assert len(files) == 1 + skill = files[0] + assert skill.name == "pour-over" + assert skill.track == "skill" + assert skill.description == "Brew pour-over coffee" + assert skill.content == _SKILL_BODY + + # Persisted as a skill-track RecallFile, isolated from the memory track. + skill_files = store.recall_file_repo.list_categories(where={"user_id": "u1", "track": "skill"}) + assert [f.name for f in skill_files.values()] == ["pour-over"] + assert store.recall_file_repo.list_categories(where={"user_id": "u1", "track": "memory"}) == {} + + # A resource -> file provenance link was recorded. + links = store.recall_file_resource_repo.list_relations(where=user) + assert len(links) == 1 + assert links[0].resource_id == res.id + assert links[0].file_id == skill.id + + +async def test_chat_track_routes_to_memory_track_file(tmp_path: Path) -> None: + service = _service(tmp_path) + store = service.database + user = {"user_id": "u1"} + + result = await _run_synthesize( + service, + _FakeClient(route=_MEMORY_ROUTE, body="## Preferences\nLikes strong coffee."), + track="chat", + user=user, + text="I really like strong coffee.", + ) + + files = list(result["files"]) + assert len(files) == 1 + assert files[0].name == "Preferences" + assert files[0].track == "memory" + assert store.recall_file_repo.list_categories(where={"user_id": "u1", "track": "skill"}) == {} + + +async def test_update_op_revises_existing_file_by_name(tmp_path: Path) -> None: + service = _service(tmp_path) + store = service.database + user: dict[str, Any] = {} + + await _run_synthesize(service, _FakeClient(), track="skill", user=user) + # A second source updates the same skill by exact name. + revised = _FakeClient(route='[{"op": "update", "name": "pour-over"}]', body="# Pour-over\nUpdated.") + await _run_synthesize(service, revised, track="skill", user=user) + + skill_files = store.recall_file_repo.list_categories(where={"track": "skill"}) + assert len(skill_files) == 1 # revised in place, not duplicated + assert next(iter(skill_files.values())).content == "# Pour-over\nUpdated." + + +async def test_workspace_track_is_resource_only_noop(tmp_path: Path) -> None: + service = _service(tmp_path) + store = service.database + user = {"user_id": "u1"} + + result = await _run_synthesize(service, _FakeClient(), track="workspace", user=user) + + assert result["files"] == [] + assert store.recall_file_repo.list_categories(where={"user_id": "u1"}) == {} + assert store.recall_file_resource_repo.list_relations(where=user) == [] + + +async def test_empty_source_is_noop(tmp_path: Path) -> None: + service = _service(tmp_path) + store = service.database + user = {"user_id": "u1"} + + result = await _run_synthesize(service, _FakeClient(), track="skill", user=user, text=" ") + + assert result["files"] == [] + assert store.recall_file_repo.list_categories(where={"user_id": "u1", "track": "skill"}) == {} + + +async def test_skill_track_creates_single_name_description_segment(tmp_path: Path) -> None: + service = _service(tmp_path) + store = service.database + user = {"user_id": "u1"} + + result = await _run_synthesize(service, _FakeClient(), track="skill", user=user) + skill = next(iter(result["files"])) + + segments = store.recall_file_segment_repo.list_segments_for_file(skill.id) + assert len(segments) == 1 + assert segments[0].text == "name: pour-over\ndescription: Brew pour-over coffee" + assert segments[0].embedding == [0.1, 0.2, 0.3] + # Segment track mirrors the owning file's track (denormalized for filtering). + assert segments[0].track == "skill" + + +async def test_memory_track_segments_are_lines_skipping_headings(tmp_path: Path) -> None: + service = _service(tmp_path) + store = service.database + user = {"user_id": "u1"} + + result = await _run_synthesize( + service, + _FakeClient(route=_MEMORY_ROUTE, body="## Preferences\nLikes strong coffee.\n\nDrinks it black."), + track="chat", + user=user, + ) + file = next(iter(result["files"])) + + segments = store.recall_file_segment_repo.list_segments_for_file(file.id) + assert [s.text for s in segments] == ["Likes strong coffee.", "Drinks it black."] + # Segment track mirrors the owning file's track (chat routes to the "memory" track). + assert all(s.track == "memory" for s in segments) + + +async def test_memory_segments_drop_and_add_on_update(tmp_path: Path) -> None: + service = _service(tmp_path) + store = service.database + user = {"user_id": "u1"} + + first = await _run_synthesize( + service, + _FakeClient(route=_MEMORY_ROUTE, body="## P\nline a\nline b"), + track="chat", + user=user, + ) + file = next(iter(first["files"])) + before = {s.text: s.id for s in store.recall_file_segment_repo.list_segments_for_file(file.id)} + assert set(before) == {"line a", "line b"} + + # An update changes only one line: "line b" -> "line c". + await _run_synthesize( + service, + _FakeClient(route='[{"op": "update", "name": "Preferences"}]', body="## P\nline a\nline c"), + track="chat", + user=user, + ) + after = {s.text: s.id for s in store.recall_file_segment_repo.list_segments_for_file(file.id)} + assert set(after) == {"line a", "line c"} + # Unchanged line keeps its original segment (not re-embedded); changed line is fresh. + assert after["line a"] == before["line a"] + assert "line b" not in after + + +async def test_update_op_for_unknown_file_is_dropped(tmp_path: Path) -> None: + service = _service(tmp_path) + store = service.database + user = {"user_id": "u1"} + + result = await _run_synthesize( + service, + _FakeClient(route='[{"op": "update", "name": "does-not-exist"}]'), + track="skill", + user=user, + ) + + assert result["files"] == [] + assert store.recall_file_repo.list_categories(where={"user_id": "u1", "track": "skill"}) == {} diff --git a/tests/test_sqlite.py b/tests/test_sqlite.py new file mode 100644 index 0000000..3031c56 --- /dev/null +++ b/tests/test_sqlite.py @@ -0,0 +1,90 @@ +"""Test SQLite database backend for MemU.""" + +import os +import tempfile + +from memu.app import MemoryService + + +def _print_results(title: str, result: dict) -> None: + print(f"\n[SQLITE] RETRIEVED - {title}") + print(" Categories:") + for cat in result.get("categories", [])[:3]: + print(f" - {cat.get('name')}: {(cat.get('summary') or cat.get('description', ''))[:80]}...") + print(" Items:") + for item in result.get("items", [])[:3]: + print(f" - [{item.get('memory_type')}] {item.get('summary', '')[:100]}...") + if result.get("resources"): + print(" Resources:") + for res in result.get("resources", [])[:3]: + print(f" - [{res.get('modality')}] {res.get('url', '')[:80]}...") + + +async def main(): + """Test with SQLite storage.""" + api_key = os.environ.get("OPENAI_API_KEY") + file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "example", "example_conversation.json")) + + # Create a temporary SQLite database file + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + sqlite_path = tmp.name + + sqlite_dsn = f"sqlite:///{sqlite_path}" + + print("\n" + "=" * 60) + print("[SQLITE] Starting test...") + print(f"[SQLITE] DSN: {sqlite_dsn}") + print("=" * 60) + + try: + service = MemoryService( + llm_profiles={"default": {"api_key": api_key}}, + database_config={ + "metadata_store": { + "provider": "sqlite", + "dsn": sqlite_dsn, + }, + # SQLite uses brute-force vector search + "vector_index": {"provider": "bruteforce"}, + }, + retrieve_config={"method": "rag"}, + ) + + # Memorize + print("\n[SQLITE] Memorizing...") + memory = await service.memorize(resource_url=file_path, modality="conversation", user={"user_id": "123"}) + for cat in memory.get("categories", []): + print(f" - {cat.get('name')}: {(cat.get('summary') or '')[:80]}...") + + queries = [ + {"role": "user", "content": {"text": "Tell me about preferences"}}, + {"role": "assistant", "content": {"text": "Sure, I'll tell you about their preferences"}}, + { + "role": "user", + "content": {"text": "What are they"}, + }, # This is the query that will be used to retrieve the memory + ] + + # RAG-based retrieval + service.retrieve_config.method = "rag" + result_rag = await service.retrieve(queries=queries, where={"user_id": "123"}) + _print_results("RAG", result_rag) + + # LLM-based retrieval + service.retrieve_config.method = "llm" + result_llm = await service.retrieve(queries=queries, where={"user_id": "123"}) + _print_results("LLM", result_llm) + + print("\n[SQLITE] Test completed!") + + finally: + # Clean up the temporary database file + if os.path.exists(sqlite_path): + os.unlink(sqlite_path) + print(f"[SQLITE] Cleaned up temporary database: {sqlite_path}") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/tests/test_tool_memory.py b/tests/test_tool_memory.py new file mode 100644 index 0000000..975a577 --- /dev/null +++ b/tests/test_tool_memory.py @@ -0,0 +1,327 @@ +"""Tests for Tool Memory feature - specialized memory type for tracking tool usage.""" + +from __future__ import annotations + +import importlib.util +import sys +from datetime import datetime +from pathlib import Path +from typing import Any + +# Add src to path for direct import - MUST be before any memu imports +src_path = Path(__file__).parent.parent / "src" +if str(src_path) not in sys.path: + sys.path.insert(0, str(src_path)) + +import pytest # noqa: E402 + +# Import directly from the models file path to avoid circular import through database/__init__.py +# We use importlib to import the module directly without triggering the package __init__ +spec = importlib.util.spec_from_file_location("models", src_path / "memu" / "database" / "models.py") +assert spec is not None +assert spec.loader is not None +models = importlib.util.module_from_spec(spec) +spec.loader.exec_module(models) + +# Rebuild models to resolve forward references with proper namespace +rebuild_ns = { + "Any": Any, + "datetime": datetime, + "EntryType": models.EntryType, + "ToolCallResult": models.ToolCallResult, +} +models.ToolCallResult.model_rebuild(_types_namespace=rebuild_ns) +models.RecallEntry.model_rebuild(_types_namespace=rebuild_ns) + +RecallEntry = models.RecallEntry +EntryType = models.EntryType +ToolCallResult = models.ToolCallResult + +# Import tool memory utility functions +util_tool_spec = importlib.util.spec_from_file_location("util_tool", src_path / "memu" / "utils" / "tool.py") +assert util_tool_spec is not None +assert util_tool_spec.loader is not None +util_tool = importlib.util.module_from_spec(util_tool_spec) +util_tool_spec.loader.exec_module(util_tool) + +add_tool_call = util_tool.add_tool_call +get_tool_statistics = util_tool.get_tool_statistics + + +class TestToolCallResult: + """Tests for ToolCallResult model.""" + + def test_create_tool_call_result(self): + """Test creating a basic ToolCallResult.""" + result = ToolCallResult( + tool_name="file_reader", + input={"path": "/data/config.json"}, + output="File content here", + success=True, + time_cost=0.5, + token_cost=100, + score=0.95, + ) + + assert result.tool_name == "file_reader" + assert result.input == {"path": "/data/config.json"} + assert result.output == "File content here" + assert result.success is True + assert result.time_cost == 0.5 + assert result.token_cost == 100 + assert result.score == 0.95 + + def test_generate_hash(self): + """Test hash generation for deduplication.""" + result = ToolCallResult( + tool_name="calculator", + input={"a": 1, "b": 2}, + output="3", + ) + + hash1 = result.generate_hash() + assert hash1 != "" + assert len(hash1) == 32 # MD5 hex digest length + + # Same input/output should generate same hash + result2 = ToolCallResult( + tool_name="calculator", + input={"a": 1, "b": 2}, + output="3", + ) + assert result2.generate_hash() == hash1 + + # Different input should generate different hash + result3 = ToolCallResult( + tool_name="calculator", + input={"a": 2, "b": 3}, + output="5", + ) + assert result3.generate_hash() != hash1 + + def test_ensure_hash(self): + """Test ensure_hash sets call_hash if empty.""" + result = ToolCallResult( + tool_name="test_tool", + input="test input", + output="test output", + ) + + assert result.call_hash == "" + result.ensure_hash() + assert result.call_hash != "" + assert len(result.call_hash) == 32 + + def test_string_input(self): + """Test ToolCallResult with string input.""" + result = ToolCallResult( + tool_name="echo", + input="hello world", + output="hello world", + ) + + result.ensure_hash() + assert result.call_hash != "" + + +class TestRecallEntryToolType: + """Tests for RecallEntry with tool type.""" + + def test_tool_memory_type_literal(self): + """Test that 'tool' is a valid EntryType.""" + from typing import get_args + + valid_types = get_args(EntryType) + assert "tool" in valid_types + + def test_create_tool_memory(self): + """Test creating a tool type memory item with tool fields in extra.""" + item = RecallEntry( + resource_id=None, + memory_type="tool", + summary="file_reader tool usage for config files", + extra={ + "when_to_use": "When needing to read configuration files", + "metadata": {"tool_name": "file_reader", "avg_success_rate": 0.95}, + }, + ) + + assert item.memory_type == "tool" + assert item.extra["when_to_use"] == "When needing to read configuration files" + assert item.extra["metadata"]["tool_name"] == "file_reader" + + def test_add_tool_call(self): + """Test adding tool call results to a tool memory.""" + item = RecallEntry( + resource_id=None, + memory_type="tool", + summary="calculator tool usage", + ) + + tool_call = ToolCallResult( + tool_name="calculator", + input={"a": 1, "b": 2}, + output="3", + success=True, + time_cost=0.1, + score=1.0, + ) + + add_tool_call(item, tool_call) + + tool_calls = item.extra.get("tool_calls", []) + assert len(tool_calls) == 1 + assert tool_calls[0]["tool_name"] == "calculator" + assert tool_calls[0]["call_hash"] != "" # ensure_hash was called + + def test_add_tool_call_wrong_type(self): + """Test that add_tool_call fails for non-tool memories.""" + item = RecallEntry( + resource_id=None, + memory_type="profile", + summary="User profile info", + ) + + tool_call = ToolCallResult( + tool_name="test", + input="test", + output="test", + ) + + with pytest.raises(ValueError, match="can only be used with tool type memories"): + add_tool_call(item, tool_call) + + def test_get_tool_statistics_empty(self): + """Test statistics for memory with no tool calls.""" + item = RecallEntry( + resource_id=None, + memory_type="tool", + summary="empty tool memory", + ) + + stats = get_tool_statistics(item) + + assert stats["total_calls"] == 0 + assert stats["recent_calls_analyzed"] == 0 + assert stats["avg_time_cost"] == 0.0 + assert stats["success_rate"] == 0.0 + assert stats["avg_score"] == 0.0 + assert stats["avg_token_cost"] == 0.0 + + def test_get_tool_statistics(self): + """Test statistics calculation for tool calls.""" + # Tool calls are stored as dicts in extra + item = RecallEntry( + resource_id=None, + memory_type="tool", + summary="calculator tool", + extra={ + "tool_calls": [ + { + "tool_name": "calc", + "input": "1+1", + "output": "2", + "success": True, + "time_cost": 0.1, + "score": 1.0, + "token_cost": 10, + }, + { + "tool_name": "calc", + "input": "2+2", + "output": "4", + "success": True, + "time_cost": 0.2, + "score": 0.9, + "token_cost": 15, + }, + { + "tool_name": "calc", + "input": "bad", + "output": "error", + "success": False, + "time_cost": 0.5, + "score": 0.0, + "token_cost": 5, + }, + ] + }, + ) + + stats = get_tool_statistics(item) + + assert stats["total_calls"] == 3 + assert stats["recent_calls_analyzed"] == 3 + assert stats["success_rate"] == pytest.approx(0.6667, rel=0.01) # 2/3 + assert stats["avg_time_cost"] == pytest.approx(0.267, rel=0.01) # (0.1+0.2+0.5)/3 + assert stats["avg_score"] == pytest.approx(0.633, rel=0.01) # (1.0+0.9+0.0)/3 + assert stats["avg_token_cost"] == pytest.approx(10.0, rel=0.01) # (10+15+5)/3 + + def test_get_tool_statistics_recent_n(self): + """Test statistics with recent_n limit.""" + item = RecallEntry( + resource_id=None, + memory_type="tool", + summary="tool with many calls", + extra={ + "tool_calls": [ + {"tool_name": "t", "input": "1", "output": "1", "success": False, "time_cost": 1.0, "score": 0.0}, + {"tool_name": "t", "input": "2", "output": "2", "success": True, "time_cost": 0.1, "score": 1.0}, + {"tool_name": "t", "input": "3", "output": "3", "success": True, "time_cost": 0.1, "score": 1.0}, + ] + }, + ) + + # Only analyze last 2 calls + stats = get_tool_statistics(item, recent_n=2) + + assert stats["total_calls"] == 3 + assert stats["recent_calls_analyzed"] == 2 + assert stats["success_rate"] == 1.0 # Both recent calls succeeded + + +class TestRecallEntryNewFields: + """Tests for tool-related fields stored in extra.""" + + def test_when_to_use_field(self): + """Test when_to_use field stored in extra for retrieval hints.""" + item = RecallEntry( + resource_id=None, + memory_type="profile", + summary="User prefers dark mode", + extra={"when_to_use": "When configuring UI settings or themes"}, + ) + + assert item.extra["when_to_use"] == "When configuring UI settings or themes" + + def test_metadata_field(self): + """Test metadata field stored in extra for type-specific data.""" + item = RecallEntry( + resource_id=None, + memory_type="event", + summary="User attended conference", + extra={ + "metadata": { + "event_date": "2026-01-15", + "location": "San Francisco", + "attendees": ["Alice", "Bob"], + } + }, + ) + + assert item.extra.get("metadata") is not None + assert item.extra["metadata"]["event_date"] == "2026-01-15" + assert item.extra["metadata"]["location"] == "San Francisco" + assert len(item.extra["metadata"]["attendees"]) == 2 + + def test_default_values(self): + """Test that extra defaults to empty dict.""" + item = RecallEntry( + resource_id=None, + memory_type="knowledge", + summary="Python is a programming language", + ) + + assert item.extra.get("when_to_use") is None + assert item.extra.get("metadata") is None + assert item.extra.get("tool_calls") is None diff --git a/tests/test_vector.py b/tests/test_vector.py new file mode 100644 index 0000000..0bcaca0 --- /dev/null +++ b/tests/test_vector.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +from memu.database.inmemory.vector import cosine_topk, cosine_topk_salience + + +def _corpus() -> list[tuple[str, list[float]]]: + return [("a", [1.0, 0.0]), ("b", [0.0, 1.0]), ("c", [0.7, 0.7])] + + +def test_cosine_topk_nonpositive_k_returns_empty() -> None: + # top_k <= 0 must return nothing, not the entire corpus (which is what the + # argpartition path did for k == 0). + assert cosine_topk([1.0, 0.0], _corpus(), k=0) == [] + assert cosine_topk([1.0, 0.0], _corpus(), k=-1) == [] + + +def test_cosine_topk_orders_by_similarity() -> None: + results = cosine_topk([1.0, 0.0], _corpus(), k=2) + assert [memory_id for memory_id, _ in results] == ["a", "c"] + + +def test_cosine_topk_salience_nonpositive_k_returns_empty() -> None: + now = datetime.now(UTC) + corpus = [("a", [1.0, 0.0], 1, now), ("b", [0.0, 1.0], 1, now)] + assert cosine_topk_salience([1.0, 0.0], corpus, k=0) == [] + assert cosine_topk_salience([1.0, 0.0], corpus, k=-1) == [] diff --git a/tests/test_vlm_preprocess.py b/tests/test_vlm_preprocess.py new file mode 100644 index 0000000..69e53b8 --- /dev/null +++ b/tests/test_vlm_preprocess.py @@ -0,0 +1,217 @@ +"""Tests for VLM (vision-language) wiring into image/video preprocessing. + +Covers: +- ``vlm_config_from_llm`` derivation (provider/credentials reuse, model pick). +- Image/video preprocessors using the VLM client (not the chat LLM client). +- ``MemoryService`` routing vision modalities to the VLM client and text + modalities to the chat LLM client. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from memu.app.service import MemoryService +from memu.app.settings import LLMConfig, vlm_config_from_llm +from memu.preprocess import preprocess_resource +from memu.preprocess.base import PreprocessContext +from memu.vlm.backends.openrouter import OpenRouterVLMBackend +from memu.vlm.http_client import HTTPVLMClient + + +class _RecordingVisionClient: + """Vision client that records calls and returns a tagged response.""" + + def __init__(self, name: str) -> None: + self.name = name + self.vision_calls: list[str] = [] + + async def vision(self, prompt: str, image_path: str, *, system_prompt: str | None = None, **_: Any) -> str: + self.vision_calls.append(image_path) + return "a catcat" + + +class _RecordingVideoClient: + """VLM client with native video support that records image/video calls.""" + + supports_video = True + + def __init__(self) -> None: + self.video_calls: list[str] = [] + self.vision_calls: list[str] = [] + + async def video(self, prompt: str, video_path: str, *, system_prompt: str | None = None, **_: Any) -> str: + self.video_calls.append(video_path) + return "a dog playing fetchdog playing" + + async def vision(self, prompt: str, image_path: str, *, system_prompt: str | None = None, **_: Any) -> str: + self.vision_calls.append(image_path) + return "a frameframe" + + +def _make_ctx(*, llm_client: Any, vlm_client: Any) -> PreprocessContext: + return PreprocessContext( + get_llm_client=lambda: llm_client, + get_vlm_client=lambda: vlm_client, + escape_prompt_value=lambda s: s, + extract_json_blob=lambda s: s, + resolve_custom_prompt=lambda _p, _v: "", + multimodal_preprocess_prompts={}, + ) + + +def test_vlm_config_from_llm_openai_sdk() -> None: + cfg = vlm_config_from_llm(LLMConfig()) + assert cfg.provider == "openai" + assert cfg.client_backend == "sdk" + assert cfg.vlm_model == "gpt-5.4" + + +def test_vlm_config_from_llm_claude_http_reuses_credentials() -> None: + llm = LLMConfig(provider="claude", client_backend="httpx", api_key="secret") + cfg = vlm_config_from_llm(llm) + assert cfg.provider == "claude" + assert cfg.client_backend == "httpx" + assert cfg.api_key == "secret" + assert cfg.base_url == llm.base_url + assert cfg.vlm_model == "claude-sonnet-4-6" + + +def test_vlm_config_from_llm_anthropic_backend_maps_provider() -> None: + # The anthropic SDK backend leaves provider generic; it must still resolve a + # Claude VLM model rather than the OpenAI default. + cfg = vlm_config_from_llm(LLMConfig(client_backend="anthropic")) + assert cfg.provider == "claude" + assert cfg.vlm_model == "claude-sonnet-4-6" + + +def test_vlm_config_unknown_provider_falls_back_to_chat_model() -> None: + # DeepSeek has no first-party VLM; fall back to the configured chat model. + llm = LLMConfig(provider="deepseek", client_backend="httpx") + cfg = vlm_config_from_llm(llm) + assert cfg.vlm_model == llm.chat_model + + +def test_image_preprocess_uses_vlm_client() -> None: + llm = _RecordingVisionClient("llm") + vlm = _RecordingVisionClient("vlm") + ctx = _make_ctx(llm_client=llm, vlm_client=vlm) + image_path = "/workspace/x.png" + + result = asyncio.run(preprocess_resource(modality="image", local_path=image_path, text=None, ctx=ctx)) + + assert vlm.vision_calls == [image_path] + assert llm.vision_calls == [] + assert result[0]["text"] == "a cat" + + +def test_service_routes_vision_modalities_to_vlm(monkeypatch: Any) -> None: + svc = MemoryService() + captured: dict[str, Any] = {} + + async def _fake_preprocess(*, local_path: str, text: str | None, modality: str, llm_client: Any) -> list: + captured["client"] = llm_client + return [] + + monkeypatch.setattr(svc, "_preprocess_resource_url", _fake_preprocess) + monkeypatch.setattr(svc, "_get_vlm_client", lambda *a, **k: "VLM_CLIENT") + monkeypatch.setattr(svc, "_get_step_llm_client", lambda *a, **k: "CHAT_CLIENT") + resource_path = "/workspace/x" + + async def _run(modality: str) -> Any: + state = {"local_path": resource_path, "raw_text": None, "modality": modality} + await svc._memorize_preprocess_multimodal(state, {}) + return captured["client"] + + assert asyncio.run(_run("image")) == "VLM_CLIENT" + assert asyncio.run(_run("video")) == "VLM_CLIENT" + assert asyncio.run(_run("document")) == "CHAT_CLIENT" + assert asyncio.run(_run("conversation")) == "CHAT_CLIENT" + + +def test_service_falls_back_to_chat_client_when_vlm_profile_missing(monkeypatch: Any) -> None: + svc = MemoryService() + captured: dict[str, Any] = {} + + async def _fake_preprocess(*, local_path: str, text: str | None, modality: str, llm_client: Any) -> list: + captured["client"] = llm_client + return [] + + monkeypatch.setattr(svc, "_preprocess_resource_url", _fake_preprocess) + monkeypatch.setattr(svc, "_get_vlm_client", lambda *a, **k: (_ for _ in ()).throw(KeyError("missing profile"))) + monkeypatch.setattr(svc, "_get_step_llm_client", lambda *a, **k: "CHAT_CLIENT") + + state = {"local_path": "/workspace/x", "raw_text": None, "modality": "image"} + asyncio.run(svc._memorize_preprocess_multimodal(state, {})) + + assert captured["client"] == "CHAT_CLIENT" + + +def test_vlm_config_from_llm_openrouter_uses_video_capable_model() -> None: + # An OpenRouter LLM profile (httpx transport) derives a VLM config that keeps + # the httpx backend and defaults to a video-capable model so native + # whole-video understanding works out of the box. + cfg = vlm_config_from_llm(LLMConfig(provider="openrouter", client_backend="httpx")) + assert cfg.provider == "openrouter" + assert cfg.client_backend == "httpx" + assert cfg.vlm_model == "minimax/minimax-m3" + + +def test_openrouter_backend_supports_native_video() -> None: + backend = OpenRouterVLMBackend() + assert backend.supports_video is True + + payload = backend.build_video_payload( + prompt="Describe this video.", + video_data_uri="data:video/mp4;base64,QUJD", + system_prompt=None, + vlm_model="minimax/minimax-m3", + max_tokens=None, + ) + content = payload["messages"][0]["content"] + video_part = next(part for part in content if part["type"] == "video_url") + assert video_part["video_url"]["url"] == "data:video/mp4;base64,QUJD" + + +def test_http_vlm_client_exposes_backend_video_capability() -> None: + # OpenRouter advertises native video; plain OpenAI-compatible does not. + openrouter = HTTPVLMClient( + base_url="https://openrouter.ai", api_key="k", vlm_model="minimax/minimax-m3", provider="openrouter" + ) + assert openrouter.supports_video is True + + openai = HTTPVLMClient(base_url="https://api.openai.com/v1", api_key="k", vlm_model="gpt-5.4", provider="openai") + assert openai.supports_video is False + + +def test_video_uses_native_video_when_supported() -> None: + # A video-capable client analyzes the whole video file directly; no frame + # extraction / image vision call happens. + client = _RecordingVideoClient() + ctx = _make_ctx(llm_client=client, vlm_client=client) + video_path = "/workspace/clip.mp4" + + result = asyncio.run( + preprocess_resource(modality="video", local_path=video_path, text=None, ctx=ctx, llm_client=client) + ) + + assert client.video_calls == [video_path] + assert client.vision_calls == [] + assert result[0]["text"] == "a dog playing fetch" + assert result[0]["caption"] == "dog playing" + + +def test_video_without_native_support_skips_no_frame_fallback() -> None: + # A client without native video support must NOT degrade to middle-frame image + # analysis: the video is skipped (no description) and vision() is never called. + client = _RecordingVisionClient("vlm") + ctx = _make_ctx(llm_client=client, vlm_client=client) + + result = asyncio.run( + preprocess_resource(modality="video", local_path="/workspace/clip.mp4", text=None, ctx=ctx, llm_client=client) + ) + + assert client.vision_calls == [] + assert result[0]["text"] is None + assert result[0]["caption"] is None diff --git a/tests/utils/test_conversation.py b/tests/utils/test_conversation.py new file mode 100644 index 0000000..7d954c0 --- /dev/null +++ b/tests/utils/test_conversation.py @@ -0,0 +1,91 @@ +import json + +import pytest + +from memu.utils.conversation import format_conversation_for_preprocess # type: ignore[import-untyped] + + +class TestFormatConversationForPreprocess: + """ + Test suite for format_conversation_for_preprocess function in src/memu/utils/conversation.py. + + Covers: + - Happy Path: Valid JSON input (list or dict wrapper). + - Edge Cases: Empty input, empty JSON structures. + - Error Handling: Invalid JSON (current implementation handles gracefully by returning raw text). + - Type Safety: Unexpected JSON types. + """ + + @pytest.mark.parametrize( + "input_json,expected_output", + [ + # Happy Path: Standard usage with list of messages + ( + json.dumps([ + {"role": "user", "content": "Hello world", "created_at": "2023-10-27T10:00:00"}, + {"role": "assistant", "content": "Hello! How can I help?", "created_at": "2023-10-27T10:00:05"}, + ]), + "[0] 2023-10-27T10:00:00 [user]: Hello world\n[1] 2023-10-27T10:00:05 [assistant]: Hello! How can I help?", + ), + # Happy Path: Dict wrapper with 'content' key + (json.dumps({"content": [{"role": "user", "content": "Wrapper test"}]}), "[0] [user]: Wrapper test"), + # Happy Path: Missing optional fields (role defaults to user, created_at omitted) + (json.dumps([{"content": "Just text"}]), "[0] [user]: Just text"), + # Happy Path: Multiline content should be collapsed + ( + json.dumps([{"role": "system", "content": "Line 1\nLine 2\nLine 3"}]), + "[0] [system]: Line 1 Line 2 Line 3", + ), + # Happy Path: Content is None/Null + (json.dumps([{"role": "user", "content": None}]), "[0] [user]: "), + # Happy Path: Content is a dict with 'text' + (json.dumps([{"role": "user", "content": {"text": "Rich content"}}]), "[0] [user]: Rich content"), + ], + ) + def test_happy_path_valid_formats(self, input_json: str, expected_output: str) -> None: + """ + Test that valid JSON inputs are correctly formatted into the expected line-based string. + """ + result = format_conversation_for_preprocess(input_json) + assert result == expected_output + + @pytest.mark.parametrize( + "edge_input,expected", + [ + ("", ""), # Empty string + (" ", ""), # Whitespace only + ("[]", ""), # Empty JSON list -> produces empty string + ], + ) + def test_edge_cases_empty(self, edge_input: str, expected: str) -> None: + """ + Test edge cases handling for empty or whitespace-only inputs, and empty JSON lists. + """ + assert format_conversation_for_preprocess(edge_input) == expected + + def test_malformed_json_handling(self) -> None: + """ + Test handling of malformed JSON strings. + + Note: The implementation swallows JSONDecodeError and returns raw text. + This test verifies that graceful fallback behavior. + """ + malformed_json = '{"role": "user", "content": "Missing brace"' + result = format_conversation_for_preprocess(malformed_json) + # Expecting raw text back as fallback + assert result == malformed_json + + def test_unexpected_json_structures(self) -> None: + """ + Test handling of valid JSON that does not match expected conversation schema. + Expectation: Returns raw text if schema extraction fails. + """ + # Empty dict -> _extract_messages returns None + assert format_conversation_for_preprocess("{}") == "{}" + + # Random non-message JSON + random_json = json.dumps({"key": "value"}) + assert format_conversation_for_preprocess(random_json) == random_json + + # Valid JSON primitives + assert format_conversation_for_preprocess("123") == "123" diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..2fd0649 --- /dev/null +++ b/uv.lock @@ -0,0 +1,2912 @@ +version = 1 +revision = 2 +requires-python = ">=3.13" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform != 'win32'", +] + +[[package]] +name = "alembic" +version = "1.17.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/a6/74c8cadc2882977d80ad756a13857857dbcf9bd405bc80b662eb10651282/alembic-1.17.2.tar.gz", hash = "sha256:bbe9751705c5e0f14877f02d46c53d10885e377e3d90eda810a016f9baa19e8e", size = 1988064, upload-time = "2025-11-14T20:35:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/88/6237e97e3385b57b5f1528647addea5cc03d4d65d5979ab24327d41fb00d/alembic-1.17.2-py3-none-any.whl", hash = "sha256:f483dd1fe93f6c5d49217055e4d15b905b425b6af906746abb35b69c1996c4e6", size = 248554, upload-time = "2025-11-14T20:35:05.699Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anthropic" +version = "0.111.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/8a/9afc7305a2ce4b52b30e137f83cd2a6a90b918b3997073db11bb5a1de55a/anthropic-0.111.0.tar.gz", hash = "sha256:39cbda0ac17a6d423e5bf609811bd69b26eddf6299d7a468126e05bc711ce826", size = 934001, upload-time = "2026-06-18T17:31:44.733Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/bb/09e82a81885d787f350fb55ca9df865b63140dd28b3b5b3104c4ae261657/anthropic-0.111.0-py3-none-any.whl", hash = "sha256:c14edb36ed80da9099acbd26b5cec810d76606c31f32a0d56a4cf9d4fa9e25ae", size = 929774, upload-time = "2026-06-18T17:31:43.116Z" }, +] + +[[package]] +name = "anyio" +version = "4.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "sniffio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/78/7d432127c41b50bccba979505f272c16cbcadcc33645d5fa3a738110ae75/anyio-4.11.0.tar.gz", hash = "sha256:82a8d0b81e318cc5ce71a5f1f8b5c4e63619620b63141ef8c995fa0db95a57c4", size = 219094, upload-time = "2025-09-23T09:19:12.58Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, +] + +[[package]] +name = "attrs" +version = "25.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/5c/685e6633917e101e5dcb62b9dd76946cbb57c26e133bae9e0cd36033c0a9/attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11", size = 934251, upload-time = "2025-10-06T13:54:44.725Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, +] + +[[package]] +name = "babel" +version = "2.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/6b/d52e42361e1aa00709585ecc30b3f9684b3ab62530771402248b1b1d6240/babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d", size = 9951852, upload-time = "2025-02-01T15:17:41.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, +] + +[[package]] +name = "backrefs" +version = "5.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/a7/312f673df6a79003279e1f55619abbe7daebbb87c17c976ddc0345c04c7b/backrefs-5.9.tar.gz", hash = "sha256:808548cb708d66b82ee231f962cb36faaf4f2baab032f2fbb783e9c2fdddaa59", size = 5765857, upload-time = "2025-06-22T19:34:13.97Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/4d/798dc1f30468134906575156c089c492cf79b5a5fd373f07fe26c4d046bf/backrefs-5.9-py310-none-any.whl", hash = "sha256:db8e8ba0e9de81fcd635f440deab5ae5f2591b54ac1ebe0550a2ca063488cd9f", size = 380267, upload-time = "2025-06-22T19:34:05.252Z" }, + { url = "https://files.pythonhosted.org/packages/55/07/f0b3375bf0d06014e9787797e6b7cc02b38ac9ff9726ccfe834d94e9991e/backrefs-5.9-py311-none-any.whl", hash = "sha256:6907635edebbe9b2dc3de3a2befff44d74f30a4562adbb8b36f21252ea19c5cf", size = 392072, upload-time = "2025-06-22T19:34:06.743Z" }, + { url = "https://files.pythonhosted.org/packages/9d/12/4f345407259dd60a0997107758ba3f221cf89a9b5a0f8ed5b961aef97253/backrefs-5.9-py312-none-any.whl", hash = "sha256:7fdf9771f63e6028d7fee7e0c497c81abda597ea45d6b8f89e8ad76994f5befa", size = 397947, upload-time = "2025-06-22T19:34:08.172Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/fa31834dc27a7f05e5290eae47c82690edc3a7b37d58f7fb35a1bdbf355b/backrefs-5.9-py313-none-any.whl", hash = "sha256:cc37b19fa219e93ff825ed1fed8879e47b4d89aa7a1884860e2db64ccd7c676b", size = 399843, upload-time = "2025-06-22T19:34:09.68Z" }, + { url = "https://files.pythonhosted.org/packages/fc/24/b29af34b2c9c41645a9f4ff117bae860291780d73880f449e0b5d948c070/backrefs-5.9-py314-none-any.whl", hash = "sha256:df5e169836cc8acb5e440ebae9aad4bf9d15e226d3bad049cf3f6a5c20cc8dc9", size = 411762, upload-time = "2025-06-22T19:34:11.037Z" }, + { url = "https://files.pythonhosted.org/packages/41/ff/392bff89415399a979be4a65357a41d92729ae8580a66073d8ec8d810f98/backrefs-5.9-py39-none-any.whl", hash = "sha256:f48ee18f6252b8f5777a22a00a09a85de0ca931658f1dd96d4406a34f3748c60", size = 380265, upload-time = "2025-06-22T19:34:12.405Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + +[[package]] +name = "certifi" +version = "2025.10.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/5b/b6ce21586237c77ce67d01dc5507039d444b630dd76611bbca2d8e5dcd91/certifi-2025.10.5.tar.gz", hash = "sha256:47c09d31ccf2acf0be3f701ea53595ee7e0b8fa08801c6624be771df09ae7b43", size = 164519, upload-time = "2025-10-05T04:12:15.808Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl", hash = "sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de", size = 163286, upload-time = "2025-10-05T04:12:14.03Z" }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, +] + +[[package]] +name = "cfgv" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/74/539e56497d9bd1d484fd863dd69cbbfa653cd2aa27abfe35653494d85e94/cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560", size = 7114, upload-time = "2023-08-12T20:38:17.776Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249, upload-time = "2023-08-12T20:38:16.269Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "claude-agent-sdk" +version = "0.1.24" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "mcp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/13/a9d577f7be428773f973dedffc94d8ee4d202b962abcfdfef130c301bac2/claude_agent_sdk-0.1.24.tar.gz", hash = "sha256:2a4ed0557f805aa7df23af1b06e460796eafcf8733deefd92d1f4700ab579e8d", size = 57115, upload-time = "2026-01-28T07:09:48.629Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/ef/3defd9febfb0360f810f35a04789af8b01a47319a2649a6c11985a298fbf/claude_agent_sdk-0.1.24-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6a8880968e42e9a4910c59311b829ac68807cba98cc89dba1f7f639afbce8989", size = 54593877, upload-time = "2026-01-28T07:09:33.934Z" }, + { url = "https://files.pythonhosted.org/packages/4a/68/05e66023dce86bd0729fc8fb908e4f6b85252bbc1cf3905fa3455e72ee4c/claude_agent_sdk-0.1.24-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:f0e13991848576ca77a8729d986be771d862d8e0b57222bd0b274d2751e29a58", size = 68779931, upload-time = "2026-01-28T07:09:37.887Z" }, + { url = "https://files.pythonhosted.org/packages/4c/b4/69bd45738264727dfce2e0cd5461ebe87f1f8562b9dde0ec68b6aa507873/claude_agent_sdk-0.1.24-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:da46ff847f2329f9ede852c916943181efbbc0e6de3803a233e66ff885b2a259", size = 70481571, upload-time = "2026-01-28T07:09:42.011Z" }, + { url = "https://files.pythonhosted.org/packages/a4/47/1f1549c1571ed59da5c4a3409b0bb172c9992278d5679dd3f6a5d7d78820/claude_agent_sdk-0.1.24-py3-none-win_amd64.whl", hash = "sha256:de246603ba2adf911ae398a5082ae65f308ff7833f9b391b3e6f2e14b468ce56", size = 72664018, upload-time = "2026-01-28T07:09:46.127Z" }, +] + +[[package]] +name = "click" +version = "8.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/d3/9dcc0f5797f070ec8edf30fbadfb200e71d9db6b84d211e3b2085a7589a0/click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc", size = 107295, upload-time = "2025-09-18T17:32:22.42Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + +[[package]] +name = "cobble" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/7a/a507c709be2c96e1bb6102eb7b7f4026c5e5e223ef7d745a17d239e9d844/cobble-0.1.4.tar.gz", hash = "sha256:de38be1539992c8a06e569630717c485a5f91be2192c461ea2b220607dfa78aa", size = 3805, upload-time = "2024-06-01T18:11:09.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/e1/3714a2f371985215c219c2a70953d38e3eed81ef165aed061d21de0e998b/cobble-0.1.4-py3-none-any.whl", hash = "sha256:36c91b1655e599fd428e2b95fdd5f0da1ca2e9f1abb0bc871dec21a0e78a2b44", size = 3984, upload-time = "2024-06-01T18:11:07.911Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coloredlogs" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "humanfriendly" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934", size = 46018, upload-time = "2021-06-11T10:22:42.561Z" }, +] + +[[package]] +name = "coverage" +version = "7.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/89/12/3e2d2ec71796e0913178478e693a06af6a3bc9f7f9cb899bf85a426d8370/coverage-7.11.1.tar.gz", hash = "sha256:b4b3a072559578129a9e863082a2972a2abd8975bc0e2ec57da96afcd6580a8a", size = 814037, upload-time = "2025-11-07T10:52:41.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/01/0c50c318f5e8f1a482da05d788d0ff06137803ed8fface4a1ba51e04b3ad/coverage-7.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:da9930594ca99d66eb6f613d7beba850db2f8dfa86810ee35ae24e4d5f2bb97d", size = 216920, upload-time = "2025-11-07T10:50:55.992Z" }, + { url = "https://files.pythonhosted.org/packages/20/11/9f038e6c2baea968c377ab355b0d1d0a46b5f38985691bf51164e1b78c1f/coverage-7.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cc47a280dc014220b0fc6e5f55082a3f51854faf08fd9635b8a4f341c46c77d3", size = 217301, upload-time = "2025-11-07T10:50:57.609Z" }, + { url = "https://files.pythonhosted.org/packages/68/cd/9dcf93d81d0cddaa0bba90c3b4580e6f1ddf833918b816930d250cc553a4/coverage-7.11.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:74003324321bbf130939146886eddf92e48e616b5910215e79dea6edeb8ee7c8", size = 248277, upload-time = "2025-11-07T10:50:59.442Z" }, + { url = "https://files.pythonhosted.org/packages/11/f5/b2c7c494046c9c783d3cac4c812fc24d6104dd36a7a598e7dd6fea3e7927/coverage-7.11.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:211f7996265daab60a8249af4ca6641b3080769cbedcffc42cc4841118f3a305", size = 250871, upload-time = "2025-11-07T10:51:01.094Z" }, + { url = "https://files.pythonhosted.org/packages/a5/5a/b359649566954498aa17d7c98093182576d9e435ceb4ea917b3b48d56f86/coverage-7.11.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70619d194d8fea0cb028cb6bb9c85b519c7509c1d1feef1eea635183bc8ecd27", size = 252115, upload-time = "2025-11-07T10:51:03.087Z" }, + { url = "https://files.pythonhosted.org/packages/f3/17/3cef1ede3739622950f0737605353b797ec564e70c9d254521b10f4b03ba/coverage-7.11.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e0208bb59d441cfa3321569040f8e455f9261256e0df776c5462a1e5a9b31e13", size = 248442, upload-time = "2025-11-07T10:51:04.888Z" }, + { url = "https://files.pythonhosted.org/packages/5f/63/d5854c47ae42d9d18855329db6bc528f5b7f4f874257edb00cf8b483f9f8/coverage-7.11.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:545714d8765bda1c51f8b1c96e0b497886a054471c68211e76ef49dd1468587d", size = 250253, upload-time = "2025-11-07T10:51:06.515Z" }, + { url = "https://files.pythonhosted.org/packages/48/e8/c7706f8a5358a59c18b489e7e19e83d6161b7c8bc60771f95920570c94a8/coverage-7.11.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d0a2b02c1e20158dd405054bcca87f91fd5b7605626aee87150819ea616edd67", size = 248217, upload-time = "2025-11-07T10:51:08.405Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c9/a2136dfb168eb09e2f6d9d6b6c986243fdc0b3866a9376adb263d3c3378b/coverage-7.11.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e0f4aa986a4308a458e0fb572faa3eb3db2ea7ce294604064b25ab32b435a468", size = 248040, upload-time = "2025-11-07T10:51:10.626Z" }, + { url = "https://files.pythonhosted.org/packages/18/9a/a63991c0608ddc6adf65e6f43124951aaf36bd79f41937b028120b8268ea/coverage-7.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d51cc6687e8bbfd1e041f52baed0f979cd592242cf50bf18399a7e03afc82d88", size = 249801, upload-time = "2025-11-07T10:51:12.63Z" }, + { url = "https://files.pythonhosted.org/packages/84/19/947acf7c0c6e90e4ec3abf474133ed36d94407d07e36eafdfd3acb59fee9/coverage-7.11.1-cp313-cp313-win32.whl", hash = "sha256:1b3067db3afe6deeca2b2c9f0ec23820d5f1bd152827acfadf24de145dfc5f66", size = 219430, upload-time = "2025-11-07T10:51:14.329Z" }, + { url = "https://files.pythonhosted.org/packages/35/54/36fef7afb3884450c7b6d494fcabe2fab7c669d547c800ca30f41c1dc212/coverage-7.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:39a4c44b0cd40e3c9d89b2b7303ebd6ab9ae8a63f9e9a8c4d65a181a0b33aebe", size = 220239, upload-time = "2025-11-07T10:51:16.418Z" }, + { url = "https://files.pythonhosted.org/packages/d3/dc/7d38bb99e8e69200b7dd5de15507226bd90eac102dfc7cc891b9934cdc76/coverage-7.11.1-cp313-cp313-win_arm64.whl", hash = "sha256:a2e3560bf82fa8169a577e054cbbc29888699526063fee26ea59ea2627fd6e73", size = 218868, upload-time = "2025-11-07T10:51:18.186Z" }, + { url = "https://files.pythonhosted.org/packages/36/c6/d1ff54fbd6bcad42dbcfd13b417e636ef84aae194353b1ef3361700f2525/coverage-7.11.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:47a4f362a10285897ab3aa7a4b37d28213a4f2626823923613d6d7a3584dd79a", size = 217615, upload-time = "2025-11-07T10:51:21.065Z" }, + { url = "https://files.pythonhosted.org/packages/73/f9/6ed59e7cf1488d6f975e5b14ef836f5e537913523e92175135f8518a83ce/coverage-7.11.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:0df35fa7419ef571db9dacd50b0517bc54dbfe37eb94043b5fc3540bff276acd", size = 217960, upload-time = "2025-11-07T10:51:22.797Z" }, + { url = "https://files.pythonhosted.org/packages/c4/74/2dab1dc2ebe16f074f80ae483b0f45faf278d102be703ac01b32cd85b6c3/coverage-7.11.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e1a2c621d341c9d56f7917e56fbb56be4f73fe0d0e8dae28352fb095060fd467", size = 259262, upload-time = "2025-11-07T10:51:24.467Z" }, + { url = "https://files.pythonhosted.org/packages/15/49/eccfe039663e29a50a54b0c2c8d076acd174d7ac50d018ef8a5b1c37c8dc/coverage-7.11.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c354b111be9b2234d9573d75dd30ca4e414b7659c730e477e89be4f620b3fb5", size = 261326, upload-time = "2025-11-07T10:51:26.232Z" }, + { url = "https://files.pythonhosted.org/packages/f0/bb/2b829aa23fd5ee8318e33cc02a606eb09900921291497963adc3f06af8bb/coverage-7.11.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4589bd44698728f600233fb2881014c9b8ec86637ef454c00939e779661dbe7e", size = 263758, upload-time = "2025-11-07T10:51:27.912Z" }, + { url = "https://files.pythonhosted.org/packages/ac/03/d44c3d70e5da275caf2cad2071da6b425412fbcb1d1d5a81f1f89b45e3f1/coverage-7.11.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c6956fc8754f2309131230272a7213a483a32ecbe29e2b9316d808a28f2f8ea1", size = 258444, upload-time = "2025-11-07T10:51:30.107Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c1/cf61d9f46ae088774c65dd3387a15dfbc72de90c1f6e105025e9eda19b42/coverage-7.11.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:63926a97ed89dc6a087369b92dcb8b9a94cead46c08b33a7f1f4818cd8b6a3c3", size = 261335, upload-time = "2025-11-07T10:51:31.814Z" }, + { url = "https://files.pythonhosted.org/packages/95/9a/b3299bb14f11f2364d78a2b9704491b15395e757af6116694731ce4e5834/coverage-7.11.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f5311ba00c53a7fb2b293fdc1f478b7286fe2a845a7ba9cda053f6e98178f0b4", size = 258951, upload-time = "2025-11-07T10:51:33.925Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a3/73cb2763e59f14ba6d8d6444b1f640a9be2242bfb59b7e50581c695db7ff/coverage-7.11.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:31bf5ffad84c974f9e72ac53493350f36b6fa396109159ec704210698f12860b", size = 257840, upload-time = "2025-11-07T10:51:36.092Z" }, + { url = "https://files.pythonhosted.org/packages/85/db/482e72589a952027e238ffa3a15f192c552e0685fd0c5220ad05b5f17d56/coverage-7.11.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:227ee59fbc4a8c57a7383a1d7af6ca94a78ae3beee4045f38684548a8479a65b", size = 260040, upload-time = "2025-11-07T10:51:38.277Z" }, + { url = "https://files.pythonhosted.org/packages/18/a1/b931d3ee099c2dca8e9ea56c07ae84c0f91562f7bbbcccab8c91b3474ef1/coverage-7.11.1-cp313-cp313t-win32.whl", hash = "sha256:a447d97b3ce680bb1da2e6bd822ebb71be6a1fb77ce2c2ad2fe4bd8aacec3058", size = 220102, upload-time = "2025-11-07T10:51:40.017Z" }, + { url = "https://files.pythonhosted.org/packages/9a/53/b553b7bfa6207def4918f0cb72884c844fa4c3f1566e58fbb4f34e54cdc5/coverage-7.11.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d6d11180437c67bde2248563a42b8e5bbf85c8df78fae13bf818ad17bfb15f02", size = 221166, upload-time = "2025-11-07T10:51:41.921Z" }, + { url = "https://files.pythonhosted.org/packages/6b/45/1c1d58b3ed585598764bd2fe41fcf60ccafe15973ad621c322ba52e22d32/coverage-7.11.1-cp313-cp313t-win_arm64.whl", hash = "sha256:1e19a4c43d612760c6f7190411fb157e2d8a6dde00c91b941d43203bd3b17f6f", size = 219439, upload-time = "2025-11-07T10:51:43.753Z" }, + { url = "https://files.pythonhosted.org/packages/d9/c2/ac2c3417eaa4de1361036ebbc7da664242b274b2e00c4b4a1cfc7b29920b/coverage-7.11.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0305463c45c5f21f0396cd5028de92b1f1387e2e0756a85dd3147daa49f7a674", size = 216967, upload-time = "2025-11-07T10:51:45.55Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a3/afef455d03c468ee303f9df9a6f407e8bea64cd576fca914ff888faf52ca/coverage-7.11.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fa4d468d5efa1eb6e3062be8bd5f45cbf28257a37b71b969a8c1da2652dfec77", size = 217298, upload-time = "2025-11-07T10:51:47.31Z" }, + { url = "https://files.pythonhosted.org/packages/9d/59/6e2fb3fb58637001132dc32228b4fb5b332d75d12f1353cb00fe084ee0ba/coverage-7.11.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d2b2f5fc8fe383cbf2d5c77d6c4b2632ede553bc0afd0cdc910fa5390046c290", size = 248337, upload-time = "2025-11-07T10:51:49.48Z" }, + { url = "https://files.pythonhosted.org/packages/1d/5e/ce442bab963e3388658da8bde6ddbd0a15beda230afafaa25e3c487dc391/coverage-7.11.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bde6488c1ad509f4fb1a4f9960fd003d5a94adef61e226246f9699befbab3276", size = 250853, upload-time = "2025-11-07T10:51:51.215Z" }, + { url = "https://files.pythonhosted.org/packages/d1/2f/43f94557924ca9b64e09f1c3876da4eec44a05a41e27b8a639d899716c0e/coverage-7.11.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a69e0d6fa0b920fe6706a898c52955ec5bcfa7e45868215159f45fd87ea6da7c", size = 252190, upload-time = "2025-11-07T10:51:53.262Z" }, + { url = "https://files.pythonhosted.org/packages/8c/fa/a04e769b92bc5628d4bd909dcc3c8219efe5e49f462e29adc43e198ecfde/coverage-7.11.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:976e51e4a549b80e4639eda3a53e95013a14ff6ad69bb58ed604d34deb0e774c", size = 248335, upload-time = "2025-11-07T10:51:55.388Z" }, + { url = "https://files.pythonhosted.org/packages/99/d0/b98ab5d2abe425c71117a7c690ead697a0b32b83256bf0f566c726b7f77b/coverage-7.11.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d61fcc4d384c82971a3d9cf00d0872881f9ded19404c714d6079b7a4547e2955", size = 250209, upload-time = "2025-11-07T10:51:57.263Z" }, + { url = "https://files.pythonhosted.org/packages/9c/3f/b9c4fbd2e6d1b64098f99fb68df7f7c1b3e0a0968d24025adb24f359cdec/coverage-7.11.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:284c5df762b533fae3ebd764e3b81c20c1c9648d93ef34469759cb4e3dfe13d0", size = 248163, upload-time = "2025-11-07T10:51:59.014Z" }, + { url = "https://files.pythonhosted.org/packages/08/fc/3e4d54fb6368b0628019eefd897fc271badbd025410fd5421a65fb58758f/coverage-7.11.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:bab32cb1d4ad2ac6dcc4e17eee5fa136c2a1d14ae914e4bce6c8b78273aece3c", size = 247983, upload-time = "2025-11-07T10:52:01.027Z" }, + { url = "https://files.pythonhosted.org/packages/b9/4a/a5700764a12e932b35afdddb2f59adbca289c1689455d06437f609f3ef35/coverage-7.11.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:36f2fed9ce392ca450fb4e283900d0b41f05c8c5db674d200f471498be3ce747", size = 249646, upload-time = "2025-11-07T10:52:02.856Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2c/45ed33d9e80a1cc9b44b4bd535d44c154d3204671c65abd90ec1e99522a2/coverage-7.11.1-cp314-cp314-win32.whl", hash = "sha256:853136cecb92a5ba1cc8f61ec6ffa62ca3c88b4b386a6c835f8b833924f9a8c5", size = 219700, upload-time = "2025-11-07T10:52:05.05Z" }, + { url = "https://files.pythonhosted.org/packages/90/d7/5845597360f6434af1290118ebe114642865f45ce47e7e822d9c07b371be/coverage-7.11.1-cp314-cp314-win_amd64.whl", hash = "sha256:77443d39143e20927259a61da0c95d55ffc31cf43086b8f0f11a92da5260d592", size = 220516, upload-time = "2025-11-07T10:52:07.259Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d0/d311a06f9cf7a48a98ffcfd0c57db0dcab6da46e75c439286a50dc648161/coverage-7.11.1-cp314-cp314-win_arm64.whl", hash = "sha256:829acb88fa47591a64bf5197e96a931ce9d4b3634c7f81a224ba3319623cdf6c", size = 219091, upload-time = "2025-11-07T10:52:09.216Z" }, + { url = "https://files.pythonhosted.org/packages/a7/3d/c6a84da4fa9b840933045b19dd19d17b892f3f2dd1612903260291416dba/coverage-7.11.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2ad1fe321d9522ea14399de83e75a11fb6a8887930c3679feb383301c28070d9", size = 217700, upload-time = "2025-11-07T10:52:11.348Z" }, + { url = "https://files.pythonhosted.org/packages/94/10/a4fc5022017dd7ac682dc423849c241dfbdad31734b8f96060d84e70b587/coverage-7.11.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f69c332f0c3d1357c74decc9b1843fcd428cf9221bf196a20ad22aa1db3e1b6c", size = 217968, upload-time = "2025-11-07T10:52:13.203Z" }, + { url = "https://files.pythonhosted.org/packages/59/2d/a554cd98924d296de5816413280ac3b09e42a05fb248d66f8d474d321938/coverage-7.11.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:576baeea4eebde684bf6c91c01e97171c8015765c8b2cfd4022a42b899897811", size = 259334, upload-time = "2025-11-07T10:52:15.079Z" }, + { url = "https://files.pythonhosted.org/packages/05/98/d484cb659ec33958ca96b6f03438f56edc23b239d1ad0417b7a97fc1848a/coverage-7.11.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:28ad84c694fa86084cfd3c1eab4149844b8cb95bd8e5cbfc4a647f3ee2cce2b3", size = 261445, upload-time = "2025-11-07T10:52:17.134Z" }, + { url = "https://files.pythonhosted.org/packages/f3/fa/920cba122cc28f4557c0507f8bd7c6e527ebcc537d0309186f66464a8fd9/coverage-7.11.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b1043ff958f09fc3f552c014d599f3c6b7088ba97d7bc1bd1cce8603cd75b520", size = 263858, upload-time = "2025-11-07T10:52:19.836Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a0/036397bdbee0f3bd46c2e26fdfbb1a61b2140bf9059240c37b61149047fa/coverage-7.11.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c6681add5060c2742dafcf29826dff1ff8eef889a3b03390daeed84361c428bd", size = 258381, upload-time = "2025-11-07T10:52:21.687Z" }, + { url = "https://files.pythonhosted.org/packages/b6/61/2533926eb8990f182eb287f4873216c8ca530cc47241144aabf46fe80abe/coverage-7.11.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:773419b225ec9a75caa1e941dd0c83a91b92c2b525269e44e6ee3e4c630607db", size = 261321, upload-time = "2025-11-07T10:52:23.612Z" }, + { url = "https://files.pythonhosted.org/packages/32/6e/618f7e203a998e4f6b8a0fa395744a416ad2adbcdc3735bc19466456718a/coverage-7.11.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a9cb272a0e0157dbb9b2fd0b201b759bd378a1a6138a16536c025c2ce4f7643b", size = 258933, upload-time = "2025-11-07T10:52:25.514Z" }, + { url = "https://files.pythonhosted.org/packages/22/40/6b1c27f772cb08a14a338647ead1254a57ee9dabbb4cacbc15df7f278741/coverage-7.11.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:e09adb2a7811dc75998eef68f47599cf699e2b62eed09c9fefaeb290b3920f34", size = 257756, upload-time = "2025-11-07T10:52:27.845Z" }, + { url = "https://files.pythonhosted.org/packages/73/07/f9cd12f71307a785ea15b009c8d8cc2543e4a867bd04b8673843970b6b43/coverage-7.11.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1335fa8c2a2fea49924d97e1e3500cfe8d7c849f5369f26bb7559ad4259ccfab", size = 260086, upload-time = "2025-11-07T10:52:29.776Z" }, + { url = "https://files.pythonhosted.org/packages/34/02/31c5394f6f5d72a466966bcfdb61ce5a19862d452816d6ffcbb44add16ee/coverage-7.11.1-cp314-cp314t-win32.whl", hash = "sha256:4782d71d2a4fa7cef95e853b7097c8bbead4dbd0e6f9c7152a6b11a194b794db", size = 220483, upload-time = "2025-11-07T10:52:31.752Z" }, + { url = "https://files.pythonhosted.org/packages/7f/96/81e1ef5fbfd5090113a96e823dbe055e4c58d96ca73b1fb0ad9d26f9ec36/coverage-7.11.1-cp314-cp314t-win_amd64.whl", hash = "sha256:939f45e66eceb63c75e8eb8fc58bb7077c00f1a41b0e15c6ef02334a933cfe93", size = 221592, upload-time = "2025-11-07T10:52:33.724Z" }, + { url = "https://files.pythonhosted.org/packages/38/7a/a5d050de44951ac453a2046a0f3fb5471a4a557f0c914d00db27d543d94c/coverage-7.11.1-cp314-cp314t-win_arm64.whl", hash = "sha256:01c575bdbef35e3f023b50a146e9a75c53816e4f2569109458155cd2315f87d9", size = 219627, upload-time = "2025-11-07T10:52:36.285Z" }, + { url = "https://files.pythonhosted.org/packages/76/32/bd9f48c28e23b2f08946f8e83983617b00619f5538dbd7e1045fa7e88c00/coverage-7.11.1-py3-none-any.whl", hash = "sha256:0fa848acb5f1da24765cee840e1afe9232ac98a8f9431c6112c15b34e880b9e8", size = 208689, upload-time = "2025-11-07T10:52:38.646Z" }, +] + +[[package]] +name = "cryptography" +version = "46.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/19/f748958276519adf6a0c1e79e7b8860b4830dda55ccdf29f2719b5fc499c/cryptography-46.0.4.tar.gz", hash = "sha256:bfd019f60f8abc2ed1b9be4ddc21cfef059c841d86d710bb69909a688cbb8f59", size = 749301, upload-time = "2026-01-28T00:24:37.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/99/157aae7949a5f30d51fcb1a9851e8ebd5c74bf99b5285d8bb4b8b9ee641e/cryptography-46.0.4-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:281526e865ed4166009e235afadf3a4c4cba6056f99336a99efba65336fd5485", size = 7173686, upload-time = "2026-01-28T00:23:07.515Z" }, + { url = "https://files.pythonhosted.org/packages/87/91/874b8910903159043b5c6a123b7e79c4559ddd1896e38967567942635778/cryptography-46.0.4-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5f14fba5bf6f4390d7ff8f086c566454bff0411f6d8aa7af79c88b6f9267aecc", size = 4275871, upload-time = "2026-01-28T00:23:09.439Z" }, + { url = "https://files.pythonhosted.org/packages/c0/35/690e809be77896111f5b195ede56e4b4ed0435b428c2f2b6d35046fbb5e8/cryptography-46.0.4-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:47bcd19517e6389132f76e2d5303ded6cf3f78903da2158a671be8de024f4cd0", size = 4423124, upload-time = "2026-01-28T00:23:11.529Z" }, + { url = "https://files.pythonhosted.org/packages/1a/5b/a26407d4f79d61ca4bebaa9213feafdd8806dc69d3d290ce24996d3cfe43/cryptography-46.0.4-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:01df4f50f314fbe7009f54046e908d1754f19d0c6d3070df1e6268c5a4af09fa", size = 4277090, upload-time = "2026-01-28T00:23:13.123Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d8/4bb7aec442a9049827aa34cee1aa83803e528fa55da9a9d45d01d1bb933e/cryptography-46.0.4-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5aa3e463596b0087b3da0dbe2b2487e9fc261d25da85754e30e3b40637d61f81", size = 4947652, upload-time = "2026-01-28T00:23:14.554Z" }, + { url = "https://files.pythonhosted.org/packages/2b/08/f83e2e0814248b844265802d081f2fac2f1cbe6cd258e72ba14ff006823a/cryptography-46.0.4-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0a9ad24359fee86f131836a9ac3bffc9329e956624a2d379b613f8f8abaf5255", size = 4455157, upload-time = "2026-01-28T00:23:16.443Z" }, + { url = "https://files.pythonhosted.org/packages/0a/05/19d849cf4096448779d2dcc9bb27d097457dac36f7273ffa875a93b5884c/cryptography-46.0.4-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:dc1272e25ef673efe72f2096e92ae39dea1a1a450dd44918b15351f72c5a168e", size = 3981078, upload-time = "2026-01-28T00:23:17.838Z" }, + { url = "https://files.pythonhosted.org/packages/e6/89/f7bac81d66ba7cde867a743ea5b37537b32b5c633c473002b26a226f703f/cryptography-46.0.4-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:de0f5f4ec8711ebc555f54735d4c673fc34b65c44283895f1a08c2b49d2fd99c", size = 4276213, upload-time = "2026-01-28T00:23:19.257Z" }, + { url = "https://files.pythonhosted.org/packages/da/9f/7133e41f24edd827020ad21b068736e792bc68eecf66d93c924ad4719fb3/cryptography-46.0.4-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:eeeb2e33d8dbcccc34d64651f00a98cb41b2dc69cef866771a5717e6734dfa32", size = 4912190, upload-time = "2026-01-28T00:23:21.244Z" }, + { url = "https://files.pythonhosted.org/packages/a6/f7/6d43cbaddf6f65b24816e4af187d211f0bc536a29961f69faedc48501d8e/cryptography-46.0.4-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:3d425eacbc9aceafd2cb429e42f4e5d5633c6f873f5e567077043ef1b9bbf616", size = 4454641, upload-time = "2026-01-28T00:23:22.866Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4f/ebd0473ad656a0ac912a16bd07db0f5d85184924e14fc88feecae2492834/cryptography-46.0.4-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91627ebf691d1ea3976a031b61fb7bac1ccd745afa03602275dda443e11c8de0", size = 4405159, upload-time = "2026-01-28T00:23:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/d1/f7/7923886f32dc47e27adeff8246e976d77258fd2aa3efdd1754e4e323bf49/cryptography-46.0.4-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2d08bc22efd73e8854b0b7caff402d735b354862f1145d7be3b9c0f740fef6a0", size = 4666059, upload-time = "2026-01-28T00:23:26.766Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a7/0fca0fd3591dffc297278a61813d7f661a14243dd60f499a7a5b48acb52a/cryptography-46.0.4-cp311-abi3-win32.whl", hash = "sha256:82a62483daf20b8134f6e92898da70d04d0ef9a75829d732ea1018678185f4f5", size = 3026378, upload-time = "2026-01-28T00:23:28.317Z" }, + { url = "https://files.pythonhosted.org/packages/2d/12/652c84b6f9873f0909374864a57b003686c642ea48c84d6c7e2c515e6da5/cryptography-46.0.4-cp311-abi3-win_amd64.whl", hash = "sha256:6225d3ebe26a55dbc8ead5ad1265c0403552a63336499564675b29eb3184c09b", size = 3478614, upload-time = "2026-01-28T00:23:30.275Z" }, + { url = "https://files.pythonhosted.org/packages/b9/27/542b029f293a5cce59349d799d4d8484b3b1654a7b9a0585c266e974a488/cryptography-46.0.4-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:485e2b65d25ec0d901bca7bcae0f53b00133bf3173916d8e421f6fddde103908", size = 7116417, upload-time = "2026-01-28T00:23:31.958Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f5/559c25b77f40b6bf828eabaf988efb8b0e17b573545edb503368ca0a2a03/cryptography-46.0.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:078e5f06bd2fa5aea5a324f2a09f914b1484f1d0c2a4d6a8a28c74e72f65f2da", size = 4264508, upload-time = "2026-01-28T00:23:34.264Z" }, + { url = "https://files.pythonhosted.org/packages/49/a1/551fa162d33074b660dc35c9bc3616fefa21a0e8c1edd27b92559902e408/cryptography-46.0.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dce1e4f068f03008da7fa51cc7abc6ddc5e5de3e3d1550334eaf8393982a5829", size = 4409080, upload-time = "2026-01-28T00:23:35.793Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6a/4d8d129a755f5d6df1bbee69ea2f35ebfa954fa1847690d1db2e8bca46a5/cryptography-46.0.4-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2067461c80271f422ee7bdbe79b9b4be54a5162e90345f86a23445a0cf3fd8a2", size = 4270039, upload-time = "2026-01-28T00:23:37.263Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f5/ed3fcddd0a5e39321e595e144615399e47e7c153a1fb8c4862aec3151ff9/cryptography-46.0.4-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:c92010b58a51196a5f41c3795190203ac52edfd5dc3ff99149b4659eba9d2085", size = 4926748, upload-time = "2026-01-28T00:23:38.884Z" }, + { url = "https://files.pythonhosted.org/packages/43/ae/9f03d5f0c0c00e85ecb34f06d3b79599f20630e4db91b8a6e56e8f83d410/cryptography-46.0.4-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:829c2b12bbc5428ab02d6b7f7e9bbfd53e33efd6672d21341f2177470171ad8b", size = 4442307, upload-time = "2026-01-28T00:23:40.56Z" }, + { url = "https://files.pythonhosted.org/packages/8b/22/e0f9f2dae8040695103369cf2283ef9ac8abe4d51f68710bec2afd232609/cryptography-46.0.4-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:62217ba44bf81b30abaeda1488686a04a702a261e26f87db51ff61d9d3510abd", size = 3959253, upload-time = "2026-01-28T00:23:42.827Z" }, + { url = "https://files.pythonhosted.org/packages/01/5b/6a43fcccc51dae4d101ac7d378a8724d1ba3de628a24e11bf2f4f43cba4d/cryptography-46.0.4-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:9c2da296c8d3415b93e6053f5a728649a87a48ce084a9aaf51d6e46c87c7f2d2", size = 4269372, upload-time = "2026-01-28T00:23:44.655Z" }, + { url = "https://files.pythonhosted.org/packages/17/b7/0f6b8c1dd0779df2b526e78978ff00462355e31c0a6f6cff8a3e99889c90/cryptography-46.0.4-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9b34d8ba84454641a6bf4d6762d15847ecbd85c1316c0a7984e6e4e9f748ec2e", size = 4891908, upload-time = "2026-01-28T00:23:46.48Z" }, + { url = "https://files.pythonhosted.org/packages/83/17/259409b8349aa10535358807a472c6a695cf84f106022268d31cea2b6c97/cryptography-46.0.4-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:df4a817fa7138dd0c96c8c8c20f04b8aaa1fac3bbf610913dcad8ea82e1bfd3f", size = 4441254, upload-time = "2026-01-28T00:23:48.403Z" }, + { url = "https://files.pythonhosted.org/packages/9c/fe/e4a1b0c989b00cee5ffa0764401767e2d1cf59f45530963b894129fd5dce/cryptography-46.0.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b1de0ebf7587f28f9190b9cb526e901bf448c9e6a99655d2b07fff60e8212a82", size = 4396520, upload-time = "2026-01-28T00:23:50.26Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/ba8fd9657d27076eb40d6a2f941b23429a3c3d2f56f5a921d6b936a27bc9/cryptography-46.0.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9b4d17bc7bd7cdd98e3af40b441feaea4c68225e2eb2341026c84511ad246c0c", size = 4651479, upload-time = "2026-01-28T00:23:51.674Z" }, + { url = "https://files.pythonhosted.org/packages/00/03/0de4ed43c71c31e4fe954edd50b9d28d658fef56555eba7641696370a8e2/cryptography-46.0.4-cp314-cp314t-win32.whl", hash = "sha256:c411f16275b0dea722d76544a61d6421e2cc829ad76eec79280dbdc9ddf50061", size = 3001986, upload-time = "2026-01-28T00:23:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/5c/70/81830b59df7682917d7a10f833c4dab2a5574cd664e86d18139f2b421329/cryptography-46.0.4-cp314-cp314t-win_amd64.whl", hash = "sha256:728fedc529efc1439eb6107b677f7f7558adab4553ef8669f0d02d42d7b959a7", size = 3468288, upload-time = "2026-01-28T00:23:55.09Z" }, + { url = "https://files.pythonhosted.org/packages/56/f7/f648fdbb61d0d45902d3f374217451385edc7e7768d1b03ff1d0e5ffc17b/cryptography-46.0.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a9556ba711f7c23f77b151d5798f3ac44a13455cc68db7697a1096e6d0563cab", size = 7169583, upload-time = "2026-01-28T00:23:56.558Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cc/8f3224cbb2a928de7298d6ed4790f5ebc48114e02bdc9559196bfb12435d/cryptography-46.0.4-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8bf75b0259e87fa70bddc0b8b4078b76e7fd512fd9afae6c1193bcf440a4dbef", size = 4275419, upload-time = "2026-01-28T00:23:58.364Z" }, + { url = "https://files.pythonhosted.org/packages/17/43/4a18faa7a872d00e4264855134ba82d23546c850a70ff209e04ee200e76f/cryptography-46.0.4-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3c268a3490df22270955966ba236d6bc4a8f9b6e4ffddb78aac535f1a5ea471d", size = 4419058, upload-time = "2026-01-28T00:23:59.867Z" }, + { url = "https://files.pythonhosted.org/packages/ee/64/6651969409821d791ba12346a124f55e1b76f66a819254ae840a965d4b9c/cryptography-46.0.4-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:812815182f6a0c1d49a37893a303b44eaac827d7f0d582cecfc81b6427f22973", size = 4278151, upload-time = "2026-01-28T00:24:01.731Z" }, + { url = "https://files.pythonhosted.org/packages/20/0b/a7fce65ee08c3c02f7a8310cc090a732344066b990ac63a9dfd0a655d321/cryptography-46.0.4-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a90e43e3ef65e6dcf969dfe3bb40cbf5aef0d523dff95bfa24256be172a845f4", size = 4939441, upload-time = "2026-01-28T00:24:03.175Z" }, + { url = "https://files.pythonhosted.org/packages/db/a7/20c5701e2cd3e1dfd7a19d2290c522a5f435dd30957d431dcb531d0f1413/cryptography-46.0.4-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a05177ff6296644ef2876fce50518dffb5bcdf903c85250974fc8bc85d54c0af", size = 4451617, upload-time = "2026-01-28T00:24:05.403Z" }, + { url = "https://files.pythonhosted.org/packages/00/dc/3e16030ea9aa47b63af6524c354933b4fb0e352257c792c4deeb0edae367/cryptography-46.0.4-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:daa392191f626d50f1b136c9b4cf08af69ca8279d110ea24f5c2700054d2e263", size = 3977774, upload-time = "2026-01-28T00:24:06.851Z" }, + { url = "https://files.pythonhosted.org/packages/42/c8/ad93f14118252717b465880368721c963975ac4b941b7ef88f3c56bf2897/cryptography-46.0.4-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e07ea39c5b048e085f15923511d8121e4a9dc45cee4e3b970ca4f0d338f23095", size = 4277008, upload-time = "2026-01-28T00:24:08.926Z" }, + { url = "https://files.pythonhosted.org/packages/00/cf/89c99698151c00a4631fbfcfcf459d308213ac29e321b0ff44ceeeac82f1/cryptography-46.0.4-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d5a45ddc256f492ce42a4e35879c5e5528c09cd9ad12420828c972951d8e016b", size = 4903339, upload-time = "2026-01-28T00:24:12.009Z" }, + { url = "https://files.pythonhosted.org/packages/03/c3/c90a2cb358de4ac9309b26acf49b2a100957e1ff5cc1e98e6c4996576710/cryptography-46.0.4-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:6bb5157bf6a350e5b28aee23beb2d84ae6f5be390b2f8ee7ea179cda077e1019", size = 4451216, upload-time = "2026-01-28T00:24:13.975Z" }, + { url = "https://files.pythonhosted.org/packages/96/2c/8d7f4171388a10208671e181ca43cdc0e596d8259ebacbbcfbd16de593da/cryptography-46.0.4-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dd5aba870a2c40f87a3af043e0dee7d9eb02d4aff88a797b48f2b43eff8c3ab4", size = 4404299, upload-time = "2026-01-28T00:24:16.169Z" }, + { url = "https://files.pythonhosted.org/packages/e9/23/cbb2036e450980f65c6e0a173b73a56ff3bccd8998965dea5cc9ddd424a5/cryptography-46.0.4-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:93d8291da8d71024379ab2cb0b5c57915300155ad42e07f76bea6ad838d7e59b", size = 4664837, upload-time = "2026-01-28T00:24:17.629Z" }, + { url = "https://files.pythonhosted.org/packages/0a/21/f7433d18fe6d5845329cbdc597e30caf983229c7a245bcf54afecc555938/cryptography-46.0.4-cp38-abi3-win32.whl", hash = "sha256:0563655cb3c6d05fb2afe693340bc050c30f9f34e15763361cf08e94749401fc", size = 3009779, upload-time = "2026-01-28T00:24:20.198Z" }, + { url = "https://files.pythonhosted.org/packages/3a/6a/bd2e7caa2facffedf172a45c1a02e551e6d7d4828658c9a245516a598d94/cryptography-46.0.4-cp38-abi3-win_amd64.whl", hash = "sha256:fa0900b9ef9c49728887d1576fd8d9e7e3ea872fa9b25ef9b64888adc434e976", size = 3466633, upload-time = "2026-01-28T00:24:21.851Z" }, +] + +[[package]] +name = "deepdiff" +version = "8.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "orderly-set" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/76/36c9aab3d5c19a94091f7c6c6e784efca50d87b124bf026c36e94719f33c/deepdiff-8.6.1.tar.gz", hash = "sha256:ec56d7a769ca80891b5200ec7bd41eec300ced91ebcc7797b41eb2b3f3ff643a", size = 634054, upload-time = "2025-09-03T19:40:41.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl", hash = "sha256:ee8708a7f7d37fb273a541fa24ad010ed484192cd0c4ffc0fa0ed5e2d4b9e78b", size = 91378, upload-time = "2025-09-03T19:40:39.679Z" }, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + +[[package]] +name = "deptry" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "packaging" }, + { name = "requirements-parser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/31/3e2f4a9b43bd807b28a49d673b9b5f8dcc7265d43950b24e875ba90e6205/deptry-0.23.1.tar.gz", hash = "sha256:5d23e0ef25f3c56405c05383a476edda55944563c5c47a3e9249ed3ec860d382", size = 460016, upload-time = "2025-07-31T05:54:49.681Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/d0/9785c0e7fdab12f5324467d70ba65ad03b9d4071a13fc182b6d98bab6208/deptry-0.23.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f0b231d098fb5b48d8973c9f192c353ffdd395770063424969fa7f15ddfea7d8", size = 1768731, upload-time = "2025-07-31T05:54:47.348Z" }, + { url = "https://files.pythonhosted.org/packages/c5/4b/46aded35e0de153936b2214e49e5935179eed9f23cbd3a9a0cd9a5ab0abd/deptry-0.23.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:bf057f514bb2fa18a2b192a7f7372bd14577ff46b11486933e8383dfef461983", size = 1667240, upload-time = "2025-07-31T05:54:43.956Z" }, + { url = "https://files.pythonhosted.org/packages/ef/f7/206330f68280a1af7edb8bea87f383dbaa4e3b02b37199d40f86e4c43048/deptry-0.23.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ee3f5663bb1c048e2aaf25a4d9e6d09cc1f3b3396ee248980878c6a6c9c0e21", size = 1772019, upload-time = "2025-07-31T05:54:31.165Z" }, + { url = "https://files.pythonhosted.org/packages/c5/80/51a9e94349b47013e2fd78fd221b12202a7866cd2e0882cfd87d63055e88/deptry-0.23.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae0366dc5f50a5fb29cf90de1110c5e368513de6c1b2dac439f2817f3f752616", size = 1855973, upload-time = "2025-07-31T05:54:37.733Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7a/bff10ddd26ce39c56a9a35bdc98fcf44c2befe5954c8da4bb895e3f750bb/deptry-0.23.1-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ab156a90a9eda5819aeb1c1da585dd4d5ec509029399a38771a49e78f40db90f", size = 1946957, upload-time = "2025-07-31T05:54:34.567Z" }, + { url = "https://files.pythonhosted.org/packages/7e/b6/c80b190cbd817d1f75f8d02d4b6f4d430b2f3014a09d3895684e291e473b/deptry-0.23.1-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:651c7eb168233755152fcc468713c024d64a03069645187edb4a17ba61ce6133", size = 2025282, upload-time = "2025-07-31T05:54:40.906Z" }, + { url = "https://files.pythonhosted.org/packages/3c/58/1dfb7a6c4ec2daf123264d2c30f53f45791fee46cd0244be5bf97597d2aa/deptry-0.23.1-cp39-abi3-win_amd64.whl", hash = "sha256:8da1e8f70e7086ebc228f3a4a3cfb5aa127b09b5eef60d694503d6bb79809025", size = 1631377, upload-time = "2025-07-31T05:54:51.951Z" }, + { url = "https://files.pythonhosted.org/packages/18/d3/667b974cf42fc50245a8028beb9966643ee214ca567cc6df6e876feca5ed/deptry-0.23.1-cp39-abi3-win_arm64.whl", hash = "sha256:f589497a5809717db4dcf2aa840f2847c0a4c489331608e538850b6a9ab1c30b", size = 1551113, upload-time = "2025-07-31T05:54:50.679Z" }, +] + +[[package]] +name = "distlib" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, +] + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "docstring-parser" +version = "0.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/12/9c22a58c0b1e29271051222d8906257616da84135af9ed167c9e28f85cb3/docstring_parser-0.16.tar.gz", hash = "sha256:538beabd0af1e2db0146b6bd3caa526c35a34d61af9fd2887f3a8a27a739aa6e", size = 26565, upload-time = "2024-03-15T10:39:44.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/7c/e9fcff7623954d86bdc17782036cbf715ecab1bec4847c008557affe1ca8/docstring_parser-0.16-py3-none-any.whl", hash = "sha256:bf0a1387354d3691d102edef7ec124f219ef639982d096e26e3b60aeffa90637", size = 36533, upload-time = "2024-03-15T10:39:41.527Z" }, +] + +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + +[[package]] +name = "fastapi" +version = "0.128.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/08/8c8508db6c7b9aae8f7175046af41baad690771c9bcde676419965e338c7/fastapi-0.128.0.tar.gz", hash = "sha256:1cc179e1cef10a6be60ffe429f79b829dce99d8de32d7acb7e6c8dfdf7f2645a", size = 365682, upload-time = "2025-12-27T15:21:13.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/05/5cbb59154b093548acd0f4c7c474a118eda06da25aa75c616b72d8fcd92a/fastapi-0.128.0-py3-none-any.whl", hash = "sha256:aebd93f9716ee3b4f4fcfe13ffb7cf308d99c9f3ab5622d8877441072561582d", size = 103094, upload-time = "2025-12-27T15:21:12.154Z" }, +] + +[[package]] +name = "filelock" +version = "3.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/46/0028a82567109b5ef6e4d2a1f04a583fb513e6cf9527fcdd09afd817deeb/filelock-3.20.0.tar.gz", hash = "sha256:711e943b4ec6be42e1d4e6690b48dc175c822967466bb31c0c293f34334c13f4", size = 18922, upload-time = "2025-10-08T18:03:50.056Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/91/7216b27286936c16f5b4d0c530087e4a54eead683e6b0b73dd0c64844af6/filelock-3.20.0-py3-none-any.whl", hash = "sha256:339b4732ffda5cd79b13f4e2711a31b0365ce445d95d243bb996273d072546a2", size = 16054, upload-time = "2025-10-08T18:03:48.35Z" }, +] + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl", hash = "sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", size = 26661, upload-time = "2025-12-19T23:16:13.622Z" }, +] + +[[package]] +name = "ghp-import" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, +] + +[[package]] +name = "greenlet" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/e5/40dbda2736893e3e53d25838e0f19a2b417dfc122b9989c91918db30b5d3/greenlet-3.3.0.tar.gz", hash = "sha256:a82bb225a4e9e4d653dd2fb7b8b2d36e4fb25bc0165422a11e48b88e9e6f78fb", size = 190651, upload-time = "2025-12-04T14:49:44.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/2f/28592176381b9ab2cafa12829ba7b472d177f3acc35d8fbcf3673d966fff/greenlet-3.3.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:a1e41a81c7e2825822f4e068c48cb2196002362619e2d70b148f20a831c00739", size = 275140, upload-time = "2025-12-04T14:23:01.282Z" }, + { url = "https://files.pythonhosted.org/packages/2c/80/fbe937bf81e9fca98c981fe499e59a3f45df2a04da0baa5c2be0dca0d329/greenlet-3.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f515a47d02da4d30caaa85b69474cec77b7929b2e936ff7fb853d42f4bf8808", size = 599219, upload-time = "2025-12-04T14:50:08.309Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ff/7c985128f0514271b8268476af89aee6866df5eec04ac17dcfbc676213df/greenlet-3.3.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d2d9fd66bfadf230b385fdc90426fcd6eb64db54b40c495b72ac0feb5766c54", size = 610211, upload-time = "2025-12-04T14:57:43.968Z" }, + { url = "https://files.pythonhosted.org/packages/79/07/c47a82d881319ec18a4510bb30463ed6891f2ad2c1901ed5ec23d3de351f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30a6e28487a790417d036088b3bcb3f3ac7d8babaa7d0139edbaddebf3af9492", size = 624311, upload-time = "2025-12-04T15:07:14.697Z" }, + { url = "https://files.pythonhosted.org/packages/fd/8e/424b8c6e78bd9837d14ff7df01a9829fc883ba2ab4ea787d4f848435f23f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:087ea5e004437321508a8d6f20efc4cfec5e3c30118e1417ea96ed1d93950527", size = 612833, upload-time = "2025-12-04T14:26:03.669Z" }, + { url = "https://files.pythonhosted.org/packages/b5/ba/56699ff9b7c76ca12f1cdc27a886d0f81f2189c3455ff9f65246780f713d/greenlet-3.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ab97cf74045343f6c60a39913fa59710e4bd26a536ce7ab2397adf8b27e67c39", size = 1567256, upload-time = "2025-12-04T15:04:25.276Z" }, + { url = "https://files.pythonhosted.org/packages/1e/37/f31136132967982d698c71a281a8901daf1a8fbab935dce7c0cf15f942cc/greenlet-3.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5375d2e23184629112ca1ea89a53389dddbffcf417dad40125713d88eb5f96e8", size = 1636483, upload-time = "2025-12-04T14:27:30.804Z" }, + { url = "https://files.pythonhosted.org/packages/7e/71/ba21c3fb8c5dce83b8c01f458a42e99ffdb1963aeec08fff5a18588d8fd7/greenlet-3.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:9ee1942ea19550094033c35d25d20726e4f1c40d59545815e1128ac58d416d38", size = 301833, upload-time = "2025-12-04T14:32:23.929Z" }, + { url = "https://files.pythonhosted.org/packages/d7/7c/f0a6d0ede2c7bf092d00bc83ad5bafb7e6ec9b4aab2fbdfa6f134dc73327/greenlet-3.3.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:60c2ef0f578afb3c8d92ea07ad327f9a062547137afe91f38408f08aacab667f", size = 275671, upload-time = "2025-12-04T14:23:05.267Z" }, + { url = "https://files.pythonhosted.org/packages/44/06/dac639ae1a50f5969d82d2e3dd9767d30d6dbdbab0e1a54010c8fe90263c/greenlet-3.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a5d554d0712ba1de0a6c94c640f7aeba3f85b3a6e1f2899c11c2c0428da9365", size = 646360, upload-time = "2025-12-04T14:50:10.026Z" }, + { url = "https://files.pythonhosted.org/packages/e0/94/0fb76fe6c5369fba9bf98529ada6f4c3a1adf19e406a47332245ef0eb357/greenlet-3.3.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3a898b1e9c5f7307ebbde4102908e6cbfcb9ea16284a3abe15cab996bee8b9b3", size = 658160, upload-time = "2025-12-04T14:57:45.41Z" }, + { url = "https://files.pythonhosted.org/packages/93/79/d2c70cae6e823fac36c3bbc9077962105052b7ef81db2f01ec3b9bf17e2b/greenlet-3.3.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dcd2bdbd444ff340e8d6bdf54d2f206ccddbb3ccfdcd3c25bf4afaa7b8f0cf45", size = 671388, upload-time = "2025-12-04T15:07:15.789Z" }, + { url = "https://files.pythonhosted.org/packages/b8/14/bab308fc2c1b5228c3224ec2bf928ce2e4d21d8046c161e44a2012b5203e/greenlet-3.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5773edda4dc00e173820722711d043799d3adb4f01731f40619e07ea2750b955", size = 660166, upload-time = "2025-12-04T14:26:05.099Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d2/91465d39164eaa0085177f61983d80ffe746c5a1860f009811d498e7259c/greenlet-3.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac0549373982b36d5fd5d30beb8a7a33ee541ff98d2b502714a09f1169f31b55", size = 1615193, upload-time = "2025-12-04T15:04:27.041Z" }, + { url = "https://files.pythonhosted.org/packages/42/1b/83d110a37044b92423084d52d5d5a3b3a73cafb51b547e6d7366ff62eff1/greenlet-3.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d198d2d977460358c3b3a4dc844f875d1adb33817f0613f663a656f463764ccc", size = 1683653, upload-time = "2025-12-04T14:27:32.366Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9a/9030e6f9aa8fd7808e9c31ba4c38f87c4f8ec324ee67431d181fe396d705/greenlet-3.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:73f51dd0e0bdb596fb0417e475fa3c5e32d4c83638296e560086b8d7da7c4170", size = 305387, upload-time = "2025-12-04T14:26:51.063Z" }, + { url = "https://files.pythonhosted.org/packages/a0/66/bd6317bc5932accf351fc19f177ffba53712a202f9df10587da8df257c7e/greenlet-3.3.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d6ed6f85fae6cdfdb9ce04c9bf7a08d666cfcfb914e7d006f44f840b46741931", size = 282638, upload-time = "2025-12-04T14:25:20.941Z" }, + { url = "https://files.pythonhosted.org/packages/30/cf/cc81cb030b40e738d6e69502ccbd0dd1bced0588e958f9e757945de24404/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9125050fcf24554e69c4cacb086b87b3b55dc395a8b3ebe6487b045b2614388", size = 651145, upload-time = "2025-12-04T14:50:11.039Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ea/1020037b5ecfe95ca7df8d8549959baceb8186031da83d5ecceff8b08cd2/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87e63ccfa13c0a0f6234ed0add552af24cc67dd886731f2261e46e241608bee3", size = 654236, upload-time = "2025-12-04T14:57:47.007Z" }, + { url = "https://files.pythonhosted.org/packages/69/cc/1e4bae2e45ca2fa55299f4e85854606a78ecc37fead20d69322f96000504/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2662433acbca297c9153a4023fe2161c8dcfdcc91f10433171cf7e7d94ba2221", size = 662506, upload-time = "2025-12-04T15:07:16.906Z" }, + { url = "https://files.pythonhosted.org/packages/57/b9/f8025d71a6085c441a7eaff0fd928bbb275a6633773667023d19179fe815/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c6e9b9c1527a78520357de498b0e709fb9e2f49c3a513afd5a249007261911b", size = 653783, upload-time = "2025-12-04T14:26:06.225Z" }, + { url = "https://files.pythonhosted.org/packages/f6/c7/876a8c7a7485d5d6b5c6821201d542ef28be645aa024cfe1145b35c120c1/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:286d093f95ec98fdd92fcb955003b8a3d054b4e2cab3e2707a5039e7b50520fd", size = 1614857, upload-time = "2025-12-04T15:04:28.484Z" }, + { url = "https://files.pythonhosted.org/packages/4f/dc/041be1dff9f23dac5f48a43323cd0789cb798342011c19a248d9c9335536/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c10513330af5b8ae16f023e8ddbfb486ab355d04467c4679c5cfe4659975dd9", size = 1676034, upload-time = "2025-12-04T14:27:33.531Z" }, +] + +[[package]] +name = "griffe" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ec/d7/6c09dd7ce4c7837e4cdb11dce980cb45ae3cd87677298dc3b781b6bce7d3/griffe-1.14.0.tar.gz", hash = "sha256:9d2a15c1eca966d68e00517de5d69dd1bc5c9f2335ef6c1775362ba5b8651a13", size = 424684, upload-time = "2025-09-05T15:02:29.167Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl", hash = "sha256:0e9d52832cccf0f7188cfe585ba962d2674b241c01916d780925df34873bceb0", size = 144439, upload-time = "2025-09-05T15:02:27.511Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "httpx-sse" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, +] + +[[package]] +name = "humanfriendly" +version = "10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, +] + +[[package]] +name = "identify" +version = "2.6.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/e7/685de97986c916a6d93b3876139e00eef26ad5bbbd61925d670ae8013449/identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf", size = 99311, upload-time = "2025-10-02T17:43:40.631Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757", size = 99183, upload-time = "2025-10-02T17:43:39.137Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jiter" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/9d/e0660989c1370e25848bb4c52d061c71837239738ad937e83edca174c273/jiter-0.12.0.tar.gz", hash = "sha256:64dfcd7d5c168b38d3f9f8bba7fc639edb3418abcc74f22fdbe6b8938293f30b", size = 168294, upload-time = "2025-11-09T20:49:23.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/a6/97209693b177716e22576ee1161674d1d58029eb178e01866a0422b69224/jiter-0.12.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6cc49d5130a14b732e0612bc76ae8db3b49898732223ef8b7599aa8d9810683e", size = 313658, upload-time = "2025-11-09T20:47:44.424Z" }, + { url = "https://files.pythonhosted.org/packages/06/4d/125c5c1537c7d8ee73ad3d530a442d6c619714b95027143f1b61c0b4dfe0/jiter-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:37f27a32ce36364d2fa4f7fdc507279db604d27d239ea2e044c8f148410defe1", size = 318605, upload-time = "2025-11-09T20:47:45.973Z" }, + { url = "https://files.pythonhosted.org/packages/99/bf/a840b89847885064c41a5f52de6e312e91fa84a520848ee56c97e4fa0205/jiter-0.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbc0944aa3d4b4773e348cda635252824a78f4ba44328e042ef1ff3f6080d1cf", size = 349803, upload-time = "2025-11-09T20:47:47.535Z" }, + { url = "https://files.pythonhosted.org/packages/8a/88/e63441c28e0db50e305ae23e19c1d8fae012d78ed55365da392c1f34b09c/jiter-0.12.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:da25c62d4ee1ffbacb97fac6dfe4dcd6759ebdc9015991e92a6eae5816287f44", size = 365120, upload-time = "2025-11-09T20:47:49.284Z" }, + { url = "https://files.pythonhosted.org/packages/0a/7c/49b02714af4343970eb8aca63396bc1c82fa01197dbb1e9b0d274b550d4e/jiter-0.12.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:048485c654b838140b007390b8182ba9774621103bd4d77c9c3f6f117474ba45", size = 479918, upload-time = "2025-11-09T20:47:50.807Z" }, + { url = "https://files.pythonhosted.org/packages/69/ba/0a809817fdd5a1db80490b9150645f3aae16afad166960bcd562be194f3b/jiter-0.12.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:635e737fbb7315bef0037c19b88b799143d2d7d3507e61a76751025226b3ac87", size = 379008, upload-time = "2025-11-09T20:47:52.211Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c3/c9fc0232e736c8877d9e6d83d6eeb0ba4e90c6c073835cc2e8f73fdeef51/jiter-0.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e017c417b1ebda911bd13b1e40612704b1f5420e30695112efdbed8a4b389ed", size = 361785, upload-time = "2025-11-09T20:47:53.512Z" }, + { url = "https://files.pythonhosted.org/packages/96/61/61f69b7e442e97ca6cd53086ddc1cf59fb830549bc72c0a293713a60c525/jiter-0.12.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:89b0bfb8b2bf2351fba36bb211ef8bfceba73ef58e7f0c68fb67b5a2795ca2f9", size = 386108, upload-time = "2025-11-09T20:47:54.893Z" }, + { url = "https://files.pythonhosted.org/packages/e9/2e/76bb3332f28550c8f1eba3bf6e5efe211efda0ddbbaf24976bc7078d42a5/jiter-0.12.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:f5aa5427a629a824a543672778c9ce0c5e556550d1569bb6ea28a85015287626", size = 519937, upload-time = "2025-11-09T20:47:56.253Z" }, + { url = "https://files.pythonhosted.org/packages/84/d6/fa96efa87dc8bff2094fb947f51f66368fa56d8d4fc9e77b25d7fbb23375/jiter-0.12.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed53b3d6acbcb0fd0b90f20c7cb3b24c357fe82a3518934d4edfa8c6898e498c", size = 510853, upload-time = "2025-11-09T20:47:58.32Z" }, + { url = "https://files.pythonhosted.org/packages/8a/28/93f67fdb4d5904a708119a6ab58a8f1ec226ff10a94a282e0215402a8462/jiter-0.12.0-cp313-cp313-win32.whl", hash = "sha256:4747de73d6b8c78f2e253a2787930f4fffc68da7fa319739f57437f95963c4de", size = 204699, upload-time = "2025-11-09T20:47:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/c4/1f/30b0eb087045a0abe2a5c9c0c0c8da110875a1d3be83afd4a9a4e548be3c/jiter-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:e25012eb0c456fcc13354255d0338cd5397cce26c77b2832b3c4e2e255ea5d9a", size = 204258, upload-time = "2025-11-09T20:48:01.01Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f4/2b4daf99b96bce6fc47971890b14b2a36aef88d7beb9f057fafa032c6141/jiter-0.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:c97b92c54fe6110138c872add030a1f99aea2401ddcdaa21edf74705a646dd60", size = 185503, upload-time = "2025-11-09T20:48:02.35Z" }, + { url = "https://files.pythonhosted.org/packages/39/ca/67bb15a7061d6fe20b9b2a2fd783e296a1e0f93468252c093481a2f00efa/jiter-0.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:53839b35a38f56b8be26a7851a48b89bc47e5d88e900929df10ed93b95fea3d6", size = 317965, upload-time = "2025-11-09T20:48:03.783Z" }, + { url = "https://files.pythonhosted.org/packages/18/af/1788031cd22e29c3b14bc6ca80b16a39a0b10e611367ffd480c06a259831/jiter-0.12.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94f669548e55c91ab47fef8bddd9c954dab1938644e715ea49d7e117015110a4", size = 345831, upload-time = "2025-11-09T20:48:05.55Z" }, + { url = "https://files.pythonhosted.org/packages/05/17/710bf8472d1dff0d3caf4ced6031060091c1320f84ee7d5dcbed1f352417/jiter-0.12.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:351d54f2b09a41600ffea43d081522d792e81dcfb915f6d2d242744c1cc48beb", size = 361272, upload-time = "2025-11-09T20:48:06.951Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f1/1dcc4618b59761fef92d10bcbb0b038b5160be653b003651566a185f1a5c/jiter-0.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2a5e90604620f94bf62264e7c2c038704d38217b7465b863896c6d7c902b06c7", size = 204604, upload-time = "2025-11-09T20:48:08.328Z" }, + { url = "https://files.pythonhosted.org/packages/d9/32/63cb1d9f1c5c6632a783c0052cde9ef7ba82688f7065e2f0d5f10a7e3edb/jiter-0.12.0-cp313-cp313t-win_arm64.whl", hash = "sha256:88ef757017e78d2860f96250f9393b7b577b06a956ad102c29c8237554380db3", size = 185628, upload-time = "2025-11-09T20:48:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/45c9f0dbe4a1416b2b9a8a6d1236459540f43d7fb8883cff769a8db0612d/jiter-0.12.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c46d927acd09c67a9fb1416df45c5a04c27e83aae969267e98fba35b74e99525", size = 312478, upload-time = "2025-11-09T20:48:10.898Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a7/54ae75613ba9e0f55fcb0bc5d1f807823b5167cc944e9333ff322e9f07dd/jiter-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:774ff60b27a84a85b27b88cd5583899c59940bcc126caca97eb2a9df6aa00c49", size = 318706, upload-time = "2025-11-09T20:48:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/2aa241ad2c10774baf6c37f8b8e1f39c07db358f1329f4eb40eba179c2a2/jiter-0.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5433fab222fb072237df3f637d01b81f040a07dcac1cb4a5c75c7aa9ed0bef1", size = 351894, upload-time = "2025-11-09T20:48:13.673Z" }, + { url = "https://files.pythonhosted.org/packages/54/4f/0f2759522719133a9042781b18cc94e335b6d290f5e2d3e6899d6af933e3/jiter-0.12.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f8c593c6e71c07866ec6bfb790e202a833eeec885022296aff6b9e0b92d6a70e", size = 365714, upload-time = "2025-11-09T20:48:15.083Z" }, + { url = "https://files.pythonhosted.org/packages/dc/6f/806b895f476582c62a2f52c453151edd8a0fde5411b0497baaa41018e878/jiter-0.12.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:90d32894d4c6877a87ae00c6b915b609406819dce8bc0d4e962e4de2784e567e", size = 478989, upload-time = "2025-11-09T20:48:16.706Z" }, + { url = "https://files.pythonhosted.org/packages/86/6c/012d894dc6e1033acd8db2b8346add33e413ec1c7c002598915278a37f79/jiter-0.12.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:798e46eed9eb10c3adbbacbd3bdb5ecd4cf7064e453d00dbef08802dae6937ff", size = 378615, upload-time = "2025-11-09T20:48:18.614Z" }, + { url = "https://files.pythonhosted.org/packages/87/30/d718d599f6700163e28e2c71c0bbaf6dace692e7df2592fd793ac9276717/jiter-0.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3f1368f0a6719ea80013a4eb90ba72e75d7ea67cfc7846db2ca504f3df0169a", size = 364745, upload-time = "2025-11-09T20:48:20.117Z" }, + { url = "https://files.pythonhosted.org/packages/8f/85/315b45ce4b6ddc7d7fceca24068543b02bdc8782942f4ee49d652e2cc89f/jiter-0.12.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:65f04a9d0b4406f7e51279710b27484af411896246200e461d80d3ba0caa901a", size = 386502, upload-time = "2025-11-09T20:48:21.543Z" }, + { url = "https://files.pythonhosted.org/packages/74/0b/ce0434fb40c5b24b368fe81b17074d2840748b4952256bab451b72290a49/jiter-0.12.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:fd990541982a24281d12b67a335e44f117e4c6cbad3c3b75c7dea68bf4ce3a67", size = 519845, upload-time = "2025-11-09T20:48:22.964Z" }, + { url = "https://files.pythonhosted.org/packages/e8/a3/7a7a4488ba052767846b9c916d208b3ed114e3eb670ee984e4c565b9cf0d/jiter-0.12.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:b111b0e9152fa7df870ecaebb0bd30240d9f7fff1f2003bcb4ed0f519941820b", size = 510701, upload-time = "2025-11-09T20:48:24.483Z" }, + { url = "https://files.pythonhosted.org/packages/c3/16/052ffbf9d0467b70af24e30f91e0579e13ded0c17bb4a8eb2aed3cb60131/jiter-0.12.0-cp314-cp314-win32.whl", hash = "sha256:a78befb9cc0a45b5a5a0d537b06f8544c2ebb60d19d02c41ff15da28a9e22d42", size = 205029, upload-time = "2025-11-09T20:48:25.749Z" }, + { url = "https://files.pythonhosted.org/packages/e4/18/3cf1f3f0ccc789f76b9a754bdb7a6977e5d1d671ee97a9e14f7eb728d80e/jiter-0.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:e1fe01c082f6aafbe5c8faf0ff074f38dfb911d53f07ec333ca03f8f6226debf", size = 204960, upload-time = "2025-11-09T20:48:27.415Z" }, + { url = "https://files.pythonhosted.org/packages/02/68/736821e52ecfdeeb0f024b8ab01b5a229f6b9293bbdb444c27efade50b0f/jiter-0.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:d72f3b5a432a4c546ea4bedc84cce0c3404874f1d1676260b9c7f048a9855451", size = 185529, upload-time = "2025-11-09T20:48:29.125Z" }, + { url = "https://files.pythonhosted.org/packages/30/61/12ed8ee7a643cce29ac97c2281f9ce3956eb76b037e88d290f4ed0d41480/jiter-0.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e6ded41aeba3603f9728ed2b6196e4df875348ab97b28fc8afff115ed42ba7a7", size = 318974, upload-time = "2025-11-09T20:48:30.87Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c6/f3041ede6d0ed5e0e79ff0de4c8f14f401bbf196f2ef3971cdbe5fd08d1d/jiter-0.12.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a947920902420a6ada6ad51892082521978e9dd44a802663b001436e4b771684", size = 345932, upload-time = "2025-11-09T20:48:32.658Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5d/4d94835889edd01ad0e2dbfc05f7bdfaed46292e7b504a6ac7839aa00edb/jiter-0.12.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:add5e227e0554d3a52cf390a7635edaffdf4f8fce4fdbcef3cc2055bb396a30c", size = 367243, upload-time = "2025-11-09T20:48:34.093Z" }, + { url = "https://files.pythonhosted.org/packages/fd/76/0051b0ac2816253a99d27baf3dda198663aff882fa6ea7deeb94046da24e/jiter-0.12.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f9b1cda8fcb736250d7e8711d4580ebf004a46771432be0ae4796944b5dfa5d", size = 479315, upload-time = "2025-11-09T20:48:35.507Z" }, + { url = "https://files.pythonhosted.org/packages/70/ae/83f793acd68e5cb24e483f44f482a1a15601848b9b6f199dacb970098f77/jiter-0.12.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:deeb12a2223fe0135c7ff1356a143d57f95bbf1f4a66584f1fc74df21d86b993", size = 380714, upload-time = "2025-11-09T20:48:40.014Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/4808a88338ad2c228b1126b93fcd8ba145e919e886fe910d578230dabe3b/jiter-0.12.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c596cc0f4cb574877550ce4ecd51f8037469146addd676d7c1a30ebe6391923f", size = 365168, upload-time = "2025-11-09T20:48:41.462Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d4/04619a9e8095b42aef436b5aeb4c0282b4ff1b27d1db1508df9f5dc82750/jiter-0.12.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ab4c823b216a4aeab3fdbf579c5843165756bd9ad87cc6b1c65919c4715f783", size = 387893, upload-time = "2025-11-09T20:48:42.921Z" }, + { url = "https://files.pythonhosted.org/packages/17/ea/d3c7e62e4546fdc39197fa4a4315a563a89b95b6d54c0d25373842a59cbe/jiter-0.12.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:e427eee51149edf962203ff8db75a7514ab89be5cb623fb9cea1f20b54f1107b", size = 520828, upload-time = "2025-11-09T20:48:44.278Z" }, + { url = "https://files.pythonhosted.org/packages/cc/0b/c6d3562a03fd767e31cb119d9041ea7958c3c80cb3d753eafb19b3b18349/jiter-0.12.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:edb868841f84c111255ba5e80339d386d937ec1fdce419518ce1bd9370fac5b6", size = 511009, upload-time = "2025-11-09T20:48:45.726Z" }, + { url = "https://files.pythonhosted.org/packages/aa/51/2cb4468b3448a8385ebcd15059d325c9ce67df4e2758d133ab9442b19834/jiter-0.12.0-cp314-cp314t-win32.whl", hash = "sha256:8bbcfe2791dfdb7c5e48baf646d37a6a3dcb5a97a032017741dea9f817dca183", size = 205110, upload-time = "2025-11-09T20:48:47.033Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c5/ae5ec83dec9c2d1af805fd5fe8f74ebded9c8670c5210ec7820ce0dbeb1e/jiter-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2fa940963bf02e1d8226027ef461e36af472dea85d36054ff835aeed944dd873", size = 205223, upload-time = "2025-11-09T20:48:49.076Z" }, + { url = "https://files.pythonhosted.org/packages/97/9a/3c5391907277f0e55195550cf3fa8e293ae9ee0c00fb402fec1e38c0c82f/jiter-0.12.0-cp314-cp314t-win_arm64.whl", hash = "sha256:506c9708dd29b27288f9f8f1140c3cb0e3d8ddb045956d7757b1fa0e0f39a473", size = 185564, upload-time = "2025-11-09T20:48:50.376Z" }, +] + +[[package]] +name = "json5" +version = "0.9.28" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/99/7cf7b9a08984203937bf8c9afd28ff64f733d0f6ccb677888f4bcd82af25/json5-0.9.28.tar.gz", hash = "sha256:1f82f36e615bc5b42f1bbd49dbc94b12563c56408c6ffa06414ea310890e9a6e", size = 41853, upload-time = "2024-11-11T17:37:03.149Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/ea/ef9cd2423087fe726f3f24b2e747ca915004e66215e36b0580c912199752/json5-0.9.28-py3-none-any.whl", hash = "sha256:29c56f1accdd8bc2e037321237662034a7e07921e2b7223281a5ce2c46f0c4df", size = 30984, upload-time = "2024-11-11T17:37:00.397Z" }, +] + +[[package]] +name = "jsonpatch" +version = "1.33" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonpointer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/78/18813351fe5d63acad16aec57f94ec2b70a09e53ca98145589e185423873/jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c", size = 21699, upload-time = "2023-06-26T12:07:29.144Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" }, +] + +[[package]] +name = "jsonpointer" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/0a/eebeb1fa92507ea94016a2a790b93c2ae41a7e18778f85471dc54475ed25/jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef", size = 9114, upload-time = "2024-06-10T19:24:42.462Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942", size = 7595, upload-time = "2024-06-10T19:24:40.698Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "langchain-core" +version = "1.2.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonpatch" }, + { name = "langsmith" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "uuid-utils" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/0e/664d8d81b3493e09cbab72448d2f9d693d1fa5aa2bcc488602203a9b6da0/langchain_core-1.2.7.tar.gz", hash = "sha256:e1460639f96c352b4a41c375f25aeb8d16ffc1769499fb1c20503aad59305ced", size = 837039, upload-time = "2026-01-09T17:44:25.505Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/6f/34a9fba14d191a67f7e2ee3dbce3e9b86d2fa7310e2c7f2c713583481bd2/langchain_core-1.2.7-py3-none-any.whl", hash = "sha256:452f4fef7a3d883357b22600788d37e3d8854ef29da345b7ac7099f33c31828b", size = 490232, upload-time = "2026-01-09T17:44:24.236Z" }, +] + +[[package]] +name = "langchain-openai" +version = "1.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "openai" }, + { name = "tiktoken" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/b7/30bfc4d1b658a9ee524bcce3b0b2ec9c45a11c853a13c4f0c9da9882784b/langchain_openai-1.1.7.tar.gz", hash = "sha256:f5ec31961ed24777548b63a5fe313548bc6e0eb9730d6552b8c6418765254c81", size = 1039134, upload-time = "2026-01-07T19:44:59.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/a1/50e7596aca775d8c3883eceeaf47489fac26c57c1abe243c00174f715a8a/langchain_openai-1.1.7-py3-none-any.whl", hash = "sha256:34e9cd686aac1a120d6472804422792bf8080a2103b5d21ee450c9e42d053815", size = 84753, upload-time = "2026-01-07T19:44:58.629Z" }, +] + +[[package]] +name = "langgraph" +version = "1.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langgraph-checkpoint" }, + { name = "langgraph-prebuilt" }, + { name = "langgraph-sdk" }, + { name = "pydantic" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/9c/dac99ab1732e9fb2d3b673482ac28f02bee222c0319a3b8f8f73d90727e6/langgraph-1.0.6.tar.gz", hash = "sha256:dd8e754c76d34a07485308d7117221acf63990e7de8f46ddf5fe256b0a22e6c5", size = 495092, upload-time = "2026-01-12T20:33:30.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/45/9960747781416bed4e531ed0c6b2f2c739bc7b5397d8e92155463735a40e/langgraph-1.0.6-py3-none-any.whl", hash = "sha256:bcfce190974519c72e29f6e5b17f0023914fd6f936bfab8894083215b271eb89", size = 157356, upload-time = "2026-01-12T20:33:29.191Z" }, +] + +[[package]] +name = "langgraph-checkpoint" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "ormsgpack" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/76/55a18c59dedf39688d72c4b06af73a5e3ea0d1a01bc867b88fbf0659f203/langgraph_checkpoint-4.0.0.tar.gz", hash = "sha256:814d1bd050fac029476558d8e68d87bce9009a0262d04a2c14b918255954a624", size = 137320, upload-time = "2026-01-12T20:30:26.38Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/de/ddd53b7032e623f3c7bcdab2b44e8bf635e468f62e10e5ff1946f62c9356/langgraph_checkpoint-4.0.0-py3-none-any.whl", hash = "sha256:3fa9b2635a7c5ac28b338f631abf6a030c3b508b7b9ce17c22611513b589c784", size = 46329, upload-time = "2026-01-12T20:30:25.2Z" }, +] + +[[package]] +name = "langgraph-prebuilt" +version = "1.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langgraph-checkpoint" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/f5/8c75dace0d729561dce2966e630c5e312193df7e5df41a7e10cd7378c3a7/langgraph_prebuilt-1.0.6.tar.gz", hash = "sha256:c5f6cf0f5a0ac47643d2e26ae6faa38cb28885ecde67911190df9e30c4f72361", size = 162623, upload-time = "2026-01-12T20:31:28.425Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/6c/4045822b0630cfc0f8624c4499ceaf90644142143c063a8dc385a7424fc3/langgraph_prebuilt-1.0.6-py3-none-any.whl", hash = "sha256:9fdc35048ff4ac985a55bd2a019a86d45b8184551504aff6780d096c678b39ae", size = 35322, upload-time = "2026-01-12T20:31:27.161Z" }, +] + +[[package]] +name = "langgraph-sdk" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "orjson" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/0f/ed0634c222eed48a31ba48eab6881f94ad690d65e44fe7ca838240a260c1/langgraph_sdk-0.3.3.tar.gz", hash = "sha256:c34c3dce3b6848755eb61f0c94369d1ba04aceeb1b76015db1ea7362c544fb26", size = 130589, upload-time = "2026-01-13T00:30:43.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/be/4ad511bacfdd854afb12974f407cb30010dceb982dc20c55491867b34526/langgraph_sdk-0.3.3-py3-none-any.whl", hash = "sha256:a52ebaf09d91143e55378bb2d0b033ed98f57f48c9ad35c8f81493b88705fc7b", size = 67021, upload-time = "2026-01-13T00:30:42.264Z" }, +] + +[[package]] +name = "langsmith" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "orjson", marker = "platform_python_implementation != 'PyPy'" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "requests-toolbelt" }, + { name = "uuid-utils" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/85/9c7933052a997da1b85bc5c774f3865e9b1da1c8d71541ea133178b13229/langsmith-0.6.4.tar.gz", hash = "sha256:36f7223a01c218079fbb17da5e536ebbaf5c1468c028abe070aa3ae59bc99ec8", size = 919964, upload-time = "2026-01-15T20:02:28.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/0f/09a6637a7ba777eb307b7c80852d9ee26438e2bdafbad6fcc849ff9d9192/langsmith-0.6.4-py3-none-any.whl", hash = "sha256:ac4835860160be371042c7adbba3cb267bcf8d96a5ea976c33a8a4acad6c5486", size = 283503, upload-time = "2026-01-15T20:02:26.662Z" }, +] + +[[package]] +name = "lazyllm" +version = "0.7.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudpickle" }, + { name = "deepdiff" }, + { name = "docstring-parser" }, + { name = "fastapi" }, + { name = "filelock" }, + { name = "json5" }, + { name = "loguru" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pydantic" }, + { name = "pyjwt" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "toml" }, + { name = "tqdm" }, + { name = "uvicorn" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/4e/fe1141c6ec8a25cd0501ca7fbe4da6fdf9a927f7ebce43c4adf8fa8672e4/lazyllm-0.7.4-py3-none-any.whl", hash = "sha256:9656b26f7ffcf9dbb89f66ecda5040645b83c6b80c13d2a81219d0b3a3103df7", size = 1021088, upload-time = "2026-01-26T09:29:00.683Z" }, +] + +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + +[[package]] +name = "lxml" +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430, upload-time = "2026-05-18T19:19:06.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/eb/7e6f37c5584ccbb2ff267f56fd0339016938c1c8684cfefab9b33ffc2f36/lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736", size = 8559780, upload-time = "2026-05-18T19:17:57.661Z" }, + { url = "https://files.pythonhosted.org/packages/a1/36/587c2521cf23a2cd6c9c22108aa7528f683a1f195ed7ccd23a4b1786ad36/lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9", size = 4618006, upload-time = "2026-05-18T19:18:04.452Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139, upload-time = "2026-05-18T19:19:04.873Z" }, + { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329, upload-time = "2026-05-18T19:19:09.728Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564, upload-time = "2026-05-18T19:19:13.608Z" }, + { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467, upload-time = "2026-05-18T19:19:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304, upload-time = "2026-05-18T19:19:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607, upload-time = "2026-05-18T19:19:22.297Z" }, + { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168, upload-time = "2026-05-18T19:19:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487, upload-time = "2026-05-18T19:19:28.603Z" }, + { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231, upload-time = "2026-05-18T19:18:42.246Z" }, + { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450, upload-time = "2026-05-18T19:18:48.013Z" }, + { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874, upload-time = "2026-05-18T19:18:51.914Z" }, + { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987, upload-time = "2026-05-18T19:18:59.715Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276, upload-time = "2026-05-18T19:19:03.834Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f2/1a2b9f1b7a49d45495369be7ef9ad05b262930f2eab3e3145706fca8083f/lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca", size = 3596903, upload-time = "2026-05-18T19:17:29.863Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/f4ffb024f238eec2131aaa09f3278fb6129cf892741bf68e1fc1afb8c100/lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660", size = 3995869, upload-time = "2026-05-18T19:18:02.596Z" }, + { url = "https://files.pythonhosted.org/packages/d1/53/70eb8c5c6037f27448f1e3c54ebede9545a801ae63f0a7254afca4fe8e45/lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc", size = 3658490, upload-time = "2026-05-19T19:22:53.846Z" }, + { url = "https://files.pythonhosted.org/packages/13/e2/2e325795566de01d0d7c3bb57d3c370616b2d07b01214e84eec5d3b10963/lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0", size = 8577146, upload-time = "2026-05-18T19:18:17.765Z" }, + { url = "https://files.pythonhosted.org/packages/93/cf/5630b5e4be7d2e6bee8efe83865c925221103cf0221303b104ce134b01e2/lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840", size = 4623866, upload-time = "2026-05-18T19:18:30.669Z" }, + { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022, upload-time = "2026-05-18T19:19:31.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695, upload-time = "2026-05-18T19:19:34.764Z" }, + { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642, upload-time = "2026-05-18T19:19:37.771Z" }, + { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338, upload-time = "2026-05-18T19:19:40.553Z" }, + { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528, upload-time = "2026-05-18T19:19:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730, upload-time = "2026-05-18T19:19:46.307Z" }, + { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530, upload-time = "2026-05-18T19:19:49.889Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670, upload-time = "2026-05-18T19:19:53.199Z" }, + { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485, upload-time = "2026-05-18T19:19:08.422Z" }, + { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635, upload-time = "2026-05-18T19:19:12.321Z" }, + { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681, upload-time = "2026-05-18T19:19:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229, upload-time = "2026-05-18T19:19:18.131Z" }, + { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191, upload-time = "2026-05-18T19:19:21.118Z" }, + { url = "https://files.pythonhosted.org/packages/d6/85/be36fb1425b30db3c3f9df75fe86343ebffb79e6320bd7f588e25bfeac39/lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603", size = 3657202, upload-time = "2026-05-18T19:17:39.509Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ce/3cf9a827342269f54d405a6202397de63f07c69cbd6ce7d183a3f0cba1e9/lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137", size = 4064497, upload-time = "2026-05-18T19:18:14.662Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3e/1a957bde8f0760039e627f94699f82caa782c9d838d86c3d28245ee67212/lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf", size = 3741991, upload-time = "2026-05-19T19:22:59.111Z" }, + { url = "https://files.pythonhosted.org/packages/78/b2/00ed55b3a2efa4658fb795c38d1090ec9b3e8a6c3683d4441fa517f09c3b/lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee", size = 8827545, upload-time = "2026-05-18T19:18:41.193Z" }, + { url = "https://files.pythonhosted.org/packages/c0/73/74573db19baa618d5f266f2407898b087ff6927115b00b71e5fc1b700847/lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c", size = 4735736, upload-time = "2026-05-18T19:18:46.761Z" }, + { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291, upload-time = "2026-05-18T19:19:56.215Z" }, + { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822, upload-time = "2026-05-18T19:19:59.223Z" }, + { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923, upload-time = "2026-05-18T19:20:01.549Z" }, + { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843, upload-time = "2026-05-18T19:20:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515, upload-time = "2026-05-18T19:20:06.381Z" }, + { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511, upload-time = "2026-05-18T19:20:09.308Z" }, + { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206, upload-time = "2026-05-18T19:20:11.704Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404, upload-time = "2026-05-18T19:20:14.064Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769, upload-time = "2026-05-18T19:19:23.758Z" }, + { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936, upload-time = "2026-05-18T19:19:27.256Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296, upload-time = "2026-05-18T19:19:29.993Z" }, + { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598, upload-time = "2026-05-18T19:19:33.523Z" }, + { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845, upload-time = "2026-05-18T19:19:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/ce/68/b30e913340c380ddac9580c6e6230991fc37240ec4f64704833e4f3e2769/lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1", size = 3897345, upload-time = "2026-05-18T19:17:33.562Z" }, + { url = "https://files.pythonhosted.org/packages/3c/4e/9eb2af5335545f9fbcd7af57bcf87c6025d31eaa31b14ec184a6c8675328/lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e", size = 4393350, upload-time = "2026-05-18T19:18:10.076Z" }, + { url = "https://files.pythonhosted.org/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" }, +] + +[[package]] +name = "magika" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "numpy" }, + { name = "onnxruntime" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/f3/3d1dcdd7b9c41d589f5cff252d32ed91cdf86ba84391cfc81d9d8773571d/magika-0.6.3.tar.gz", hash = "sha256:7cc52aa7359af861957043e2bf7265ed4741067251c104532765cd668c0c0cb1", size = 3042784, upload-time = "2025-10-30T15:22:34.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/e4/35c323beb3280482c94299d61626116856ac2d4ec16ecef50afc4fdd4291/magika-0.6.3-py3-none-any.whl", hash = "sha256:eda443d08006ee495e02083b32e51b98cb3696ab595a7d13900d8e2ef506ec9d", size = 2969474, upload-time = "2025-10-30T15:22:25.298Z" }, + { url = "https://files.pythonhosted.org/packages/25/8f/132b0d7cd51c02c39fd52658a5896276c30c8cc2fd453270b19db8c40f7e/magika-0.6.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:86901e64b05dde5faff408c9b8245495b2e1fd4c226e3393d3d2a3fee65c504b", size = 13358841, upload-time = "2025-10-30T15:22:27.413Z" }, + { url = "https://files.pythonhosted.org/packages/c4/03/5ed859be502903a68b7b393b17ae0283bf34195cfcca79ce2dc25b9290e7/magika-0.6.3-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:3d9661eedbdf445ac9567e97e7ceefb93545d77a6a32858139ea966b5806fb64", size = 15367335, upload-time = "2025-10-30T15:22:29.907Z" }, + { url = "https://files.pythonhosted.org/packages/7b/9e/f8ee7d644affa3b80efdd623a3d75865c8f058f3950cb87fb0c48e3559bc/magika-0.6.3-py3-none-win_amd64.whl", hash = "sha256:e57f75674447b20cab4db928ae58ab264d7d8582b55183a0b876711c2b2787f3", size = 12692831, upload-time = "2025-10-30T15:22:32.063Z" }, +] + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + +[[package]] +name = "mammoth" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cobble" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/3c/a58418d2af00f2da60d4a51e18cd0311307b72d48d2fffec36a97b4a5e44/mammoth-1.11.0.tar.gz", hash = "sha256:a0f59e442f34d5b6447f4b0999306cbf3e67aaabfa8cb516f878fb1456744637", size = 53142, upload-time = "2025-09-19T10:35:20.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/54/2e39566a131b13f6d8d193f974cb6a34e81bb7cc2fa6f7e03de067b36588/mammoth-1.11.0-py2.py3-none-any.whl", hash = "sha256:c077ab0d450bd7c0c6ecd529a23bf7e0fa8190c929e28998308ff4eada3f063b", size = 54752, upload-time = "2025-09-19T10:35:18.699Z" }, +] + +[[package]] +name = "markdown" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/7dd27d9d863b3376fcf23a5a13cb5d024aed1db46f963f1b5735ae43b3be/markdown-3.10.tar.gz", hash = "sha256:37062d4f2aa4b2b6b32aefb80faa300f82cc790cb949a35b8caede34f2b68c0e", size = 364931, upload-time = "2025-11-03T19:51:15.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl", hash = "sha256:b5b99d6951e2e4948d939255596523444c0e677c669700b1d17aa4a8a464cb7c", size = 107678, upload-time = "2025-11-03T19:51:13.887Z" }, +] + +[[package]] +name = "markdownify" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816, upload-time = "2025-11-16T19:21:18.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724, upload-time = "2025-11-16T19:21:17.622Z" }, +] + +[[package]] +name = "markitdown" +version = "0.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "charset-normalizer" }, + { name = "defusedxml" }, + { name = "magika" }, + { name = "markdownify" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/b7/91fe0e2df07107ab701a15c8ad3213135707e4d6206ae9bd8f457a7ad86a/markitdown-0.1.6.tar.gz", hash = "sha256:e5bdbaffd971b29598c7c39ef0e9afce2f08c0751fbfa4e4257678ebaf8cfc7e", size = 50795, upload-time = "2026-05-26T22:43:59.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/30/8031f183ee86ea8ac4e7ea1296bab4cca1bee2fd036a26df69764eb7ca74/markitdown-0.1.6-py3-none-any.whl", hash = "sha256:07b2d5bf87b5c53e13a9f2fdc440df8ccc85e26f40c1e557781727b700049775", size = 70032, upload-time = "2026-05-26T22:44:03.209Z" }, +] + +[package.optional-dependencies] +docx = [ + { name = "lxml" }, + { name = "mammoth" }, +] +pdf = [ + { name = "pdfminer-six" }, + { name = "pdfplumber" }, +] +pptx = [ + { name = "python-pptx" }, +] +xls = [ + { name = "pandas" }, + { name = "xlrd" }, +] +xlsx = [ + { name = "openpyxl" }, + { name = "pandas" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "mcp" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "jsonschema" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/6d/62e76bbb8144d6ed86e202b5edd8a4cb631e7c8130f3f4893c3f90262b10/mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66", size = 608005, upload-time = "2026-01-24T19:40:32.468Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/d9/eaa1f80170d2b7c5ba23f3b59f766f3a0bb41155fbc32a69adfa1adaaef9/mcp-1.26.0-py3-none-any.whl", hash = "sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca", size = 233615, upload-time = "2026-01-24T19:40:30.652Z" }, +] + +[[package]] +name = "memu-py" +version = "1.5.1" +source = { editable = "." } +dependencies = [ + { name = "alembic" }, + { name = "anthropic" }, + { name = "defusedxml" }, + { name = "httpx" }, + { name = "langchain-core" }, + { name = "numpy" }, + { name = "openai" }, + { name = "pendulum" }, + { name = "pydantic" }, + { name = "sqlmodel" }, +] + +[package.optional-dependencies] +claude = [ + { name = "claude-agent-sdk" }, +] +document = [ + { name = "markitdown", extra = ["docx", "pdf", "pptx", "xls", "xlsx"] }, +] +langgraph = [ + { name = "langchain-core" }, + { name = "langgraph" }, +] +lazyllm = [ + { name = "lazyllm" }, +] +postgres = [ + { name = "pgvector" }, + { name = "sqlalchemy", extra = ["postgresql-psycopgbinary"] }, +] + +[package.dev-dependencies] +dev = [ + { name = "deptry" }, + { name = "langchain-openai" }, + { name = "langgraph" }, + { name = "mkdocs" }, + { name = "mkdocs-material" }, + { name = "mkdocstrings" }, + { name = "mkdocstrings-python" }, + { name = "mypy" }, + { name = "pre-commit" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, + { name = "ruff" }, + { name = "types-defusedxml" }, +] +docs = [ + { name = "mkdocs" }, + { name = "mkdocs-material" }, + { name = "mkdocstrings" }, + { name = "mkdocstrings-python" }, +] +lint = [ + { name = "deptry" }, + { name = "mypy" }, + { name = "pre-commit" }, + { name = "ruff" }, + { name = "types-defusedxml" }, +] +test = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", specifier = ">=1.14.0" }, + { name = "anthropic", specifier = ">=0.40.0" }, + { name = "claude-agent-sdk", marker = "extra == 'claude'", specifier = ">=0.1.24" }, + { name = "defusedxml", specifier = ">=0.7.1" }, + { name = "httpx", specifier = ">=0.28.1" }, + { name = "langchain-core", specifier = ">=1.2.7" }, + { name = "langchain-core", marker = "extra == 'langgraph'", specifier = ">=0.1.0" }, + { name = "langgraph", marker = "extra == 'langgraph'", specifier = ">=0.0.10" }, + { name = "lazyllm", marker = "extra == 'lazyllm'", specifier = ">=0.7.3" }, + { name = "markitdown", extras = ["docx", "pptx", "xlsx", "xls", "pdf"], marker = "extra == 'document'", specifier = ">=0.1.0" }, + { name = "numpy", specifier = ">=2.3.4" }, + { name = "openai", specifier = ">=2.8.0" }, + { name = "pendulum", specifier = ">=3.1.0" }, + { name = "pgvector", marker = "extra == 'postgres'", specifier = ">=0.3.4" }, + { name = "pydantic", specifier = ">=2.12.4" }, + { name = "sqlalchemy", extras = ["postgresql-psycopgbinary"], marker = "extra == 'postgres'", specifier = ">=2.0.36" }, + { name = "sqlmodel", specifier = ">=0.0.27" }, +] +provides-extras = ["postgres", "langgraph", "claude", "lazyllm", "document"] + +[package.metadata.requires-dev] +dev = [ + { name = "deptry", specifier = ">=0.23.1" }, + { name = "langchain-openai", specifier = ">=1.1.7" }, + { name = "langgraph", specifier = ">=1.0.6" }, + { name = "mkdocs", specifier = ">=1.6.1" }, + { name = "mkdocs-material", specifier = ">=9.6.23" }, + { name = "mkdocstrings", specifier = ">=0.30.1" }, + { name = "mkdocstrings-python", specifier = ">=1.18.2" }, + { name = "mypy", specifier = ">=1.18.2" }, + { name = "pre-commit", specifier = ">=4.3.0" }, + { name = "pytest", specifier = ">=8.4.2" }, + { name = "pytest-asyncio", specifier = ">=0.24.0" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, + { name = "ruff", specifier = ">=0.14.3" }, + { name = "types-defusedxml", specifier = ">=0.7.0" }, +] +docs = [ + { name = "mkdocs", specifier = ">=1.6.1" }, + { name = "mkdocs-material", specifier = ">=9.6.23" }, + { name = "mkdocstrings", specifier = ">=0.30.1" }, + { name = "mkdocstrings-python", specifier = ">=1.18.2" }, +] +lint = [ + { name = "deptry", specifier = ">=0.23.1" }, + { name = "mypy", specifier = ">=1.18.2" }, + { name = "pre-commit", specifier = ">=4.3.0" }, + { name = "ruff", specifier = ">=0.14.3" }, + { name = "types-defusedxml", specifier = ">=0.7.0" }, +] +test = [ + { name = "pytest", specifier = ">=8.4.2" }, + { name = "pytest-asyncio", specifier = ">=0.24.0" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, +] + +[[package]] +name = "mergedeep" +version = "1.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, +] + +[[package]] +name = "mkdocs" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "ghp-import" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mergedeep" }, + { name = "mkdocs-get-deps" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "pyyaml" }, + { name = "pyyaml-env-tag" }, + { name = "watchdog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" }, +] + +[[package]] +name = "mkdocs-autorefs" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mkdocs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/fa/9124cd63d822e2bcbea1450ae68cdc3faf3655c69b455f3a7ed36ce6c628/mkdocs_autorefs-1.4.3.tar.gz", hash = "sha256:beee715b254455c4aa93b6ef3c67579c399ca092259cc41b7d9342573ff1fc75", size = 55425, upload-time = "2025-08-26T14:23:17.223Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/4d/7123b6fa2278000688ebd338e2a06d16870aaf9eceae6ba047ea05f92df1/mkdocs_autorefs-1.4.3-py3-none-any.whl", hash = "sha256:469d85eb3114801d08e9cc55d102b3ba65917a869b893403b8987b601cf55dc9", size = 25034, upload-time = "2025-08-26T14:23:15.906Z" }, +] + +[[package]] +name = "mkdocs-get-deps" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mergedeep" }, + { name = "platformdirs" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/f5/ed29cd50067784976f25ed0ed6fcd3c2ce9eb90650aa3b2796ddf7b6870b/mkdocs_get_deps-0.2.0.tar.gz", hash = "sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c", size = 10239, upload-time = "2023-11-20T17:51:09.981Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/d4/029f984e8d3f3b6b726bd33cafc473b75e9e44c0f7e80a5b29abc466bdea/mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134", size = 9521, upload-time = "2023-11-20T17:51:08.587Z" }, +] + +[[package]] +name = "mkdocs-material" +version = "9.6.23" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "babel" }, + { name = "backrefs" }, + { name = "colorama" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "mkdocs" }, + { name = "mkdocs-material-extensions" }, + { name = "paginate" }, + { name = "pygments" }, + { name = "pymdown-extensions" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/de/cc1d5139c2782b1a49e1ed1845b3298ed6076b9ba1c740ad7c952d8ffcf9/mkdocs_material-9.6.23.tar.gz", hash = "sha256:62ebc9cdbe90e1ae4f4e9b16a6aa5c69b93474c7b9e79ebc0b11b87f9f055e00", size = 4048130, upload-time = "2025-11-01T16:33:11.782Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/df/bc583e857174b0dc6df67d555123533f09e7e1ac0f3fae7693fb6840c0a3/mkdocs_material-9.6.23-py3-none-any.whl", hash = "sha256:3bf3f1d82d269f3a14ed6897bfc3a844cc05e1dc38045386691b91d7e6945332", size = 9210689, upload-time = "2025-11-01T16:33:08.196Z" }, +] + +[[package]] +name = "mkdocs-material-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847, upload-time = "2023-11-22T19:09:45.208Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728, upload-time = "2023-11-22T19:09:43.465Z" }, +] + +[[package]] +name = "mkdocstrings" +version = "0.30.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mkdocs" }, + { name = "mkdocs-autorefs" }, + { name = "pymdown-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/33/2fa3243439f794e685d3e694590d28469a9b8ea733af4b48c250a3ffc9a0/mkdocstrings-0.30.1.tar.gz", hash = "sha256:84a007aae9b707fb0aebfc9da23db4b26fc9ab562eb56e335e9ec480cb19744f", size = 106350, upload-time = "2025-09-19T10:49:26.446Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/2c/f0dc4e1ee7f618f5bff7e05898d20bf8b6e7fa612038f768bfa295f136a4/mkdocstrings-0.30.1-py3-none-any.whl", hash = "sha256:41bd71f284ca4d44a668816193e4025c950b002252081e387433656ae9a70a82", size = 36704, upload-time = "2025-09-19T10:49:24.805Z" }, +] + +[[package]] +name = "mkdocstrings-python" +version = "1.18.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "griffe" }, + { name = "mkdocs-autorefs" }, + { name = "mkdocstrings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/ae/58ab2bfbee2792e92a98b97e872f7c003deb903071f75d8d83aa55db28fa/mkdocstrings_python-1.18.2.tar.gz", hash = "sha256:4ad536920a07b6336f50d4c6d5603316fafb1172c5c882370cbbc954770ad323", size = 207972, upload-time = "2025-08-28T16:11:19.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/8f/ce008599d9adebf33ed144e7736914385e8537f5fc686fdb7cceb8c22431/mkdocstrings_python-1.18.2-py3-none-any.whl", hash = "sha256:944fe6deb8f08f33fa936d538233c4036e9f53e840994f6146e8e94eb71b600d", size = 138215, upload-time = "2025-08-28T16:11:18.176Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "mypy" +version = "1.18.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/77/8f0d0001ffad290cef2f7f216f96c814866248a0b92a722365ed54648e7e/mypy-1.18.2.tar.gz", hash = "sha256:06a398102a5f203d7477b2923dda3634c36727fa5c237d8f859ef90c42a9924b", size = 3448846, upload-time = "2025-09-19T00:11:10.519Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/04/7f462e6fbba87a72bc8097b93f6842499c428a6ff0c81dd46948d175afe8/mypy-1.18.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:07b8b0f580ca6d289e69209ec9d3911b4a26e5abfde32228a288eb79df129fcc", size = 12898728, upload-time = "2025-09-19T00:10:01.33Z" }, + { url = "https://files.pythonhosted.org/packages/99/5b/61ed4efb64f1871b41fd0b82d29a64640f3516078f6c7905b68ab1ad8b13/mypy-1.18.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ed4482847168439651d3feee5833ccedbf6657e964572706a2adb1f7fa4dfe2e", size = 11910758, upload-time = "2025-09-19T00:10:42.607Z" }, + { url = "https://files.pythonhosted.org/packages/3c/46/d297d4b683cc89a6e4108c4250a6a6b717f5fa96e1a30a7944a6da44da35/mypy-1.18.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3ad2afadd1e9fea5cf99a45a822346971ede8685cc581ed9cd4d42eaf940986", size = 12475342, upload-time = "2025-09-19T00:11:00.371Z" }, + { url = "https://files.pythonhosted.org/packages/83/45/4798f4d00df13eae3bfdf726c9244bcb495ab5bd588c0eed93a2f2dd67f3/mypy-1.18.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a431a6f1ef14cf8c144c6b14793a23ec4eae3db28277c358136e79d7d062f62d", size = 13338709, upload-time = "2025-09-19T00:11:03.358Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/479f7358d9625172521a87a9271ddd2441e1dab16a09708f056e97007207/mypy-1.18.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7ab28cc197f1dd77a67e1c6f35cd1f8e8b73ed2217e4fc005f9e6a504e46e7ba", size = 13529806, upload-time = "2025-09-19T00:10:26.073Z" }, + { url = "https://files.pythonhosted.org/packages/71/cf/ac0f2c7e9d0ea3c75cd99dff7aec1c9df4a1376537cb90e4c882267ee7e9/mypy-1.18.2-cp313-cp313-win_amd64.whl", hash = "sha256:0e2785a84b34a72ba55fb5daf079a1003a34c05b22238da94fcae2bbe46f3544", size = 9833262, upload-time = "2025-09-19T00:10:40.035Z" }, + { url = "https://files.pythonhosted.org/packages/5a/0c/7d5300883da16f0063ae53996358758b2a2df2a09c72a5061fa79a1f5006/mypy-1.18.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:62f0e1e988ad41c2a110edde6c398383a889d95b36b3e60bcf155f5164c4fdce", size = 12893775, upload-time = "2025-09-19T00:10:03.814Z" }, + { url = "https://files.pythonhosted.org/packages/50/df/2cffbf25737bdb236f60c973edf62e3e7b4ee1c25b6878629e88e2cde967/mypy-1.18.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8795a039bab805ff0c1dfdb8cd3344642c2b99b8e439d057aba30850b8d3423d", size = 11936852, upload-time = "2025-09-19T00:10:51.631Z" }, + { url = "https://files.pythonhosted.org/packages/be/50/34059de13dd269227fb4a03be1faee6e2a4b04a2051c82ac0a0b5a773c9a/mypy-1.18.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ca1e64b24a700ab5ce10133f7ccd956a04715463d30498e64ea8715236f9c9c", size = 12480242, upload-time = "2025-09-19T00:11:07.955Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/040983fad5132d85914c874a2836252bbc57832065548885b5bb5b0d4359/mypy-1.18.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d924eef3795cc89fecf6bedc6ed32b33ac13e8321344f6ddbf8ee89f706c05cb", size = 13326683, upload-time = "2025-09-19T00:09:55.572Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ba/89b2901dd77414dd7a8c8729985832a5735053be15b744c18e4586e506ef/mypy-1.18.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20c02215a080e3a2be3aa50506c67242df1c151eaba0dcbc1e4e557922a26075", size = 13514749, upload-time = "2025-09-19T00:10:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/25/bc/cc98767cffd6b2928ba680f3e5bc969c4152bf7c2d83f92f5a504b92b0eb/mypy-1.18.2-cp314-cp314-win_amd64.whl", hash = "sha256:749b5f83198f1ca64345603118a6f01a4e99ad4bf9d103ddc5a3200cc4614adf", size = 9982959, upload-time = "2025-09-19T00:10:37.344Z" }, + { url = "https://files.pythonhosted.org/packages/87/e3/be76d87158ebafa0309946c4a73831974d4d6ab4f4ef40c3b53a385a66fd/mypy-1.18.2-py3-none-any.whl", hash = "sha256:22a1748707dd62b58d2ae53562ffc4d7f8bcc727e8ac7cbc69c053ddc874d47e", size = 2352367, upload-time = "2025-09-19T00:10:15.489Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437, upload-time = "2024-06-04T18:44:11.171Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, +] + +[[package]] +name = "numpy" +version = "2.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/f4/098d2270d52b41f1bd7db9fc288aaa0400cb48c2a3e2af6fa365d9720947/numpy-2.3.4.tar.gz", hash = "sha256:a7d018bfedb375a8d979ac758b120ba846a7fe764911a64465fd87b8729f4a6a", size = 20582187, upload-time = "2025-10-15T16:18:11.77Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/7e/b72610cc91edf138bc588df5150957a4937221ca6058b825b4725c27be62/numpy-2.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c090d4860032b857d94144d1a9976b8e36709e40386db289aaf6672de2a81966", size = 20950335, upload-time = "2025-10-15T16:16:10.304Z" }, + { url = "https://files.pythonhosted.org/packages/3e/46/bdd3370dcea2f95ef14af79dbf81e6927102ddf1cc54adc0024d61252fd9/numpy-2.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a13fc473b6db0be619e45f11f9e81260f7302f8d180c49a22b6e6120022596b3", size = 14179878, upload-time = "2025-10-15T16:16:12.595Z" }, + { url = "https://files.pythonhosted.org/packages/ac/01/5a67cb785bda60f45415d09c2bc245433f1c68dd82eef9c9002c508b5a65/numpy-2.3.4-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:3634093d0b428e6c32c3a69b78e554f0cd20ee420dcad5a9f3b2a63762ce4197", size = 5108673, upload-time = "2025-10-15T16:16:14.877Z" }, + { url = "https://files.pythonhosted.org/packages/c2/cd/8428e23a9fcebd33988f4cb61208fda832800ca03781f471f3727a820704/numpy-2.3.4-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:043885b4f7e6e232d7df4f51ffdef8c36320ee9d5f227b380ea636722c7ed12e", size = 6641438, upload-time = "2025-10-15T16:16:16.805Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d1/913fe563820f3c6b079f992458f7331278dcd7ba8427e8e745af37ddb44f/numpy-2.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4ee6a571d1e4f0ea6d5f22d6e5fbd6ed1dc2b18542848e1e7301bd190500c9d7", size = 14281290, upload-time = "2025-10-15T16:16:18.764Z" }, + { url = "https://files.pythonhosted.org/packages/9e/7e/7d306ff7cb143e6d975cfa7eb98a93e73495c4deabb7d1b5ecf09ea0fd69/numpy-2.3.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc8a63918b04b8571789688b2780ab2b4a33ab44bfe8ccea36d3eba51228c953", size = 16636543, upload-time = "2025-10-15T16:16:21.072Z" }, + { url = "https://files.pythonhosted.org/packages/47/6a/8cfc486237e56ccfb0db234945552a557ca266f022d281a2f577b98e955c/numpy-2.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:40cc556d5abbc54aabe2b1ae287042d7bdb80c08edede19f0c0afb36ae586f37", size = 16056117, upload-time = "2025-10-15T16:16:23.369Z" }, + { url = "https://files.pythonhosted.org/packages/b1/0e/42cb5e69ea901e06ce24bfcc4b5664a56f950a70efdcf221f30d9615f3f3/numpy-2.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ecb63014bb7f4ce653f8be7f1df8cbc6093a5a2811211770f6606cc92b5a78fd", size = 18577788, upload-time = "2025-10-15T16:16:27.496Z" }, + { url = "https://files.pythonhosted.org/packages/86/92/41c3d5157d3177559ef0a35da50f0cda7fa071f4ba2306dd36818591a5bc/numpy-2.3.4-cp313-cp313-win32.whl", hash = "sha256:e8370eb6925bb8c1c4264fec52b0384b44f675f191df91cbe0140ec9f0955646", size = 6282620, upload-time = "2025-10-15T16:16:29.811Z" }, + { url = "https://files.pythonhosted.org/packages/09/97/fd421e8bc50766665ad35536c2bb4ef916533ba1fdd053a62d96cc7c8b95/numpy-2.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:56209416e81a7893036eea03abcb91c130643eb14233b2515c90dcac963fe99d", size = 12784672, upload-time = "2025-10-15T16:16:31.589Z" }, + { url = "https://files.pythonhosted.org/packages/ad/df/5474fb2f74970ca8eb978093969b125a84cc3d30e47f82191f981f13a8a0/numpy-2.3.4-cp313-cp313-win_arm64.whl", hash = "sha256:a700a4031bc0fd6936e78a752eefb79092cecad2599ea9c8039c548bc097f9bc", size = 10196702, upload-time = "2025-10-15T16:16:33.902Z" }, + { url = "https://files.pythonhosted.org/packages/11/83/66ac031464ec1767ea3ed48ce40f615eb441072945e98693bec0bcd056cc/numpy-2.3.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:86966db35c4040fdca64f0816a1c1dd8dbd027d90fca5a57e00e1ca4cd41b879", size = 21049003, upload-time = "2025-10-15T16:16:36.101Z" }, + { url = "https://files.pythonhosted.org/packages/5f/99/5b14e0e686e61371659a1d5bebd04596b1d72227ce36eed121bb0aeab798/numpy-2.3.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:838f045478638b26c375ee96ea89464d38428c69170360b23a1a50fa4baa3562", size = 14302980, upload-time = "2025-10-15T16:16:39.124Z" }, + { url = "https://files.pythonhosted.org/packages/2c/44/e9486649cd087d9fc6920e3fc3ac2aba10838d10804b1e179fb7cbc4e634/numpy-2.3.4-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d7315ed1dab0286adca467377c8381cd748f3dc92235f22a7dfc42745644a96a", size = 5231472, upload-time = "2025-10-15T16:16:41.168Z" }, + { url = "https://files.pythonhosted.org/packages/3e/51/902b24fa8887e5fe2063fd61b1895a476d0bbf46811ab0c7fdf4bd127345/numpy-2.3.4-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:84f01a4d18b2cc4ade1814a08e5f3c907b079c847051d720fad15ce37aa930b6", size = 6739342, upload-time = "2025-10-15T16:16:43.777Z" }, + { url = "https://files.pythonhosted.org/packages/34/f1/4de9586d05b1962acdcdb1dc4af6646361a643f8c864cef7c852bf509740/numpy-2.3.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:817e719a868f0dacde4abdfc5c1910b301877970195db9ab6a5e2c4bd5b121f7", size = 14354338, upload-time = "2025-10-15T16:16:46.081Z" }, + { url = "https://files.pythonhosted.org/packages/1f/06/1c16103b425de7969d5a76bdf5ada0804b476fed05d5f9e17b777f1cbefd/numpy-2.3.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85e071da78d92a214212cacea81c6da557cab307f2c34b5f85b628e94803f9c0", size = 16702392, upload-time = "2025-10-15T16:16:48.455Z" }, + { url = "https://files.pythonhosted.org/packages/34/b2/65f4dc1b89b5322093572b6e55161bb42e3e0487067af73627f795cc9d47/numpy-2.3.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2ec646892819370cf3558f518797f16597b4e4669894a2ba712caccc9da53f1f", size = 16134998, upload-time = "2025-10-15T16:16:51.114Z" }, + { url = "https://files.pythonhosted.org/packages/d4/11/94ec578896cdb973aaf56425d6c7f2aff4186a5c00fac15ff2ec46998b46/numpy-2.3.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:035796aaaddfe2f9664b9a9372f089cfc88bd795a67bd1bfe15e6e770934cf64", size = 18651574, upload-time = "2025-10-15T16:16:53.429Z" }, + { url = "https://files.pythonhosted.org/packages/62/b7/7efa763ab33dbccf56dade36938a77345ce8e8192d6b39e470ca25ff3cd0/numpy-2.3.4-cp313-cp313t-win32.whl", hash = "sha256:fea80f4f4cf83b54c3a051f2f727870ee51e22f0248d3114b8e755d160b38cfb", size = 6413135, upload-time = "2025-10-15T16:16:55.992Z" }, + { url = "https://files.pythonhosted.org/packages/43/70/aba4c38e8400abcc2f345e13d972fb36c26409b3e644366db7649015f291/numpy-2.3.4-cp313-cp313t-win_amd64.whl", hash = "sha256:15eea9f306b98e0be91eb344a94c0e630689ef302e10c2ce5f7e11905c704f9c", size = 12928582, upload-time = "2025-10-15T16:16:57.943Z" }, + { url = "https://files.pythonhosted.org/packages/67/63/871fad5f0073fc00fbbdd7232962ea1ac40eeaae2bba66c76214f7954236/numpy-2.3.4-cp313-cp313t-win_arm64.whl", hash = "sha256:b6c231c9c2fadbae4011ca5e7e83e12dc4a5072f1a1d85a0a7b3ed754d145a40", size = 10266691, upload-time = "2025-10-15T16:17:00.048Z" }, + { url = "https://files.pythonhosted.org/packages/72/71/ae6170143c115732470ae3a2d01512870dd16e0953f8a6dc89525696069b/numpy-2.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:81c3e6d8c97295a7360d367f9f8553973651b76907988bb6066376bc2252f24e", size = 20955580, upload-time = "2025-10-15T16:17:02.509Z" }, + { url = "https://files.pythonhosted.org/packages/af/39/4be9222ffd6ca8a30eda033d5f753276a9c3426c397bb137d8e19dedd200/numpy-2.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7c26b0b2bf58009ed1f38a641f3db4be8d960a417ca96d14e5b06df1506d41ff", size = 14188056, upload-time = "2025-10-15T16:17:04.873Z" }, + { url = "https://files.pythonhosted.org/packages/6c/3d/d85f6700d0a4aa4f9491030e1021c2b2b7421b2b38d01acd16734a2bfdc7/numpy-2.3.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:62b2198c438058a20b6704351b35a1d7db881812d8512d67a69c9de1f18ca05f", size = 5116555, upload-time = "2025-10-15T16:17:07.499Z" }, + { url = "https://files.pythonhosted.org/packages/bf/04/82c1467d86f47eee8a19a464c92f90a9bb68ccf14a54c5224d7031241ffb/numpy-2.3.4-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:9d729d60f8d53a7361707f4b68a9663c968882dd4f09e0d58c044c8bf5faee7b", size = 6643581, upload-time = "2025-10-15T16:17:09.774Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d3/c79841741b837e293f48bd7db89d0ac7a4f2503b382b78a790ef1dc778a5/numpy-2.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd0c630cf256b0a7fd9d0a11c9413b42fef5101219ce6ed5a09624f5a65392c7", size = 14299186, upload-time = "2025-10-15T16:17:11.937Z" }, + { url = "https://files.pythonhosted.org/packages/e8/7e/4a14a769741fbf237eec5a12a2cbc7a4c4e061852b6533bcb9e9a796c908/numpy-2.3.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5e081bc082825f8b139f9e9fe42942cb4054524598aaeb177ff476cc76d09d2", size = 16638601, upload-time = "2025-10-15T16:17:14.391Z" }, + { url = "https://files.pythonhosted.org/packages/93/87/1c1de269f002ff0a41173fe01dcc925f4ecff59264cd8f96cf3b60d12c9b/numpy-2.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:15fb27364ed84114438fff8aaf998c9e19adbeba08c0b75409f8c452a8692c52", size = 16074219, upload-time = "2025-10-15T16:17:17.058Z" }, + { url = "https://files.pythonhosted.org/packages/cd/28/18f72ee77408e40a76d691001ae599e712ca2a47ddd2c4f695b16c65f077/numpy-2.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:85d9fb2d8cd998c84d13a79a09cc0c1091648e848e4e6249b0ccd7f6b487fa26", size = 18576702, upload-time = "2025-10-15T16:17:19.379Z" }, + { url = "https://files.pythonhosted.org/packages/c3/76/95650169b465ececa8cf4b2e8f6df255d4bf662775e797ade2025cc51ae6/numpy-2.3.4-cp314-cp314-win32.whl", hash = "sha256:e73d63fd04e3a9d6bc187f5455d81abfad05660b212c8804bf3b407e984cd2bc", size = 6337136, upload-time = "2025-10-15T16:17:22.886Z" }, + { url = "https://files.pythonhosted.org/packages/dc/89/a231a5c43ede5d6f77ba4a91e915a87dea4aeea76560ba4d2bf185c683f0/numpy-2.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:3da3491cee49cf16157e70f607c03a217ea6647b1cea4819c4f48e53d49139b9", size = 12920542, upload-time = "2025-10-15T16:17:24.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/0c/ae9434a888f717c5ed2ff2393b3f344f0ff6f1c793519fa0c540461dc530/numpy-2.3.4-cp314-cp314-win_arm64.whl", hash = "sha256:6d9cd732068e8288dbe2717177320723ccec4fb064123f0caf9bbd90ab5be868", size = 10480213, upload-time = "2025-10-15T16:17:26.935Z" }, + { url = "https://files.pythonhosted.org/packages/83/4b/c4a5f0841f92536f6b9592694a5b5f68c9ab37b775ff342649eadf9055d3/numpy-2.3.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:22758999b256b595cf0b1d102b133bb61866ba5ceecf15f759623b64c020c9ec", size = 21052280, upload-time = "2025-10-15T16:17:29.638Z" }, + { url = "https://files.pythonhosted.org/packages/3e/80/90308845fc93b984d2cc96d83e2324ce8ad1fd6efea81b324cba4b673854/numpy-2.3.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9cb177bc55b010b19798dc5497d540dea67fd13a8d9e882b2dae71de0cf09eb3", size = 14302930, upload-time = "2025-10-15T16:17:32.384Z" }, + { url = "https://files.pythonhosted.org/packages/3d/4e/07439f22f2a3b247cec4d63a713faae55e1141a36e77fb212881f7cda3fb/numpy-2.3.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:0f2bcc76f1e05e5ab58893407c63d90b2029908fa41f9f1cc51eecce936c3365", size = 5231504, upload-time = "2025-10-15T16:17:34.515Z" }, + { url = "https://files.pythonhosted.org/packages/ab/de/1e11f2547e2fe3d00482b19721855348b94ada8359aef5d40dd57bfae9df/numpy-2.3.4-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:8dc20bde86802df2ed8397a08d793da0ad7a5fd4ea3ac85d757bf5dd4ad7c252", size = 6739405, upload-time = "2025-10-15T16:17:36.128Z" }, + { url = "https://files.pythonhosted.org/packages/3b/40/8cd57393a26cebe2e923005db5134a946c62fa56a1087dc7c478f3e30837/numpy-2.3.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e199c087e2aa71c8f9ce1cb7a8e10677dc12457e7cc1be4798632da37c3e86e", size = 14354866, upload-time = "2025-10-15T16:17:38.884Z" }, + { url = "https://files.pythonhosted.org/packages/93/39/5b3510f023f96874ee6fea2e40dfa99313a00bf3ab779f3c92978f34aace/numpy-2.3.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85597b2d25ddf655495e2363fe044b0ae999b75bc4d630dc0d886484b03a5eb0", size = 16703296, upload-time = "2025-10-15T16:17:41.564Z" }, + { url = "https://files.pythonhosted.org/packages/41/0d/19bb163617c8045209c1996c4e427bccbc4bbff1e2c711f39203c8ddbb4a/numpy-2.3.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:04a69abe45b49c5955923cf2c407843d1c85013b424ae8a560bba16c92fe44a0", size = 16136046, upload-time = "2025-10-15T16:17:43.901Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c1/6dba12fdf68b02a21ac411c9df19afa66bed2540f467150ca64d246b463d/numpy-2.3.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e1708fac43ef8b419c975926ce1eaf793b0c13b7356cfab6ab0dc34c0a02ac0f", size = 18652691, upload-time = "2025-10-15T16:17:46.247Z" }, + { url = "https://files.pythonhosted.org/packages/f8/73/f85056701dbbbb910c51d846c58d29fd46b30eecd2b6ba760fc8b8a1641b/numpy-2.3.4-cp314-cp314t-win32.whl", hash = "sha256:863e3b5f4d9915aaf1b8ec79ae560ad21f0b8d5e3adc31e73126491bb86dee1d", size = 6485782, upload-time = "2025-10-15T16:17:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/17/90/28fa6f9865181cb817c2471ee65678afa8a7e2a1fb16141473d5fa6bacc3/numpy-2.3.4-cp314-cp314t-win_amd64.whl", hash = "sha256:962064de37b9aef801d33bc579690f8bfe6c5e70e29b61783f60bcba838a14d6", size = 13113301, upload-time = "2025-10-15T16:17:50.938Z" }, + { url = "https://files.pythonhosted.org/packages/54/23/08c002201a8e7e1f9afba93b97deceb813252d9cfd0d3351caed123dcf97/numpy-2.3.4-cp314-cp314t-win_arm64.whl", hash = "sha256:8b5a9a39c45d852b62693d9b3f3e0fe052541f804296ff401a72a1b60edafb29", size = 10547532, upload-time = "2025-10-15T16:17:53.48Z" }, +] + +[[package]] +name = "onnxruntime" +version = "1.20.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coloredlogs" }, + { name = "flatbuffers" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "sympy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/71/c5d980ac4189589267a06f758bd6c5667d07e55656bed6c6c0580733ad07/onnxruntime-1.20.1-cp313-cp313-macosx_13_0_universal2.whl", hash = "sha256:cc01437a32d0042b606f462245c8bbae269e5442797f6213e36ce61d5abdd8cc", size = 31007574, upload-time = "2024-11-21T00:49:23.225Z" }, + { url = "https://files.pythonhosted.org/packages/81/0d/13bbd9489be2a6944f4a940084bfe388f1100472f38c07080a46fbd4ab96/onnxruntime-1.20.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb44b08e017a648924dbe91b82d89b0c105b1adcfe31e90d1dc06b8677ad37be", size = 11951459, upload-time = "2024-11-21T00:49:26.269Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ea/4454ae122874fd52bbb8a961262de81c5f932edeb1b72217f594c700d6ef/onnxruntime-1.20.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bda6aebdf7917c1d811f21d41633df00c58aff2bef2f598f69289c1f1dabc4b3", size = 13331620, upload-time = "2024-11-21T00:49:28.875Z" }, + { url = "https://files.pythonhosted.org/packages/d8/e0/50db43188ca1c945decaa8fc2a024c33446d31afed40149897d4f9de505f/onnxruntime-1.20.1-cp313-cp313-win_amd64.whl", hash = "sha256:d30367df7e70f1d9fc5a6a68106f5961686d39b54d3221f760085524e8d38e16", size = 11331758, upload-time = "2024-11-21T00:49:31.417Z" }, + { url = "https://files.pythonhosted.org/packages/d8/55/3821c5fd60b52a6c82a00bba18531793c93c4addfe64fbf061e235c5617a/onnxruntime-1.20.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9158465745423b2b5d97ed25aa7740c7d38d2993ee2e5c3bfacb0c4145c49d8", size = 11950342, upload-time = "2024-11-21T00:49:34.164Z" }, + { url = "https://files.pythonhosted.org/packages/14/56/fd990ca222cef4f9f4a9400567b9a15b220dee2eafffb16b2adbc55c8281/onnxruntime-1.20.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0df6f2df83d61f46e842dbcde610ede27218947c33e994545a22333491e72a3b", size = 13337040, upload-time = "2024-11-21T00:49:37.271Z" }, +] + +[[package]] +name = "openai" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/0c/b9321e12f89e236f5e9a46346c30fb801818e22ba33b798a5aca84be895c/openai-2.8.0.tar.gz", hash = "sha256:4851908f6d6fcacbd47ba659c5ac084f7725b752b6bfa1e948b6fbfc111a6bad", size = 602412, upload-time = "2025-11-13T18:15:25.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/e1/0a6560bab7fb7b5a88d35a505b859c6d969cb2fa2681b568eb5d95019dec/openai-2.8.0-py3-none-any.whl", hash = "sha256:ba975e347f6add2fe13529ccb94d54a578280e960765e5224c34b08d7e029ddf", size = 1022692, upload-time = "2025-11-13T18:15:23.621Z" }, +] + +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + +[[package]] +name = "orderly-set" +version = "5.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/88/39c83c35d5e97cc203e9e77a4f93bf87ec89cf6a22ac4818fdcc65d66584/orderly_set-5.5.0.tar.gz", hash = "sha256:e87185c8e4d8afa64e7f8160ee2c542a475b738bc891dc3f58102e654125e6ce", size = 27414, upload-time = "2025-07-10T20:10:55.885Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl", hash = "sha256:46f0b801948e98f427b412fcabb831677194c05c3b699b80de260374baa0b1e7", size = 13068, upload-time = "2025-07-10T20:10:54.377Z" }, +] + +[[package]] +name = "orjson" +version = "3.11.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/b8/333fdb27840f3bf04022d21b654a35f58e15407183aeb16f3b41aa053446/orjson-3.11.5.tar.gz", hash = "sha256:82393ab47b4fe44ffd0a7659fa9cfaacc717eb617c93cde83795f14af5c2e9d5", size = 5972347, upload-time = "2025-12-06T15:55:39.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/43/61a77040ce59f1569edf38f0b9faadc90c8cf7e9bec2e0df51d0132c6bb7/orjson-3.11.5-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:3b01799262081a4c47c035dd77c1301d40f568f77cc7ec1bb7db5d63b0a01629", size = 245271, upload-time = "2025-12-06T15:54:40.878Z" }, + { url = "https://files.pythonhosted.org/packages/55/f9/0f79be617388227866d50edd2fd320cb8fb94dc1501184bb1620981a0aba/orjson-3.11.5-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:61de247948108484779f57a9f406e4c84d636fa5a59e411e6352484985e8a7c3", size = 129422, upload-time = "2025-12-06T15:54:42.403Z" }, + { url = "https://files.pythonhosted.org/packages/77/42/f1bf1549b432d4a78bfa95735b79b5dac75b65b5bb815bba86ad406ead0a/orjson-3.11.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:894aea2e63d4f24a7f04a1908307c738d0dce992e9249e744b8f4e8dd9197f39", size = 132060, upload-time = "2025-12-06T15:54:43.531Z" }, + { url = "https://files.pythonhosted.org/packages/25/49/825aa6b929f1a6ed244c78acd7b22c1481fd7e5fda047dc8bf4c1a807eb6/orjson-3.11.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ddc21521598dbe369d83d4d40338e23d4101dad21dae0e79fa20465dbace019f", size = 130391, upload-time = "2025-12-06T15:54:45.059Z" }, + { url = "https://files.pythonhosted.org/packages/42/ec/de55391858b49e16e1aa8f0bbbb7e5997b7345d8e984a2dec3746d13065b/orjson-3.11.5-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7cce16ae2f5fb2c53c3eafdd1706cb7b6530a67cc1c17abe8ec747f5cd7c0c51", size = 135964, upload-time = "2025-12-06T15:54:46.576Z" }, + { url = "https://files.pythonhosted.org/packages/1c/40/820bc63121d2d28818556a2d0a09384a9f0262407cf9fa305e091a8048df/orjson-3.11.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e46c762d9f0e1cfb4ccc8515de7f349abbc95b59cb5a2bd68df5973fdef913f8", size = 139817, upload-time = "2025-12-06T15:54:48.084Z" }, + { url = "https://files.pythonhosted.org/packages/09/c7/3a445ca9a84a0d59d26365fd8898ff52bdfcdcb825bcc6519830371d2364/orjson-3.11.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d7345c759276b798ccd6d77a87136029e71e66a8bbf2d2755cbdde1d82e78706", size = 137336, upload-time = "2025-12-06T15:54:49.426Z" }, + { url = "https://files.pythonhosted.org/packages/9a/b3/dc0d3771f2e5d1f13368f56b339c6782f955c6a20b50465a91acb79fe961/orjson-3.11.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75bc2e59e6a2ac1dd28901d07115abdebc4563b5b07dd612bf64260a201b1c7f", size = 138993, upload-time = "2025-12-06T15:54:50.939Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a2/65267e959de6abe23444659b6e19c888f242bf7725ff927e2292776f6b89/orjson-3.11.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:54aae9b654554c3b4edd61896b978568c6daa16af96fa4681c9b5babd469f863", size = 141070, upload-time = "2025-12-06T15:54:52.414Z" }, + { url = "https://files.pythonhosted.org/packages/63/c9/da44a321b288727a322c6ab17e1754195708786a04f4f9d2220a5076a649/orjson-3.11.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:4bdd8d164a871c4ec773f9de0f6fe8769c2d6727879c37a9666ba4183b7f8228", size = 413505, upload-time = "2025-12-06T15:54:53.67Z" }, + { url = "https://files.pythonhosted.org/packages/7f/17/68dc14fa7000eefb3d4d6d7326a190c99bb65e319f02747ef3ebf2452f12/orjson-3.11.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:a261fef929bcf98a60713bf5e95ad067cea16ae345d9a35034e73c3990e927d2", size = 151342, upload-time = "2025-12-06T15:54:55.113Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c5/ccee774b67225bed630a57478529fc026eda33d94fe4c0eac8fe58d4aa52/orjson-3.11.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c028a394c766693c5c9909dec76b24f37e6a1b91999e8d0c0d5feecbe93c3e05", size = 141823, upload-time = "2025-12-06T15:54:56.331Z" }, + { url = "https://files.pythonhosted.org/packages/67/80/5d00e4155d0cd7390ae2087130637671da713959bb558db9bac5e6f6b042/orjson-3.11.5-cp313-cp313-win32.whl", hash = "sha256:2cc79aaad1dfabe1bd2d50ee09814a1253164b3da4c00a78c458d82d04b3bdef", size = 135236, upload-time = "2025-12-06T15:54:57.507Z" }, + { url = "https://files.pythonhosted.org/packages/95/fe/792cc06a84808dbdc20ac6eab6811c53091b42f8e51ecebf14b540e9cfe4/orjson-3.11.5-cp313-cp313-win_amd64.whl", hash = "sha256:ff7877d376add4e16b274e35a3f58b7f37b362abf4aa31863dadacdd20e3a583", size = 133167, upload-time = "2025-12-06T15:54:58.71Z" }, + { url = "https://files.pythonhosted.org/packages/46/2c/d158bd8b50e3b1cfdcf406a7e463f6ffe3f0d167b99634717acdaf5e299f/orjson-3.11.5-cp313-cp313-win_arm64.whl", hash = "sha256:59ac72ea775c88b163ba8d21b0177628bd015c5dd060647bbab6e22da3aad287", size = 126712, upload-time = "2025-12-06T15:54:59.892Z" }, + { url = "https://files.pythonhosted.org/packages/c2/60/77d7b839e317ead7bb225d55bb50f7ea75f47afc489c81199befc5435b50/orjson-3.11.5-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:e446a8ea0a4c366ceafc7d97067bfd55292969143b57e3c846d87fc701e797a0", size = 245252, upload-time = "2025-12-06T15:55:01.127Z" }, + { url = "https://files.pythonhosted.org/packages/f1/aa/d4639163b400f8044cef0fb9aa51b0337be0da3a27187a20d1166e742370/orjson-3.11.5-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:53deb5addae9c22bbe3739298f5f2196afa881ea75944e7720681c7080909a81", size = 129419, upload-time = "2025-12-06T15:55:02.723Z" }, + { url = "https://files.pythonhosted.org/packages/30/94/9eabf94f2e11c671111139edf5ec410d2f21e6feee717804f7e8872d883f/orjson-3.11.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82cd00d49d6063d2b8791da5d4f9d20539c5951f965e45ccf4e96d33505ce68f", size = 132050, upload-time = "2025-12-06T15:55:03.918Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c8/ca10f5c5322f341ea9a9f1097e140be17a88f88d1cfdd29df522970d9744/orjson-3.11.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3fd15f9fc8c203aeceff4fda211157fad114dde66e92e24097b3647a08f4ee9e", size = 130370, upload-time = "2025-12-06T15:55:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/25/d4/e96824476d361ee2edd5c6290ceb8d7edf88d81148a6ce172fc00278ca7f/orjson-3.11.5-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df95000fbe6777bf9820ae82ab7578e8662051bb5f83d71a28992f539d2cda7", size = 136012, upload-time = "2025-12-06T15:55:06.402Z" }, + { url = "https://files.pythonhosted.org/packages/85/8e/9bc3423308c425c588903f2d103cfcfe2539e07a25d6522900645a6f257f/orjson-3.11.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a8d676748fca47ade5bc3da7430ed7767afe51b2f8100e3cd65e151c0eaceb", size = 139809, upload-time = "2025-12-06T15:55:07.656Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3c/b404e94e0b02a232b957c54643ce68d0268dacb67ac33ffdee24008c8b27/orjson-3.11.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa0f513be38b40234c77975e68805506cad5d57b3dfd8fe3baa7f4f4051e15b4", size = 137332, upload-time = "2025-12-06T15:55:08.961Z" }, + { url = "https://files.pythonhosted.org/packages/51/30/cc2d69d5ce0ad9b84811cdf4a0cd5362ac27205a921da524ff42f26d65e0/orjson-3.11.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa1863e75b92891f553b7922ce4ee10ed06db061e104f2b7815de80cdcb135ad", size = 138983, upload-time = "2025-12-06T15:55:10.595Z" }, + { url = "https://files.pythonhosted.org/packages/0e/87/de3223944a3e297d4707d2fe3b1ffb71437550e165eaf0ca8bbe43ccbcb1/orjson-3.11.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4be86b58e9ea262617b8ca6251a2f0d63cc132a6da4b5fcc8e0a4128782c829", size = 141069, upload-time = "2025-12-06T15:55:11.832Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/81d5087ae74be33bcae3ff2d80f5ccaa4a8fedc6d39bf65a427a95b8977f/orjson-3.11.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:b923c1c13fa02084eb38c9c065afd860a5cff58026813319a06949c3af5732ac", size = 413491, upload-time = "2025-12-06T15:55:13.314Z" }, + { url = "https://files.pythonhosted.org/packages/d0/6f/f6058c21e2fc1efaf918986dbc2da5cd38044f1a2d4b7b91ad17c4acf786/orjson-3.11.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:1b6bd351202b2cd987f35a13b5e16471cf4d952b42a73c391cc537974c43ef6d", size = 151375, upload-time = "2025-12-06T15:55:14.715Z" }, + { url = "https://files.pythonhosted.org/packages/54/92/c6921f17d45e110892899a7a563a925b2273d929959ce2ad89e2525b885b/orjson-3.11.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bb150d529637d541e6af06bbe3d02f5498d628b7f98267ff87647584293ab439", size = 141850, upload-time = "2025-12-06T15:55:15.94Z" }, + { url = "https://files.pythonhosted.org/packages/88/86/cdecb0140a05e1a477b81f24739da93b25070ee01ce7f7242f44a6437594/orjson-3.11.5-cp314-cp314-win32.whl", hash = "sha256:9cc1e55c884921434a84a0c3dd2699eb9f92e7b441d7f53f3941079ec6ce7499", size = 135278, upload-time = "2025-12-06T15:55:17.202Z" }, + { url = "https://files.pythonhosted.org/packages/e4/97/b638d69b1e947d24f6109216997e38922d54dcdcdb1b11c18d7efd2d3c59/orjson-3.11.5-cp314-cp314-win_amd64.whl", hash = "sha256:a4f3cb2d874e03bc7767c8f88adaa1a9a05cecea3712649c3b58589ec7317310", size = 133170, upload-time = "2025-12-06T15:55:18.468Z" }, + { url = "https://files.pythonhosted.org/packages/8f/dd/f4fff4a6fe601b4f8f3ba3aa6da8ac33d17d124491a3b804c662a70e1636/orjson-3.11.5-cp314-cp314-win_arm64.whl", hash = "sha256:38b22f476c351f9a1c43e5b07d8b5a02eb24a6ab8e75f700f7d479d4568346a5", size = 126713, upload-time = "2025-12-06T15:55:19.738Z" }, +] + +[[package]] +name = "ormsgpack" +version = "1.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/96/34c40d621996c2f377a18decbd3c59f031dde73c3ba47d1e1e8f29a05aaa/ormsgpack-1.12.1.tar.gz", hash = "sha256:a3877fde1e4f27a39f92681a0aab6385af3a41d0c25375d33590ae20410ea2ac", size = 39476, upload-time = "2025-12-14T07:57:43.248Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/42/f110dfe7cf23a52a82e23eb23d9a6a76ae495447d474686dfa758f3d71d6/ormsgpack-1.12.1-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:9663d6b3ecc917c063d61a99169ce196a80f3852e541ae404206836749459279", size = 376746, upload-time = "2025-12-14T07:57:17.699Z" }, + { url = "https://files.pythonhosted.org/packages/11/76/b386e508a8ae207daec240201a81adb26467bf99b163560724e86bd9ff33/ormsgpack-1.12.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32e85cfbaf01a94a92520e7fe7851cfcfe21a5698299c28ab86194895f9b9233", size = 202489, upload-time = "2025-12-14T07:57:18.807Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0e/5db7a63f387149024572daa3d9512fe8fb14bf4efa0722d6d491bed280e7/ormsgpack-1.12.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dabfd2c24b59c7c69870a5ecee480dfae914a42a0c2e7c9d971cf531e2ba471a", size = 210757, upload-time = "2025-12-14T07:57:19.893Z" }, + { url = "https://files.pythonhosted.org/packages/64/79/3a9899e57cb57430bd766fc1b4c9ad410cb2ba6070bc8cf6301e7d385768/ormsgpack-1.12.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bbf2b64afeded34ccd8e25402e4bca038757913931fa0d693078d75563f6f9", size = 211518, upload-time = "2025-12-14T07:57:20.972Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cd/4f41710ae9fe50d7fcbe476793b3c487746d0e1cc194cc0fee42ff6d989b/ormsgpack-1.12.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9959a71dde1bd0ced84af17facc06a8afada495a34e9cb1bad8e9b20d4c59cef", size = 386251, upload-time = "2025-12-14T07:57:22.099Z" }, + { url = "https://files.pythonhosted.org/packages/bf/54/ba0c97d6231b1f01daafaa520c8cce1e1b7fceaae6fdc1c763925874a7de/ormsgpack-1.12.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:e9be0e3b62d758f21f5b20e0e06b3a240ec546c4a327bf771f5825462aa74714", size = 479607, upload-time = "2025-12-14T07:57:23.525Z" }, + { url = "https://files.pythonhosted.org/packages/18/75/19a9a97a462776d525baf41cfb7072734528775f0a3d5fbfab3aa7756b9b/ormsgpack-1.12.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a29d49ab7fdd77ea787818e60cb4ef491708105b9c4c9b0f919201625eb036b5", size = 388062, upload-time = "2025-12-14T07:57:24.616Z" }, + { url = "https://files.pythonhosted.org/packages/a8/6a/ec26e3f44e9632ecd2f43638b7b37b500eaea5d79cab984ad0b94be14f82/ormsgpack-1.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:c418390b47a1d367e803f6c187f77e4d67c7ae07ba962e3a4a019001f4b0291a", size = 116195, upload-time = "2025-12-14T07:57:25.626Z" }, + { url = "https://files.pythonhosted.org/packages/7d/64/bfa5f4a34d0f15c6aba1b73e73f7441a66d635bd03249d334a4796b7a924/ormsgpack-1.12.1-cp313-cp313-win_arm64.whl", hash = "sha256:cfa22c91cffc10a7fbd43729baff2de7d9c28cef2509085a704168ae31f02568", size = 109986, upload-time = "2025-12-14T07:57:26.569Z" }, + { url = "https://files.pythonhosted.org/packages/87/0e/78e5697164e3223b9b216c13e99f1acbc1ee9833490d68842b13da8ba883/ormsgpack-1.12.1-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:b93c91efb1a70751a1902a5b43b27bd8fd38e0ca0365cf2cde2716423c15c3a6", size = 376758, upload-time = "2025-12-14T07:57:27.641Z" }, + { url = "https://files.pythonhosted.org/packages/2c/0e/3a3cbb64703263d7bbaed7effa3ce78cb9add360a60aa7c544d7df28b641/ormsgpack-1.12.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cf0ea0389167b5fa8d2933dd3f33e887ec4ba68f89c25214d7eec4afd746d22", size = 202487, upload-time = "2025-12-14T07:57:29.051Z" }, + { url = "https://files.pythonhosted.org/packages/d7/2c/807ebe2b77995599bbb1dec8c3f450d5d7dddee14ce3e1e71dc60e2e2a74/ormsgpack-1.12.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4c29af837f35af3375070689e781161e7cf019eb2f7cd641734ae45cd001c0d", size = 210853, upload-time = "2025-12-14T07:57:30.508Z" }, + { url = "https://files.pythonhosted.org/packages/25/57/2cdfc354e3ad8e847628f511f4d238799d90e9e090941e50b9d5ba955ae2/ormsgpack-1.12.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:336fc65aa0fe65896a3dabaae31e332a0a98b4a00ad7b0afde21a7505fd23ff3", size = 211545, upload-time = "2025-12-14T07:57:31.585Z" }, + { url = "https://files.pythonhosted.org/packages/76/1d/c6fda560e4a8ff865b3aec8a86f7c95ab53f4532193a6ae4ab9db35f85aa/ormsgpack-1.12.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:940f60aabfefe71dd6b82cb33f4ff10b2e7f5fcfa5f103cdb0a23b6aae4c713c", size = 386333, upload-time = "2025-12-14T07:57:32.957Z" }, + { url = "https://files.pythonhosted.org/packages/fc/3e/715081b36fceb8b497c68b87d384e1cc6d9c9c130ce3b435634d3d785b86/ormsgpack-1.12.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:596ad9e1b6d4c95595c54aaf49b1392609ca68f562ce06f4f74a5bc4053bcda4", size = 479701, upload-time = "2025-12-14T07:57:34.686Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cf/01ad04def42b3970fc1a302c07f4b46339edf62ef9650247097260471f40/ormsgpack-1.12.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:575210e8fcbc7b0375026ba040a5eef223e9f66a4453d9623fc23282ae09c3c8", size = 388148, upload-time = "2025-12-14T07:57:35.771Z" }, + { url = "https://files.pythonhosted.org/packages/15/91/1fff2fc2b5943c740028f339154e7103c8f2edf1a881d9fbba2ce11c3b1d/ormsgpack-1.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:647daa3718572280893456be44c60aea6690b7f2edc54c55648ee66e8f06550f", size = 116201, upload-time = "2025-12-14T07:57:36.763Z" }, + { url = "https://files.pythonhosted.org/packages/ed/66/142b542aed3f96002c7d1c33507ca6e1e0d0a42b9253ab27ef7ed5793bd9/ormsgpack-1.12.1-cp314-cp314-win_arm64.whl", hash = "sha256:a8b3ab762a6deaf1b6490ab46dda0c51528cf8037e0246c40875c6fe9e37b699", size = 110029, upload-time = "2025-12-14T07:57:37.703Z" }, + { url = "https://files.pythonhosted.org/packages/38/b3/ef4494438c90359e1547eaed3c5ec46e2c431d59a3de2af4e70ebd594c49/ormsgpack-1.12.1-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:12087214e436c1f6c28491949571abea759a63111908c4f7266586d78144d7a8", size = 376777, upload-time = "2025-12-14T07:57:38.795Z" }, + { url = "https://files.pythonhosted.org/packages/05/a0/1149a7163f8b0dfbc64bf9099b6f16d102ad3b03bcc11afee198d751da2d/ormsgpack-1.12.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e6d54c14cf86ef13f10ccade94d1e7de146aa9b17d371e18b16e95f329393b7", size = 202490, upload-time = "2025-12-14T07:57:40.168Z" }, + { url = "https://files.pythonhosted.org/packages/68/82/f2ec5e758d6a7106645cca9bb7137d98bce5d363789fa94075be6572057c/ormsgpack-1.12.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f3584d07882b7ea2a1a589f795a3af97fe4c2932b739408e6d1d9d286cad862", size = 211733, upload-time = "2025-12-14T07:57:42.253Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "paginate" +version = "0.5.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252, upload-time = "2024-08-25T14:17:24.139Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" }, + { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" }, + { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" }, + { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" }, + { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" }, + { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" }, + { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" }, + { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, + { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, + { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, + { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, + { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "pdfminer-six" +version = "20260107" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/a4/5cec1112009f0439a5ca6afa8ace321f0ab2f48da3255b7a1c8953014670/pdfminer_six-20260107.tar.gz", hash = "sha256:96bfd431e3577a55a0efd25676968ca4ce8fd5b53f14565f85716ff363889602", size = 8512094, upload-time = "2026-01-07T13:29:12.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/8b/28c4eaec9d6b036a52cb44720408f26b1a143ca9bce76cc19e8f5de00ab4/pdfminer_six-20260107-py3-none-any.whl", hash = "sha256:366585ba97e80dffa8f00cebe303d2f381884d8637af4ce422f1df3ef38111a9", size = 6592252, upload-time = "2026-01-07T13:29:10.742Z" }, +] + +[[package]] +name = "pdfplumber" +version = "0.11.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pdfminer-six" }, + { name = "pillow" }, + { name = "pypdfium2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/56/6f450312ba05a27d7713b73857c1a25100dbda04fbc1331b13fb227a607d/pdfplumber-0.11.10.tar.gz", hash = "sha256:b95b2d28c66efb0a794a83b88c6c6aea5987532a445d20a1cbcfa657022e6e57", size = 102892, upload-time = "2026-06-15T03:31:31.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/9a/07d658e1e7fad860f1c541ab941348125dbdab773be3a0afaf32361866c7/pdfplumber-0.11.10-py3-none-any.whl", hash = "sha256:7741ea81bf165b474b153e6789d10d18e06b6ddcf3ec84289c3ef2fed6802580", size = 60047, upload-time = "2026-06-15T03:31:29.702Z" }, +] + +[[package]] +name = "pendulum" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/7c/009c12b86c7cc6c403aec80f8a4308598dfc5995e5c523a5491faaa3952e/pendulum-3.1.0.tar.gz", hash = "sha256:66f96303560f41d097bee7d2dc98ffca716fbb3a832c4b3062034c2d45865015", size = 85930, upload-time = "2025-04-19T14:30:01.675Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/1f/af928ba4aa403dac9569f787adcf024005e7654433d71f7a84e608716837/pendulum-3.1.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:28658b0baf4b30eb31d096a375983cfed033e60c0a7bbe94fa23f06cd779b50b", size = 336209, upload-time = "2025-04-19T14:01:42.775Z" }, + { url = "https://files.pythonhosted.org/packages/b6/16/b010643007ba964c397da7fa622924423883c1bbff1a53f9d1022cd7f024/pendulum-3.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b114dcb99ce511cb8f5495c7b6f0056b2c3dba444ef1ea6e48030d7371bd531a", size = 323132, upload-time = "2025-04-19T14:01:44.577Z" }, + { url = "https://files.pythonhosted.org/packages/64/19/c3c47aeecb5d9bceb0e89faafd800d39809b696c5b7bba8ec8370ad5052c/pendulum-3.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2404a6a54c80252ea393291f0b7f35525a61abae3d795407f34e118a8f133a18", size = 341509, upload-time = "2025-04-19T14:01:46.084Z" }, + { url = "https://files.pythonhosted.org/packages/38/cf/c06921ff6b860ff7e62e70b8e5d4dc70e36f5abb66d168bd64d51760bc4e/pendulum-3.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d06999790d9ee9962a1627e469f98568bf7ad1085553fa3c30ed08b3944a14d7", size = 378674, upload-time = "2025-04-19T14:01:47.727Z" }, + { url = "https://files.pythonhosted.org/packages/62/0b/a43953b9eba11e82612b033ac5133f716f1b76b6108a65da6f408b3cc016/pendulum-3.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94751c52f6b7c306734d1044c2c6067a474237e1e5afa2f665d1fbcbbbcf24b3", size = 436133, upload-time = "2025-04-19T14:01:49.126Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a0/ec3d70b3b96e23ae1d039f132af35e17704c22a8250d1887aaefea4d78a6/pendulum-3.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5553ac27be05e997ec26d7f004cf72788f4ce11fe60bb80dda604a64055b29d0", size = 351232, upload-time = "2025-04-19T14:01:50.575Z" }, + { url = "https://files.pythonhosted.org/packages/f4/97/aba23f1716b82f6951ba2b1c9178a2d107d1e66c102762a9bf19988547ea/pendulum-3.1.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:f8dee234ca6142bf0514368d01a72945a44685aaa2fc4c14c98d09da9437b620", size = 521563, upload-time = "2025-04-19T14:01:51.9Z" }, + { url = "https://files.pythonhosted.org/packages/01/33/2c0d5216cc53d16db0c4b3d510f141ee0a540937f8675948541190fbd48b/pendulum-3.1.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7378084fe54faab4ee481897a00b710876f2e901ded6221671e827a253e643f2", size = 523221, upload-time = "2025-04-19T14:01:53.275Z" }, + { url = "https://files.pythonhosted.org/packages/51/89/8de955c339c31aeae77fd86d3225509b998c81875e9dba28cb88b8cbf4b3/pendulum-3.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:8539db7ae2c8da430ac2515079e288948c8ebf7eb1edd3e8281b5cdf433040d6", size = 260501, upload-time = "2025-04-19T14:01:54.749Z" }, + { url = "https://files.pythonhosted.org/packages/15/c3/226a3837363e94f8722461848feec18bfdd7d5172564d53aa3c3397ff01e/pendulum-3.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:1ce26a608e1f7387cd393fba2a129507c4900958d4f47b90757ec17656856571", size = 253087, upload-time = "2025-04-19T14:01:55.998Z" }, + { url = "https://files.pythonhosted.org/packages/6e/23/e98758924d1b3aac11a626268eabf7f3cf177e7837c28d47bf84c64532d0/pendulum-3.1.0-py3-none-any.whl", hash = "sha256:f9178c2a8e291758ade1e8dd6371b1d26d08371b4c7730a6e9a3ef8b16ebae0f", size = 111799, upload-time = "2025-04-19T14:02:34.739Z" }, +] + +[[package]] +name = "pgvector" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/6c/6d8b4b03b958c02fa8687ec6063c49d952a189f8c91ebbe51e877dfab8f7/pgvector-0.4.2.tar.gz", hash = "sha256:322cac0c1dc5d41c9ecf782bd9991b7966685dee3a00bc873631391ed949513a", size = 31354, upload-time = "2025-12-05T01:07:17.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/26/6cee8a1ce8c43625ec561aff19df07f9776b7525d9002c86bceb3e0ac970/pgvector-0.4.2-py3-none-any.whl", hash = "sha256:549d45f7a18593783d5eec609ea1684a724ba8405c4cb182a0b2b08aeff04e08", size = 27441, upload-time = "2025-12-05T01:07:16.536Z" }, +] + +[[package]] +name = "pillow" +version = "12.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c", size = 4100837, upload-time = "2026-04-01T14:43:41.506Z" }, + { url = "https://files.pythonhosted.org/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2", size = 4176528, upload-time = "2026-04-01T14:43:43.773Z" }, + { url = "https://files.pythonhosted.org/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c", size = 3640401, upload-time = "2026-04-01T14:43:45.87Z" }, + { url = "https://files.pythonhosted.org/packages/34/46/6c717baadcd62bc8ed51d238d521ab651eaa74838291bda1f86fe1f864c9/pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795", size = 5308094, upload-time = "2026-04-01T14:43:48.438Z" }, + { url = "https://files.pythonhosted.org/packages/71/43/905a14a8b17fdb1ccb58d282454490662d2cb89a6bfec26af6d3520da5ec/pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f", size = 4695402, upload-time = "2026-04-01T14:43:51.292Z" }, + { url = "https://files.pythonhosted.org/packages/73/dd/42107efcb777b16fa0393317eac58f5b5cf30e8392e266e76e51cff28c3d/pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed", size = 6280005, upload-time = "2026-04-01T14:43:54.242Z" }, + { url = "https://files.pythonhosted.org/packages/a8/68/b93e09e5e8549019e61acf49f65b1a8530765a7f812c77a7461bca7e4494/pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9", size = 8090669, upload-time = "2026-04-01T14:43:57.335Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/3ccb54ce8ec4ddd1accd2d89004308b7b0b21c4ac3d20fa70af4760a4330/pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed", size = 6395194, upload-time = "2026-04-01T14:43:59.864Z" }, + { url = "https://files.pythonhosted.org/packages/67/ee/21d4e8536afd1a328f01b359b4d3997b291ffd35a237c877b331c1c3b71c/pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3", size = 7082423, upload-time = "2026-04-01T14:44:02.74Z" }, + { url = "https://files.pythonhosted.org/packages/78/5f/e9f86ab0146464e8c133fe85df987ed9e77e08b29d8d35f9f9f4d6f917ba/pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9", size = 6505667, upload-time = "2026-04-01T14:44:05.381Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1e/409007f56a2fdce61584fd3acbc2bbc259857d555196cedcadc68c015c82/pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795", size = 7208580, upload-time = "2026-04-01T14:44:08.39Z" }, + { url = "https://files.pythonhosted.org/packages/23/c4/7349421080b12fb35414607b8871e9534546c128a11965fd4a7002ccfbee/pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e", size = 6375896, upload-time = "2026-04-01T14:44:11.197Z" }, + { url = "https://files.pythonhosted.org/packages/3f/82/8a3739a5e470b3c6cbb1d21d315800d8e16bff503d1f16b03a4ec3212786/pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b", size = 7081266, upload-time = "2026-04-01T14:44:13.947Z" }, + { url = "https://files.pythonhosted.org/packages/c3/25/f968f618a062574294592f668218f8af564830ccebdd1fa6200f598e65c5/pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06", size = 2463508, upload-time = "2026-04-01T14:44:16.312Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a4/b342930964e3cb4dce5038ae34b0eab4653334995336cd486c5a8c25a00c/pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b", size = 5309927, upload-time = "2026-04-01T14:44:18.89Z" }, + { url = "https://files.pythonhosted.org/packages/9f/de/23198e0a65a9cf06123f5435a5d95cea62a635697f8f03d134d3f3a96151/pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f", size = 4698624, upload-time = "2026-04-01T14:44:21.115Z" }, + { url = "https://files.pythonhosted.org/packages/01/a6/1265e977f17d93ea37aa28aa81bad4fa597933879fac2520d24e021c8da3/pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612", size = 6321252, upload-time = "2026-04-01T14:44:23.663Z" }, + { url = "https://files.pythonhosted.org/packages/3c/83/5982eb4a285967baa70340320be9f88e57665a387e3a53a7f0db8231a0cd/pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c", size = 8126550, upload-time = "2026-04-01T14:44:26.772Z" }, + { url = "https://files.pythonhosted.org/packages/4e/48/6ffc514adce69f6050d0753b1a18fd920fce8cac87620d5a31231b04bfc5/pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea", size = 6433114, upload-time = "2026-04-01T14:44:29.615Z" }, + { url = "https://files.pythonhosted.org/packages/36/a3/f9a77144231fb8d40ee27107b4463e205fa4677e2ca2548e14da5cf18dce/pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4", size = 7115667, upload-time = "2026-04-01T14:44:32.773Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fc/ac4ee3041e7d5a565e1c4fd72a113f03b6394cc72ab7089d27608f8aaccb/pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4", size = 6538966, upload-time = "2026-04-01T14:44:35.252Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a8/27fb307055087f3668f6d0a8ccb636e7431d56ed0750e07a60547b1e083e/pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea", size = 7238241, upload-time = "2026-04-01T14:44:37.875Z" }, + { url = "https://files.pythonhosted.org/packages/ad/4b/926ab182c07fccae9fcb120043464e1ff1564775ec8864f21a0ebce6ac25/pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24", size = 6379592, upload-time = "2026-04-01T14:44:40.336Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c4/f9e476451a098181b30050cc4c9a3556b64c02cf6497ea421ac047e89e4b/pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98", size = 7085542, upload-time = "2026-04-01T14:44:43.251Z" }, + { url = "https://files.pythonhosted.org/packages/00/a4/285f12aeacbe2d6dc36c407dfbbe9e96d4a80b0fb710a337f6d2ad978c75/pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453", size = 2465765, upload-time = "2026-04-01T14:44:45.996Z" }, + { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848, upload-time = "2026-04-01T14:44:48.48Z" }, + { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515, upload-time = "2026-04-01T14:44:51.353Z" }, + { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159, upload-time = "2026-04-01T14:44:53.588Z" }, + { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185, upload-time = "2026-04-01T14:44:56.039Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386, upload-time = "2026-04-01T14:44:58.663Z" }, + { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384, upload-time = "2026-04-01T14:45:01.5Z" }, + { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599, upload-time = "2026-04-01T14:45:04.5Z" }, + { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021, upload-time = "2026-04-01T14:45:07.117Z" }, + { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360, upload-time = "2026-04-01T14:45:09.763Z" }, + { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628, upload-time = "2026-04-01T14:45:12.378Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321, upload-time = "2026-04-01T14:45:15.122Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723, upload-time = "2026-04-01T14:45:17.797Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400, upload-time = "2026-04-01T14:45:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835, upload-time = "2026-04-01T14:45:23.162Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225, upload-time = "2026-04-01T14:45:25.637Z" }, + { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541, upload-time = "2026-04-01T14:45:28.355Z" }, + { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251, upload-time = "2026-04-01T14:45:30.924Z" }, + { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807, upload-time = "2026-04-01T14:45:33.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935, upload-time = "2026-04-01T14:45:36.623Z" }, + { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720, upload-time = "2026-04-01T14:45:39.258Z" }, + { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498, upload-time = "2026-04-01T14:45:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413, upload-time = "2026-04-01T14:45:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084, upload-time = "2026-04-01T14:45:47.568Z" }, + { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152, upload-time = "2026-04-01T14:45:50.032Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/33/9611380c2bdb1225fdef633e2a9610622310fed35ab11dac9620972ee088/platformdirs-4.5.0.tar.gz", hash = "sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312", size = 21632, upload-time = "2025-10-08T17:44:48.791Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pre-commit" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/29/7cf5bbc236333876e4b41f56e06857a87937ce4bf91e117a6991a2dbb02a/pre_commit-4.3.0.tar.gz", hash = "sha256:499fe450cc9d42e9d58e606262795ecb64dd05438943c62b66f6a8673da30b16", size = 193792, upload-time = "2025-08-09T18:56:14.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8", size = 220965, upload-time = "2025-08-09T18:56:13.192Z" }, +] + +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + +[[package]] +name = "psutil" +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/5a/07871137bb752428aa4b659f910b399ba6f291156bdea939be3e96cae7cb/psutil-6.1.1.tar.gz", hash = "sha256:cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5", size = 508502, upload-time = "2024-12-19T18:21:20.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/99/ca79d302be46f7bdd8321089762dd4476ee725fce16fc2b2e1dbba8cac17/psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8", size = 247511, upload-time = "2024-12-19T18:21:45.163Z" }, + { url = "https://files.pythonhosted.org/packages/0b/6b/73dbde0dd38f3782905d4587049b9be64d76671042fdcaf60e2430c6796d/psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377", size = 248985, upload-time = "2024-12-19T18:21:49.254Z" }, + { url = "https://files.pythonhosted.org/packages/17/38/c319d31a1d3f88c5b79c68b3116c129e5133f1822157dd6da34043e32ed6/psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003", size = 284488, upload-time = "2024-12-19T18:21:51.638Z" }, + { url = "https://files.pythonhosted.org/packages/9c/39/0f88a830a1c8a3aba27fededc642da37613c57cbff143412e3536f89784f/psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160", size = 287477, upload-time = "2024-12-19T18:21:55.306Z" }, + { url = "https://files.pythonhosted.org/packages/47/da/99f4345d4ddf2845cb5b5bd0d93d554e84542d116934fde07a0c50bd4e9f/psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3", size = 289017, upload-time = "2024-12-19T18:21:57.875Z" }, + { url = "https://files.pythonhosted.org/packages/38/53/bd755c2896f4461fd4f36fa6a6dcb66a88a9e4b9fd4e5b66a77cf9d4a584/psutil-6.1.1-cp37-abi3-win32.whl", hash = "sha256:eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53", size = 250602, upload-time = "2024-12-19T18:22:08.808Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d7/7831438e6c3ebbfa6e01a927127a6cb42ad3ab844247f3c5b96bea25d73d/psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649", size = 254444, upload-time = "2024-12-19T18:22:11.335Z" }, +] + +[[package]] +name = "psycopg" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/1a/7d9ef4fdc13ef7f15b934c393edc97a35c281bb7d3c3329fbfcbe915a7c2/psycopg-3.3.2.tar.gz", hash = "sha256:707a67975ee214d200511177a6a80e56e654754c9afca06a7194ea6bbfde9ca7", size = 165630, upload-time = "2025-12-06T17:34:53.899Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/51/2779ccdf9305981a06b21a6b27e8547c948d85c41c76ff434192784a4c93/psycopg-3.3.2-py3-none-any.whl", hash = "sha256:3e94bc5f4690247d734599af56e51bae8e0db8e4311ea413f801fef82b14a99b", size = 212774, upload-time = "2025-12-06T17:31:41.414Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/73/7ca7cb22b9ac7393fb5de7d28ca97e8347c375c8498b3bff2c99c1f38038/psycopg_binary-3.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fc5a189e89cbfff174588665bb18d28d2d0428366cc9dae5864afcaa2e57380b", size = 4579068, upload-time = "2025-12-06T17:33:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/f5/42/0cf38ff6c62c792fc5b55398a853a77663210ebd51ed6f0c4a05b06f95a6/psycopg_binary-3.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:083c2e182be433f290dc2c516fd72b9b47054fcd305cce791e0a50d9e93e06f2", size = 4657520, upload-time = "2025-12-06T17:33:42.536Z" }, + { url = "https://files.pythonhosted.org/packages/3b/60/df846bc84cbf2231e01b0fff48b09841fe486fa177665e50f4995b1bfa44/psycopg_binary-3.3.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:ac230e3643d1c436a2dfb59ca84357dfc6862c9f372fc5dbd96bafecae581f9f", size = 5452086, upload-time = "2025-12-06T17:33:46.54Z" }, + { url = "https://files.pythonhosted.org/packages/ab/85/30c846a00db86b1b53fd5bfd4b4edfbd0c00de8f2c75dd105610bd7568fc/psycopg_binary-3.3.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d8c899a540f6c7585cee53cddc929dd4d2db90fd828e37f5d4017b63acbc1a5d", size = 5131125, upload-time = "2025-12-06T17:33:50.413Z" }, + { url = "https://files.pythonhosted.org/packages/6d/15/9968732013373f36f8a2a3fb76104dffc8efd9db78709caa5ae1a87b1f80/psycopg_binary-3.3.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50ff10ab8c0abdb5a5451b9315538865b50ba64c907742a1385fdf5f5772b73e", size = 6722914, upload-time = "2025-12-06T17:33:54.544Z" }, + { url = "https://files.pythonhosted.org/packages/b2/ba/29e361fe02143ac5ff5a1ca3e45697344cfbebe2eaf8c4e7eec164bff9a0/psycopg_binary-3.3.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:23d2594af848c1fd3d874a9364bef50730124e72df7bb145a20cb45e728c50ed", size = 4966081, upload-time = "2025-12-06T17:33:58.477Z" }, + { url = "https://files.pythonhosted.org/packages/99/45/1be90c8f1a1a237046903e91202fb06708745c179f220b361d6333ed7641/psycopg_binary-3.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ea4fe6b4ead3bbbe27244ea224fcd1f53cb119afc38b71a2f3ce570149a03e30", size = 4493332, upload-time = "2025-12-06T17:34:02.011Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b5/bbdc07d5f0a5e90c617abd624368182aa131485e18038b2c6c85fc054aed/psycopg_binary-3.3.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:742ce48cde825b8e52fb1a658253d6d1ff66d152081cbc76aa45e2986534858d", size = 4170781, upload-time = "2025-12-06T17:34:05.298Z" }, + { url = "https://files.pythonhosted.org/packages/d1/2a/0d45e4f4da2bd78c3237ffa03475ef3751f69a81919c54a6e610eb1a7c96/psycopg_binary-3.3.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e22bf6b54df994aff37ab52695d635f1ef73155e781eee1f5fa75bc08b58c8da", size = 3910544, upload-time = "2025-12-06T17:34:08.251Z" }, + { url = "https://files.pythonhosted.org/packages/3a/62/a8e0f092f4dbef9a94b032fb71e214cf0a375010692fbe7493a766339e47/psycopg_binary-3.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8db9034cde3bcdafc66980f0130813f5c5d19e74b3f2a19fb3cfbc25ad113121", size = 4220070, upload-time = "2025-12-06T17:34:11.392Z" }, + { url = "https://files.pythonhosted.org/packages/09/e6/5fc8d8aff8afa114bb4a94a0341b9309311e8bf3ab32d816032f8b984d4e/psycopg_binary-3.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:df65174c7cf6b05ea273ce955927d3270b3a6e27b0b12762b009ce6082b8d3fc", size = 3540922, upload-time = "2025-12-06T17:34:14.88Z" }, + { url = "https://files.pythonhosted.org/packages/bd/75/ad18c0b97b852aba286d06befb398cc6d383e9dfd0a518369af275a5a526/psycopg_binary-3.3.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9ca24062cd9b2270e4d77576042e9cc2b1d543f09da5aba1f1a3d016cea28390", size = 4596371, upload-time = "2025-12-06T17:34:18.007Z" }, + { url = "https://files.pythonhosted.org/packages/5a/79/91649d94c8d89f84af5da7c9d474bfba35b08eb8f492ca3422b08f0a6427/psycopg_binary-3.3.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c749770da0947bc972e512f35366dd4950c0e34afad89e60b9787a37e97cb443", size = 4675139, upload-time = "2025-12-06T17:34:21.374Z" }, + { url = "https://files.pythonhosted.org/packages/56/ac/b26e004880f054549ec9396594e1ffe435810b0673e428e619ed722e4244/psycopg_binary-3.3.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:03b7cd73fb8c45d272a34ae7249713e32492891492681e3cf11dff9531cf37e9", size = 5456120, upload-time = "2025-12-06T17:34:25.102Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/410681dccd6f2999fb115cc248521ec50dd2b0aba66ae8de7e81efdebbee/psycopg_binary-3.3.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:43b130e3b6edcb5ee856c7167ccb8561b473308c870ed83978ae478613764f1c", size = 5133484, upload-time = "2025-12-06T17:34:28.933Z" }, + { url = "https://files.pythonhosted.org/packages/66/30/ebbab99ea2cfa099d7b11b742ce13415d44f800555bfa4ad2911dc645b71/psycopg_binary-3.3.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c1feba5a8c617922321aef945865334e468337b8fc5c73074f5e63143013b5a", size = 6731818, upload-time = "2025-12-06T17:34:33.094Z" }, + { url = "https://files.pythonhosted.org/packages/70/02/d260646253b7ad805d60e0de47f9b811d6544078452579466a098598b6f4/psycopg_binary-3.3.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cabb2a554d9a0a6bf84037d86ca91782f087dfff2a61298d0b00c19c0bc43f6d", size = 4983859, upload-time = "2025-12-06T17:34:36.457Z" }, + { url = "https://files.pythonhosted.org/packages/72/8d/e778d7bad1a7910aa36281f092bd85c5702f508fd9bb0ea2020ffbb6585c/psycopg_binary-3.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:74bc306c4b4df35b09bc8cecf806b271e1c5d708f7900145e4e54a2e5dedfed0", size = 4516388, upload-time = "2025-12-06T17:34:40.129Z" }, + { url = "https://files.pythonhosted.org/packages/bd/f1/64e82098722e2ab3521797584caf515284be09c1e08a872551b6edbb0074/psycopg_binary-3.3.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:d79b0093f0fbf7a962d6a46ae292dc056c65d16a8ee9361f3cfbafd4c197ab14", size = 4192382, upload-time = "2025-12-06T17:34:43.279Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d0/c20f4e668e89494972e551c31be2a0016e3f50d552d7ae9ac07086407599/psycopg_binary-3.3.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1586e220be05547c77afc326741dd41cc7fba38a81f9931f616ae98865439678", size = 3928660, upload-time = "2025-12-06T17:34:46.757Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e1/99746c171de22539fd5eb1c9ca21dc805b54cfae502d7451d237d1dbc349/psycopg_binary-3.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:458696a5fa5dad5b6fb5d5862c22454434ce4fe1cf66ca6c0de5f904cbc1ae3e", size = 4239169, upload-time = "2025-12-06T17:34:49.751Z" }, + { url = "https://files.pythonhosted.org/packages/72/f7/212343c1c9cfac35fd943c527af85e9091d633176e2a407a0797856ff7b9/psycopg_binary-3.3.2-cp314-cp314-win_amd64.whl", hash = "sha256:04bb2de4ba69d6f8395b446ede795e8884c040ec71d01dd07ac2b2d18d4153d1", size = 3642122, upload-time = "2025-12-06T17:34:52.506Z" }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/ac7e0aae12027748076d72a8764ff1c9d82ca75a7a52622e67ed3f765c54/pydantic_settings-2.12.0.tar.gz", hash = "sha256:005538ef951e3c2a68e1c08b292b5f2e71490def8589d4221b95dab00dafcfd0", size = 194184, upload-time = "2025-11-10T14:25:47.013Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809", size = 51880, upload-time = "2025-11-10T14:25:45.546Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.10.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785, upload-time = "2024-11-28T03:43:29.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + +[[package]] +name = "pymdown-extensions" +version = "10.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/b3/6d2b3f149bc5413b0a29761c2c5832d8ce904a1d7f621e86616d96f505cc/pymdown_extensions-10.16.1.tar.gz", hash = "sha256:aace82bcccba3efc03e25d584e6a22d27a8e17caa3f4dd9f207e49b787aa9a91", size = 853277, upload-time = "2025-07-28T16:19:34.167Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/06/43084e6cbd4b3bc0e80f6be743b2e79fbc6eed8de9ad8c629939fa55d972/pymdown_extensions-10.16.1-py3-none-any.whl", hash = "sha256:d6ba157a6c03146a7fb122b2b9a121300056384eafeec9c9f9e584adfdb2a32d", size = 266178, upload-time = "2025-07-28T16:19:31.401Z" }, +] + +[[package]] +name = "pypdfium2" +version = "5.10.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/78/d9b45abb97a3686643f7c6472a5f7688f2013a373226121dc76b9debbacf/pypdfium2-5.10.1.tar.gz", hash = "sha256:f257d2011eb43c846b7e9f5a802e28646b29732763e4a35dd6ca76f9be580538", size = 272963, upload-time = "2026-06-15T10:09:16.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/49/0b85fcc0d236582143a25cf275b0b4f5d786f51eb07a89ec9c79d43efe18/pypdfium2-5.10.1-py3-none-android_23_arm64_v8a.whl", hash = "sha256:13abf7a9f5e0ddebc8bbcccea5f13ae5abe8a298ea219e125b0fc24c1d2171b4", size = 3409176, upload-time = "2026-06-15T10:08:36.017Z" }, + { url = "https://files.pythonhosted.org/packages/20/d2/2f522c5b2ad5166edf256bb4dbab97de5d07b2573f8a5630ddfcd1f8d4ee/pypdfium2-5.10.1-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:a0dc52b56631e2f7edcdb22bae3b155aa840bec32c5bd05781e90baaecee88e7", size = 2866175, upload-time = "2026-06-15T10:08:37.955Z" }, + { url = "https://files.pythonhosted.org/packages/a7/be/477548c026c2badfdbf4afc3358b7135121fb5bec2e2effcb67e3a674d0b/pypdfium2-5.10.1-py3-none-macosx_12_0_arm64.whl", hash = "sha256:ebb9e63f92d15fc41b359fe7a187233dfae37548800e1fa09cb2fc466ac89951", size = 3621427, upload-time = "2026-06-15T10:08:39.969Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9b/0131c7f711b62c6edd6b200e9eb6340be6de4f6dc5baae625e3394d8d5fb/pypdfium2-5.10.1-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:d04f2050b6b32bb18624688b600543342a4ab3aacf64bf66521a5af72bbc7de1", size = 3682825, upload-time = "2026-06-15T10:08:42.181Z" }, + { url = "https://files.pythonhosted.org/packages/9d/3d/bddfceb6e67e54d6dd1ab6c0f1feff796a89596e40f6345a3b4cc6a3d408/pypdfium2-5.10.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0de53d2710ca9509fc2812340acc57c3f043697609068592d87de654f8cabf44", size = 3682206, upload-time = "2026-06-15T10:08:44.087Z" }, + { url = "https://files.pythonhosted.org/packages/58/08/dedeb25c6645fc8a5eda24f54e0b1d083b7334ababf5f518bb939d729cc0/pypdfium2-5.10.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:10a26ce04795f8ec079e81c707fcb5737061e8a78025babc3d6e36642e9c903a", size = 3413720, upload-time = "2026-06-15T10:08:45.739Z" }, + { url = "https://files.pythonhosted.org/packages/cc/48/69e1fc8b1216005243c6415183bbf6de1cda3f5a06758b3fa4a26a7385c6/pypdfium2-5.10.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be6d2a8d1bcfd777188e7aa55a25c83f34d54e8350ad8810fb32017787d9b0d9", size = 3812913, upload-time = "2026-06-15T10:08:47.45Z" }, + { url = "https://files.pythonhosted.org/packages/72/7f/132455a58ad736d76815c6cd1307532c3f433299d945b9d2f8cc2387c309/pypdfium2-5.10.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cf4d2527f79f31c550490cc74c9f32e19385a944630a3ef4cd4d9b6f961fbf77", size = 4223220, upload-time = "2026-06-15T10:08:49.099Z" }, + { url = "https://files.pythonhosted.org/packages/bf/8c/4d5804eca598bbe894e0a9a510807e221c1623e7276ce6b68fa2660dc933/pypdfium2-5.10.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ba127750bc3f4161461538d532d74491cd976f584f1753a6cee9cb821338ec1", size = 3738950, upload-time = "2026-06-15T10:08:50.821Z" }, + { url = "https://files.pythonhosted.org/packages/d1/7f/baac59bf14ff914d97789ee0368c22ae233aa857aa9c0726bcb515dbc4d7/pypdfium2-5.10.1-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:80f30517ee089dfbbc6e9de6da365b6e8c0ce8f80c40b7d4025a3b1d3bbd8a70", size = 4029869, upload-time = "2026-06-15T10:08:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d7/a5d58a0bcba31a0e37ed636a76ef3d2d215733f28af61637287d061b0c54/pypdfium2-5.10.1-py3-none-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ad5f5de15febb788c6eb3853e58aceda9cd8c5187c92472abaeecf9558deb0cf", size = 3990927, upload-time = "2026-06-15T10:08:54.555Z" }, + { url = "https://files.pythonhosted.org/packages/da/1a/98eebd14b36812176297cf765d504ebeebc982f895c8a3a9fbd2717797de/pypdfium2-5.10.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:8947fa3cd808da33960bdf8ff9e5247aa94ccd94f0b09bb3402e99498d83bcc9", size = 4989624, upload-time = "2026-06-15T10:08:56.284Z" }, + { url = "https://files.pythonhosted.org/packages/1c/20/f2e124d607b8bb90a9f1ce976afff38c70e215cd7ea86af784cb2e8a19dd/pypdfium2-5.10.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:139a6387a3a2652f288e53164268bb03af4fa221d78484ee18407053a60082a3", size = 4535124, upload-time = "2026-06-15T10:08:58.214Z" }, + { url = "https://files.pythonhosted.org/packages/59/ef/469ea87f668a32ff3280ea15e522e3a7858d2c80f1ecc320ece1244624c9/pypdfium2-5.10.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:172ff3e10358d66456e27fb0b8b5098e28ec24e51072eb4b5b86077d550e21bb", size = 5229373, upload-time = "2026-06-15T10:09:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/c3/48/c7ed3001f0c5e28114c98bf918c8121e422a65ba7323e1e12f3f28e2d278/pypdfium2-5.10.1-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:09fda0609dc4749c9865a0315c447d6f2583a580693bcabd69980d3fbb22ad51", size = 5140010, upload-time = "2026-06-15T10:09:02.734Z" }, + { url = "https://files.pythonhosted.org/packages/b2/bc/00b731bfc1fdc0f3c7d91108fceb54978ff4f5a92336820e5ed6c12535ff/pypdfium2-5.10.1-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:6b01adcfe9aaf7a635a59bed5687fd4fd7b0da292664f050d4ebd2bfa5c70584", size = 4643310, upload-time = "2026-06-15T10:09:04.794Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e8/3ad242233f657c19092a8c83684c8154b8d02a826535a6a2591d91aa5dde/pypdfium2-5.10.1-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:610e14c37d2090b826dccc0604fc7e7612c0ce591190780ae228abdb9abb971e", size = 5087879, upload-time = "2026-06-15T10:09:06.861Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/9d0747a02ac3021ed3db7ac27c5187d97e78b0253a4bdfbf386666968474/pypdfium2-5.10.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f2803020952afa57e1e148adc19369b835154e1fa251a1ca85008ab6466a710f", size = 5047369, upload-time = "2026-06-15T10:09:08.883Z" }, + { url = "https://files.pythonhosted.org/packages/9e/41/7d5187e9527eae81890a3b13193442112ef788d61ddaa68a6d59ac447bec/pypdfium2-5.10.1-py3-none-win32.whl", hash = "sha256:8702bb4f01ddfc8e7757b41b4c2c8392ac17c9f0234476e1e69672ea7c6d6aa0", size = 3680056, upload-time = "2026-06-15T10:09:10.95Z" }, + { url = "https://files.pythonhosted.org/packages/16/1d/c62bd59dd8345cc4b640f942f465633f6b07b859d01ddb648610a7bf5c7c/pypdfium2-5.10.1-py3-none-win_amd64.whl", hash = "sha256:58da5b51fb7884c7d21a05062ab13edb011d1a08dfd9694f3d5d685df62796b9", size = 3812105, upload-time = "2026-06-15T10:09:12.684Z" }, + { url = "https://files.pythonhosted.org/packages/10/d5/21bac39125df8a93e99c04583486b58a62b5997d6b3541e3ad0f69053392/pypdfium2-5.10.1-py3-none-win_arm64.whl", hash = "sha256:e3301c2f7a66fb8cb57dba857d0c9e90215e178f6602a87c5a306cd98513dab8", size = 3600043, upload-time = "2026-06-15T10:09:14.606Z" }, +] + +[[package]] +name = "pyreadline3" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/6d/f94028646d7bbe6d9d873c47ee7c246f2d29129d253f0d96cb6fcab70733/pyreadline3-3.5.6.tar.gz", hash = "sha256:61e53218b99656091ddb077df9e71f25850e72e030b6183b39c9b7e6e4f4a9bf", size = 100368, upload-time = "2026-05-14T17:55:04.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/5e/35c856e186b74678c24927847ad9895a51f1bc02a0c6126477a6c6040064/pyreadline3-3.5.6-py3-none-any.whl", hash = "sha256:8449b734232e42a5dcd74048e39b60db2839a4c38cf3ae2bf7707d58b5389c0d", size = 85243, upload-time = "2026-05-14T17:55:03.262Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/26/19cadc79a718c5edbec86fd4919a6b6d3f681039a2f6d66d14be94e75fb9/python_dotenv-1.2.1.tar.gz", hash = "sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6", size = 44221, upload-time = "2025-10-26T15:12:10.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" }, +] + +[[package]] +name = "python-multipart" +version = "0.0.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/01/979e98d542a70714b0cb2b6728ed0b7c46792b695e3eaec3e20711271ca3/python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58", size = 37612, upload-time = "2026-01-25T10:15:56.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/d0/397f9626e711ff749a95d96b7af99b9c566a9bb5129b8e4c10fc4d100304/python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155", size = 24579, upload-time = "2026-01-25T10:15:54.811Z" }, +] + +[[package]] +name = "python-pptx" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "pillow" }, + { name = "typing-extensions" }, + { name = "xlsxwriter" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297, upload-time = "2024-08-07T17:33:37.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788, upload-time = "2024-08-07T17:33:28.192Z" }, +] + +[[package]] +name = "pywin32" +version = "311" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/be/3fd5de0979fcb3994bfee0d65ed8ca9506a8a1260651b86174f6a86f52b3/pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d", size = 8705700, upload-time = "2025-07-14T20:13:26.471Z" }, + { url = "https://files.pythonhosted.org/packages/e3/28/e0a1909523c6890208295a29e05c2adb2126364e289826c0a8bc7297bd5c/pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d", size = 9494700, upload-time = "2025-07-14T20:13:28.243Z" }, + { url = "https://files.pythonhosted.org/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a", size = 8709318, upload-time = "2025-07-14T20:13:30.348Z" }, + { url = "https://files.pythonhosted.org/packages/c9/31/097f2e132c4f16d99a22bfb777e0fd88bd8e1c634304e102f313af69ace5/pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee", size = 8840714, upload-time = "2025-07-14T20:13:32.449Z" }, + { url = "https://files.pythonhosted.org/packages/90/4b/07c77d8ba0e01349358082713400435347df8426208171ce297da32c313d/pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87", size = 9656800, upload-time = "2025-07-14T20:13:34.312Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540, upload-time = "2025-07-14T20:13:36.379Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "pyyaml-env-tag" +version = "1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "regex" +version = "2026.1.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/86/07d5056945f9ec4590b518171c4254a5925832eb727b56d3c38a7476f316/regex-2026.1.15.tar.gz", hash = "sha256:164759aa25575cbc0651bef59a0b18353e54300d79ace8084c818ad8ac72b7d5", size = 414811, upload-time = "2026-01-14T23:18:02.775Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/2e/6870bb16e982669b674cce3ee9ff2d1d46ab80528ee6bcc20fb2292efb60/regex-2026.1.15-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e69d0deeb977ffe7ed3d2e4439360089f9c3f217ada608f0f88ebd67afb6385e", size = 489164, upload-time = "2026-01-14T23:15:13.962Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/9774542e203849b0286badf67199970a44ebdb0cc5fb739f06e47ada72f8/regex-2026.1.15-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3601ffb5375de85a16f407854d11cca8fe3f5febbe3ac78fb2866bb220c74d10", size = 291218, upload-time = "2026-01-14T23:15:15.647Z" }, + { url = "https://files.pythonhosted.org/packages/b2/87/b0cda79f22b8dee05f774922a214da109f9a4c0eca5da2c9d72d77ea062c/regex-2026.1.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4c5ef43b5c2d4114eb8ea424bb8c9cec01d5d17f242af88b2448f5ee81caadbc", size = 288895, upload-time = "2026-01-14T23:15:17.788Z" }, + { url = "https://files.pythonhosted.org/packages/3b/6a/0041f0a2170d32be01ab981d6346c83a8934277d82c780d60b127331f264/regex-2026.1.15-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:968c14d4f03e10b2fd960f1d5168c1f0ac969381d3c1fcc973bc45fb06346599", size = 798680, upload-time = "2026-01-14T23:15:19.342Z" }, + { url = "https://files.pythonhosted.org/packages/58/de/30e1cfcdbe3e891324aa7568b7c968771f82190df5524fabc1138cb2d45a/regex-2026.1.15-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56a5595d0f892f214609c9f76b41b7428bed439d98dc961efafdd1354d42baae", size = 864210, upload-time = "2026-01-14T23:15:22.005Z" }, + { url = "https://files.pythonhosted.org/packages/64/44/4db2f5c5ca0ccd40ff052ae7b1e9731352fcdad946c2b812285a7505ca75/regex-2026.1.15-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf650f26087363434c4e560011f8e4e738f6f3e029b85d4904c50135b86cfa5", size = 912358, upload-time = "2026-01-14T23:15:24.569Z" }, + { url = "https://files.pythonhosted.org/packages/79/b6/e6a5665d43a7c42467138c8a2549be432bad22cbd206f5ec87162de74bd7/regex-2026.1.15-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18388a62989c72ac24de75f1449d0fb0b04dfccd0a1a7c1c43af5eb503d890f6", size = 803583, upload-time = "2026-01-14T23:15:26.526Z" }, + { url = "https://files.pythonhosted.org/packages/e7/53/7cd478222169d85d74d7437e74750005e993f52f335f7c04ff7adfda3310/regex-2026.1.15-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6d220a2517f5893f55daac983bfa9fe998a7dbcaee4f5d27a88500f8b7873788", size = 775782, upload-time = "2026-01-14T23:15:29.352Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b5/75f9a9ee4b03a7c009fe60500fe550b45df94f0955ca29af16333ef557c5/regex-2026.1.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c9c08c2fbc6120e70abff5d7f28ffb4d969e14294fb2143b4b5c7d20e46d1714", size = 787978, upload-time = "2026-01-14T23:15:31.295Z" }, + { url = "https://files.pythonhosted.org/packages/72/b3/79821c826245bbe9ccbb54f6eadb7879c722fd3e0248c17bfc90bf54e123/regex-2026.1.15-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7ef7d5d4bd49ec7364315167a4134a015f61e8266c6d446fc116a9ac4456e10d", size = 858550, upload-time = "2026-01-14T23:15:33.558Z" }, + { url = "https://files.pythonhosted.org/packages/4a/85/2ab5f77a1c465745bfbfcb3ad63178a58337ae8d5274315e2cc623a822fa/regex-2026.1.15-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:6e42844ad64194fa08d5ccb75fe6a459b9b08e6d7296bd704460168d58a388f3", size = 763747, upload-time = "2026-01-14T23:15:35.206Z" }, + { url = "https://files.pythonhosted.org/packages/6d/84/c27df502d4bfe2873a3e3a7cf1bdb2b9cc10284d1a44797cf38bed790470/regex-2026.1.15-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:cfecdaa4b19f9ca534746eb3b55a5195d5c95b88cac32a205e981ec0a22b7d31", size = 850615, upload-time = "2026-01-14T23:15:37.523Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b7/658a9782fb253680aa8ecb5ccbb51f69e088ed48142c46d9f0c99b46c575/regex-2026.1.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08df9722d9b87834a3d701f3fca570b2be115654dbfd30179f30ab2f39d606d3", size = 789951, upload-time = "2026-01-14T23:15:39.582Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2a/5928af114441e059f15b2f63e188bd00c6529b3051c974ade7444b85fcda/regex-2026.1.15-cp313-cp313-win32.whl", hash = "sha256:d426616dae0967ca225ab12c22274eb816558f2f99ccb4a1d52ca92e8baf180f", size = 266275, upload-time = "2026-01-14T23:15:42.108Z" }, + { url = "https://files.pythonhosted.org/packages/4f/16/5bfbb89e435897bff28cf0352a992ca719d9e55ebf8b629203c96b6ce4f7/regex-2026.1.15-cp313-cp313-win_amd64.whl", hash = "sha256:febd38857b09867d3ed3f4f1af7d241c5c50362e25ef43034995b77a50df494e", size = 277145, upload-time = "2026-01-14T23:15:44.244Z" }, + { url = "https://files.pythonhosted.org/packages/56/c1/a09ff7392ef4233296e821aec5f78c51be5e91ffde0d163059e50fd75835/regex-2026.1.15-cp313-cp313-win_arm64.whl", hash = "sha256:8e32f7896f83774f91499d239e24cebfadbc07639c1494bb7213983842348337", size = 270411, upload-time = "2026-01-14T23:15:45.858Z" }, + { url = "https://files.pythonhosted.org/packages/3c/38/0cfd5a78e5c6db00e6782fdae70458f89850ce95baa5e8694ab91d89744f/regex-2026.1.15-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:ec94c04149b6a7b8120f9f44565722c7ae31b7a6d2275569d2eefa76b83da3be", size = 492068, upload-time = "2026-01-14T23:15:47.616Z" }, + { url = "https://files.pythonhosted.org/packages/50/72/6c86acff16cb7c959c4355826bbf06aad670682d07c8f3998d9ef4fee7cd/regex-2026.1.15-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:40c86d8046915bb9aeb15d3f3f15b6fd500b8ea4485b30e1bbc799dab3fe29f8", size = 292756, upload-time = "2026-01-14T23:15:49.307Z" }, + { url = "https://files.pythonhosted.org/packages/4e/58/df7fb69eadfe76526ddfce28abdc0af09ffe65f20c2c90932e89d705153f/regex-2026.1.15-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:726ea4e727aba21643205edad8f2187ec682d3305d790f73b7a51c7587b64bdd", size = 291114, upload-time = "2026-01-14T23:15:51.484Z" }, + { url = "https://files.pythonhosted.org/packages/ed/6c/a4011cd1cf96b90d2cdc7e156f91efbd26531e822a7fbb82a43c1016678e/regex-2026.1.15-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1cb740d044aff31898804e7bf1181cc72c03d11dfd19932b9911ffc19a79070a", size = 807524, upload-time = "2026-01-14T23:15:53.102Z" }, + { url = "https://files.pythonhosted.org/packages/1d/25/a53ffb73183f69c3e9f4355c4922b76d2840aee160af6af5fac229b6201d/regex-2026.1.15-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05d75a668e9ea16f832390d22131fe1e8acc8389a694c8febc3e340b0f810b93", size = 873455, upload-time = "2026-01-14T23:15:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/66/0b/8b47fc2e8f97d9b4a851736f3890a5f786443aa8901061c55f24c955f45b/regex-2026.1.15-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d991483606f3dbec93287b9f35596f41aa2e92b7c2ebbb935b63f409e243c9af", size = 915007, upload-time = "2026-01-14T23:15:57.041Z" }, + { url = "https://files.pythonhosted.org/packages/c2/fa/97de0d681e6d26fabe71968dbee06dd52819e9a22fdce5dac7256c31ed84/regex-2026.1.15-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:194312a14819d3e44628a44ed6fea6898fdbecb0550089d84c403475138d0a09", size = 812794, upload-time = "2026-01-14T23:15:58.916Z" }, + { url = "https://files.pythonhosted.org/packages/22/38/e752f94e860d429654aa2b1c51880bff8dfe8f084268258adf9151cf1f53/regex-2026.1.15-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe2fda4110a3d0bc163c2e0664be44657431440722c5c5315c65155cab92f9e5", size = 781159, upload-time = "2026-01-14T23:16:00.817Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a7/d739ffaef33c378fc888302a018d7f81080393d96c476b058b8c64fd2b0d/regex-2026.1.15-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:124dc36c85d34ef2d9164da41a53c1c8c122cfb1f6e1ec377a1f27ee81deb794", size = 795558, upload-time = "2026-01-14T23:16:03.267Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c4/542876f9a0ac576100fc73e9c75b779f5c31e3527576cfc9cb3009dcc58a/regex-2026.1.15-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:a1774cd1981cd212506a23a14dba7fdeaee259f5deba2df6229966d9911e767a", size = 868427, upload-time = "2026-01-14T23:16:05.646Z" }, + { url = "https://files.pythonhosted.org/packages/fc/0f/d5655bea5b22069e32ae85a947aa564912f23758e112cdb74212848a1a1b/regex-2026.1.15-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b5f7d8d2867152cdb625e72a530d2ccb48a3d199159144cbdd63870882fb6f80", size = 769939, upload-time = "2026-01-14T23:16:07.542Z" }, + { url = "https://files.pythonhosted.org/packages/20/06/7e18a4fa9d326daeda46d471a44ef94201c46eaa26dbbb780b5d92cbfdda/regex-2026.1.15-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:492534a0ab925d1db998defc3c302dae3616a2fc3fe2e08db1472348f096ddf2", size = 854753, upload-time = "2026-01-14T23:16:10.395Z" }, + { url = "https://files.pythonhosted.org/packages/3b/67/dc8946ef3965e166f558ef3b47f492bc364e96a265eb4a2bb3ca765c8e46/regex-2026.1.15-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c661fc820cfb33e166bf2450d3dadbda47c8d8981898adb9b6fe24e5e582ba60", size = 799559, upload-time = "2026-01-14T23:16:12.347Z" }, + { url = "https://files.pythonhosted.org/packages/a5/61/1bba81ff6d50c86c65d9fd84ce9699dd106438ee4cdb105bf60374ee8412/regex-2026.1.15-cp313-cp313t-win32.whl", hash = "sha256:99ad739c3686085e614bf77a508e26954ff1b8f14da0e3765ff7abbf7799f952", size = 268879, upload-time = "2026-01-14T23:16:14.049Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5e/cef7d4c5fb0ea3ac5c775fd37db5747f7378b29526cc83f572198924ff47/regex-2026.1.15-cp313-cp313t-win_amd64.whl", hash = "sha256:32655d17905e7ff8ba5c764c43cb124e34a9245e45b83c22e81041e1071aee10", size = 280317, upload-time = "2026-01-14T23:16:15.718Z" }, + { url = "https://files.pythonhosted.org/packages/b4/52/4317f7a5988544e34ab57b4bde0f04944c4786128c933fb09825924d3e82/regex-2026.1.15-cp313-cp313t-win_arm64.whl", hash = "sha256:b2a13dd6a95e95a489ca242319d18fc02e07ceb28fa9ad146385194d95b3c829", size = 271551, upload-time = "2026-01-14T23:16:17.533Z" }, + { url = "https://files.pythonhosted.org/packages/52/0a/47fa888ec7cbbc7d62c5f2a6a888878e76169170ead271a35239edd8f0e8/regex-2026.1.15-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:d920392a6b1f353f4aa54328c867fec3320fa50657e25f64abf17af054fc97ac", size = 489170, upload-time = "2026-01-14T23:16:19.835Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c4/d000e9b7296c15737c9301708e9e7fbdea009f8e93541b6b43bdb8219646/regex-2026.1.15-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:b5a28980a926fa810dbbed059547b02783952e2efd9c636412345232ddb87ff6", size = 291146, upload-time = "2026-01-14T23:16:21.541Z" }, + { url = "https://files.pythonhosted.org/packages/f9/b6/921cc61982e538682bdf3bdf5b2c6ab6b34368da1f8e98a6c1ddc503c9cf/regex-2026.1.15-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:621f73a07595d83f28952d7bd1e91e9d1ed7625fb7af0064d3516674ec93a2a2", size = 288986, upload-time = "2026-01-14T23:16:23.381Z" }, + { url = "https://files.pythonhosted.org/packages/ca/33/eb7383dde0bbc93f4fb9d03453aab97e18ad4024ac7e26cef8d1f0a2cff0/regex-2026.1.15-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3d7d92495f47567a9b1669c51fc8d6d809821849063d168121ef801bbc213846", size = 799098, upload-time = "2026-01-14T23:16:25.088Z" }, + { url = "https://files.pythonhosted.org/packages/27/56/b664dccae898fc8d8b4c23accd853f723bde0f026c747b6f6262b688029c/regex-2026.1.15-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8dd16fba2758db7a3780a051f245539c4451ca20910f5a5e6ea1c08d06d4a76b", size = 864980, upload-time = "2026-01-14T23:16:27.297Z" }, + { url = "https://files.pythonhosted.org/packages/16/40/0999e064a170eddd237bae9ccfcd8f28b3aa98a38bf727a086425542a4fc/regex-2026.1.15-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1e1808471fbe44c1a63e5f577a1d5f02fe5d66031dcbdf12f093ffc1305a858e", size = 911607, upload-time = "2026-01-14T23:16:29.235Z" }, + { url = "https://files.pythonhosted.org/packages/07/78/c77f644b68ab054e5a674fb4da40ff7bffb2c88df58afa82dbf86573092d/regex-2026.1.15-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0751a26ad39d4f2ade8fe16c59b2bf5cb19eb3d2cd543e709e583d559bd9efde", size = 803358, upload-time = "2026-01-14T23:16:31.369Z" }, + { url = "https://files.pythonhosted.org/packages/27/31/d4292ea8566eaa551fafc07797961c5963cf5235c797cc2ae19b85dfd04d/regex-2026.1.15-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0f0c7684c7f9ca241344ff95a1de964f257a5251968484270e91c25a755532c5", size = 775833, upload-time = "2026-01-14T23:16:33.141Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b2/cff3bf2fea4133aa6fb0d1e370b37544d18c8350a2fa118c7e11d1db0e14/regex-2026.1.15-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:74f45d170a21df41508cb67165456538425185baaf686281fa210d7e729abc34", size = 788045, upload-time = "2026-01-14T23:16:35.005Z" }, + { url = "https://files.pythonhosted.org/packages/8d/99/2cb9b69045372ec877b6f5124bda4eb4253bc58b8fe5848c973f752bc52c/regex-2026.1.15-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f1862739a1ffb50615c0fde6bae6569b5efbe08d98e59ce009f68a336f64da75", size = 859374, upload-time = "2026-01-14T23:16:36.919Z" }, + { url = "https://files.pythonhosted.org/packages/09/16/710b0a5abe8e077b1729a562d2f297224ad079f3a66dce46844c193416c8/regex-2026.1.15-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:453078802f1b9e2b7303fb79222c054cb18e76f7bdc220f7530fdc85d319f99e", size = 763940, upload-time = "2026-01-14T23:16:38.685Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d1/7585c8e744e40eb3d32f119191969b91de04c073fca98ec14299041f6e7e/regex-2026.1.15-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:a30a68e89e5a218b8b23a52292924c1f4b245cb0c68d1cce9aec9bbda6e2c160", size = 850112, upload-time = "2026-01-14T23:16:40.646Z" }, + { url = "https://files.pythonhosted.org/packages/af/d6/43e1dd85df86c49a347aa57c1f69d12c652c7b60e37ec162e3096194a278/regex-2026.1.15-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9479cae874c81bf610d72b85bb681a94c95722c127b55445285fb0e2c82db8e1", size = 789586, upload-time = "2026-01-14T23:16:42.799Z" }, + { url = "https://files.pythonhosted.org/packages/93/38/77142422f631e013f316aaae83234c629555729a9fbc952b8a63ac91462a/regex-2026.1.15-cp314-cp314-win32.whl", hash = "sha256:d639a750223132afbfb8f429c60d9d318aeba03281a5f1ab49f877456448dcf1", size = 271691, upload-time = "2026-01-14T23:16:44.671Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a9/ab16b4649524ca9e05213c1cdbb7faa85cc2aa90a0230d2f796cbaf22736/regex-2026.1.15-cp314-cp314-win_amd64.whl", hash = "sha256:4161d87f85fa831e31469bfd82c186923070fc970b9de75339b68f0c75b51903", size = 280422, upload-time = "2026-01-14T23:16:46.607Z" }, + { url = "https://files.pythonhosted.org/packages/be/2a/20fd057bf3521cb4791f69f869635f73e0aaf2b9ad2d260f728144f9047c/regex-2026.1.15-cp314-cp314-win_arm64.whl", hash = "sha256:91c5036ebb62663a6b3999bdd2e559fd8456d17e2b485bf509784cd31a8b1705", size = 273467, upload-time = "2026-01-14T23:16:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/ad/77/0b1e81857060b92b9cad239104c46507dd481b3ff1fa79f8e7f865aae38a/regex-2026.1.15-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ee6854c9000a10938c79238de2379bea30c82e4925a371711af45387df35cab8", size = 492073, upload-time = "2026-01-14T23:16:51.154Z" }, + { url = "https://files.pythonhosted.org/packages/70/f3/f8302b0c208b22c1e4f423147e1913fd475ddd6230565b299925353de644/regex-2026.1.15-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c2b80399a422348ce5de4fe40c418d6299a0fa2803dd61dc0b1a2f28e280fcf", size = 292757, upload-time = "2026-01-14T23:16:53.08Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f0/ef55de2460f3b4a6da9d9e7daacd0cb79d4ef75c64a2af316e68447f0df0/regex-2026.1.15-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:dca3582bca82596609959ac39e12b7dad98385b4fefccb1151b937383cec547d", size = 291122, upload-time = "2026-01-14T23:16:55.383Z" }, + { url = "https://files.pythonhosted.org/packages/cf/55/bb8ccbacabbc3a11d863ee62a9f18b160a83084ea95cdfc5d207bfc3dd75/regex-2026.1.15-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef71d476caa6692eea743ae5ea23cde3260677f70122c4d258ca952e5c2d4e84", size = 807761, upload-time = "2026-01-14T23:16:57.251Z" }, + { url = "https://files.pythonhosted.org/packages/8f/84/f75d937f17f81e55679a0509e86176e29caa7298c38bd1db7ce9c0bf6075/regex-2026.1.15-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c243da3436354f4af6c3058a3f81a97d47ea52c9bd874b52fd30274853a1d5df", size = 873538, upload-time = "2026-01-14T23:16:59.349Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d9/0da86327df70349aa8d86390da91171bd3ca4f0e7c1d1d453a9c10344da3/regex-2026.1.15-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8355ad842a7c7e9e5e55653eade3b7d1885ba86f124dd8ab1f722f9be6627434", size = 915066, upload-time = "2026-01-14T23:17:01.607Z" }, + { url = "https://files.pythonhosted.org/packages/2a/5e/f660fb23fc77baa2a61aa1f1fe3a4eea2bbb8a286ddec148030672e18834/regex-2026.1.15-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f192a831d9575271a22d804ff1a5355355723f94f31d9eef25f0d45a152fdc1a", size = 812938, upload-time = "2026-01-14T23:17:04.366Z" }, + { url = "https://files.pythonhosted.org/packages/69/33/a47a29bfecebbbfd1e5cd3f26b28020a97e4820f1c5148e66e3b7d4b4992/regex-2026.1.15-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:166551807ec20d47ceaeec380081f843e88c8949780cd42c40f18d16168bed10", size = 781314, upload-time = "2026-01-14T23:17:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/65/ec/7ec2bbfd4c3f4e494a24dec4c6943a668e2030426b1b8b949a6462d2c17b/regex-2026.1.15-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f9ca1cbdc0fbfe5e6e6f8221ef2309988db5bcede52443aeaee9a4ad555e0dac", size = 795652, upload-time = "2026-01-14T23:17:08.521Z" }, + { url = "https://files.pythonhosted.org/packages/46/79/a5d8651ae131fe27d7c521ad300aa7f1c7be1dbeee4d446498af5411b8a9/regex-2026.1.15-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b30bcbd1e1221783c721483953d9e4f3ab9c5d165aa709693d3f3946747b1aea", size = 868550, upload-time = "2026-01-14T23:17:10.573Z" }, + { url = "https://files.pythonhosted.org/packages/06/b7/25635d2809664b79f183070786a5552dd4e627e5aedb0065f4e3cf8ee37d/regex-2026.1.15-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2a8d7b50c34578d0d3bf7ad58cde9652b7d683691876f83aedc002862a35dc5e", size = 769981, upload-time = "2026-01-14T23:17:12.871Z" }, + { url = "https://files.pythonhosted.org/packages/16/8b/fc3fcbb2393dcfa4a6c5ffad92dc498e842df4581ea9d14309fcd3c55fb9/regex-2026.1.15-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9d787e3310c6a6425eb346be4ff2ccf6eece63017916fd77fe8328c57be83521", size = 854780, upload-time = "2026-01-14T23:17:14.837Z" }, + { url = "https://files.pythonhosted.org/packages/d0/38/dde117c76c624713c8a2842530be9c93ca8b606c0f6102d86e8cd1ce8bea/regex-2026.1.15-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:619843841e220adca114118533a574a9cd183ed8a28b85627d2844c500a2b0db", size = 799778, upload-time = "2026-01-14T23:17:17.369Z" }, + { url = "https://files.pythonhosted.org/packages/e3/0d/3a6cfa9ae99606afb612d8fb7a66b245a9d5ff0f29bb347c8a30b6ad561b/regex-2026.1.15-cp314-cp314t-win32.whl", hash = "sha256:e90b8db97f6f2c97eb045b51a6b2c5ed69cedd8392459e0642d4199b94fabd7e", size = 274667, upload-time = "2026-01-14T23:17:19.301Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b2/297293bb0742fd06b8d8e2572db41a855cdf1cae0bf009b1cb74fe07e196/regex-2026.1.15-cp314-cp314t-win_amd64.whl", hash = "sha256:5ef19071f4ac9f0834793af85bd04a920b4407715624e40cb7a0631a11137cdf", size = 284386, upload-time = "2026-01-14T23:17:21.231Z" }, + { url = "https://files.pythonhosted.org/packages/95/e4/a3b9480c78cf8ee86626cb06f8d931d74d775897d44201ccb813097ae697/regex-2026.1.15-cp314-cp314t-win_arm64.whl", hash = "sha256:ca89c5e596fc05b015f27561b3793dc2fa0917ea0d7507eebb448efd35274a70", size = 274837, upload-time = "2026-01-14T23:17:23.146Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, +] + +[[package]] +name = "requirements-parser" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/96/fb6dbfebb524d5601d359a47c78fe7ba1eef90fc4096404aa60c9a906fbb/requirements_parser-0.13.0.tar.gz", hash = "sha256:0843119ca2cb2331de4eb31b10d70462e39ace698fd660a915c247d2301a4418", size = 22630, upload-time = "2025-05-21T13:42:05.464Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/60/50fbb6ffb35f733654466f1a90d162bcbea358adc3b0871339254fbc37b2/requirements_parser-0.13.0-py3-none-any.whl", hash = "sha256:2b3173faecf19ec5501971b7222d38f04cb45bb9d87d0ad629ca71e2e62ded14", size = 14782, upload-time = "2025-05-21T13:42:04.007Z" }, +] + +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, + { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, + { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, + { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, + { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, + { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, + { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, + { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, + { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, + { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, + { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, + { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, + { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, + { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, + { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, + { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, + { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, + { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, + { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, + { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, + { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, + { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, + { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, + { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, + { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, + { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, + { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, +] + +[[package]] +name = "ruff" +version = "0.14.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/55/cccfca45157a2031dcbb5a462a67f7cf27f8b37d4b3b1cd7438f0f5c1df6/ruff-0.14.4.tar.gz", hash = "sha256:f459a49fe1085a749f15414ca76f61595f1a2cc8778ed7c279b6ca2e1fd19df3", size = 5587844, upload-time = "2025-11-06T22:07:45.033Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/b9/67240254166ae1eaa38dec32265e9153ac53645a6c6670ed36ad00722af8/ruff-0.14.4-py3-none-linux_armv6l.whl", hash = "sha256:e6604613ffbcf2297cd5dcba0e0ac9bd0c11dc026442dfbb614504e87c349518", size = 12606781, upload-time = "2025-11-06T22:07:01.841Z" }, + { url = "https://files.pythonhosted.org/packages/46/c8/09b3ab245d8652eafe5256ab59718641429f68681ee713ff06c5c549f156/ruff-0.14.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d99c0b52b6f0598acede45ee78288e5e9b4409d1ce7f661f0fa36d4cbeadf9a4", size = 12946765, upload-time = "2025-11-06T22:07:05.858Z" }, + { url = "https://files.pythonhosted.org/packages/14/bb/1564b000219144bf5eed2359edc94c3590dd49d510751dad26202c18a17d/ruff-0.14.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9358d490ec030f1b51d048a7fd6ead418ed0826daf6149e95e30aa67c168af33", size = 11928120, upload-time = "2025-11-06T22:07:08.023Z" }, + { url = "https://files.pythonhosted.org/packages/a3/92/d5f1770e9988cc0742fefaa351e840d9aef04ec24ae1be36f333f96d5704/ruff-0.14.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81b40d27924f1f02dfa827b9c0712a13c0e4b108421665322218fc38caf615c2", size = 12370877, upload-time = "2025-11-06T22:07:10.015Z" }, + { url = "https://files.pythonhosted.org/packages/e2/29/e9282efa55f1973d109faf839a63235575519c8ad278cc87a182a366810e/ruff-0.14.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f5e649052a294fe00818650712083cddc6cc02744afaf37202c65df9ea52efa5", size = 12408538, upload-time = "2025-11-06T22:07:13.085Z" }, + { url = "https://files.pythonhosted.org/packages/8e/01/930ed6ecfce130144b32d77d8d69f5c610e6d23e6857927150adf5d7379a/ruff-0.14.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa082a8f878deeba955531f975881828fd6afd90dfa757c2b0808aadb437136e", size = 13141942, upload-time = "2025-11-06T22:07:15.386Z" }, + { url = "https://files.pythonhosted.org/packages/6a/46/a9c89b42b231a9f487233f17a89cbef9d5acd538d9488687a02ad288fa6b/ruff-0.14.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:1043c6811c2419e39011890f14d0a30470f19d47d197c4858b2787dfa698f6c8", size = 14544306, upload-time = "2025-11-06T22:07:17.631Z" }, + { url = "https://files.pythonhosted.org/packages/78/96/9c6cf86491f2a6d52758b830b89b78c2ae61e8ca66b86bf5a20af73d20e6/ruff-0.14.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a9f3a936ac27fb7c2a93e4f4b943a662775879ac579a433291a6f69428722649", size = 14210427, upload-time = "2025-11-06T22:07:19.832Z" }, + { url = "https://files.pythonhosted.org/packages/71/f4/0666fe7769a54f63e66404e8ff698de1dcde733e12e2fd1c9c6efb689cb5/ruff-0.14.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:95643ffd209ce78bc113266b88fba3d39e0461f0cbc8b55fb92505030fb4a850", size = 13658488, upload-time = "2025-11-06T22:07:22.32Z" }, + { url = "https://files.pythonhosted.org/packages/ee/79/6ad4dda2cfd55e41ac9ed6d73ef9ab9475b1eef69f3a85957210c74ba12c/ruff-0.14.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:456daa2fa1021bc86ca857f43fe29d5d8b3f0e55e9f90c58c317c1dcc2afc7b5", size = 13354908, upload-time = "2025-11-06T22:07:24.347Z" }, + { url = "https://files.pythonhosted.org/packages/b5/60/f0b6990f740bb15c1588601d19d21bcc1bd5de4330a07222041678a8e04f/ruff-0.14.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:f911bba769e4a9f51af6e70037bb72b70b45a16db5ce73e1f72aefe6f6d62132", size = 13587803, upload-time = "2025-11-06T22:07:26.327Z" }, + { url = "https://files.pythonhosted.org/packages/c9/da/eaaada586f80068728338e0ef7f29ab3e4a08a692f92eb901a4f06bbff24/ruff-0.14.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:76158a7369b3979fa878612c623a7e5430c18b2fd1c73b214945c2d06337db67", size = 12279654, upload-time = "2025-11-06T22:07:28.46Z" }, + { url = "https://files.pythonhosted.org/packages/66/d4/b1d0e82cf9bf8aed10a6d45be47b3f402730aa2c438164424783ac88c0ed/ruff-0.14.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f3b8f3b442d2b14c246e7aeca2e75915159e06a3540e2f4bed9f50d062d24469", size = 12357520, upload-time = "2025-11-06T22:07:31.468Z" }, + { url = "https://files.pythonhosted.org/packages/04/f4/53e2b42cc82804617e5c7950b7079d79996c27e99c4652131c6a1100657f/ruff-0.14.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c62da9a06779deecf4d17ed04939ae8b31b517643b26370c3be1d26f3ef7dbde", size = 12719431, upload-time = "2025-11-06T22:07:33.831Z" }, + { url = "https://files.pythonhosted.org/packages/a2/94/80e3d74ed9a72d64e94a7b7706b1c1ebaa315ef2076fd33581f6a1cd2f95/ruff-0.14.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5a443a83a1506c684e98acb8cb55abaf3ef725078be40237463dae4463366349", size = 13464394, upload-time = "2025-11-06T22:07:35.905Z" }, + { url = "https://files.pythonhosted.org/packages/54/1a/a49f071f04c42345c793d22f6cf5e0920095e286119ee53a64a3a3004825/ruff-0.14.4-py3-none-win32.whl", hash = "sha256:643b69cb63cd996f1fc7229da726d07ac307eae442dd8974dbc7cf22c1e18fff", size = 12493429, upload-time = "2025-11-06T22:07:38.43Z" }, + { url = "https://files.pythonhosted.org/packages/bc/22/e58c43e641145a2b670328fb98bc384e20679b5774258b1e540207580266/ruff-0.14.4-py3-none-win_amd64.whl", hash = "sha256:26673da283b96fe35fa0c939bf8411abec47111644aa9f7cfbd3c573fb125d2c", size = 13635380, upload-time = "2025-11-06T22:07:40.496Z" }, + { url = "https://files.pythonhosted.org/packages/30/bd/4168a751ddbbf43e86544b4de8b5c3b7be8d7167a2a5cb977d274e04f0a1/ruff-0.14.4-py3-none-win_arm64.whl", hash = "sha256:dd09c292479596b0e6fec8cd95c65c3a6dc68e9ad17b8f2382130f87ff6a75bb", size = 12663065, upload-time = "2025-11-06T22:07:42.603Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.8.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/f9/5e4491e5ccf42f5d9cfc663741d261b3e6e1683ae7812114e7636409fcc6/sqlalchemy-2.0.45.tar.gz", hash = "sha256:1632a4bda8d2d25703fdad6363058d882541bdaaee0e5e3ddfa0cd3229efce88", size = 9869912, upload-time = "2025-12-09T21:05:16.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/c8/7cc5221b47a54edc72a0140a1efa56e0a2730eefa4058d7ed0b4c4357ff8/sqlalchemy-2.0.45-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe187fc31a54d7fd90352f34e8c008cf3ad5d064d08fedd3de2e8df83eb4a1cf", size = 3277082, upload-time = "2025-12-09T22:11:06.167Z" }, + { url = "https://files.pythonhosted.org/packages/0e/50/80a8d080ac7d3d321e5e5d420c9a522b0aa770ec7013ea91f9a8b7d36e4a/sqlalchemy-2.0.45-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:672c45cae53ba88e0dad74b9027dddd09ef6f441e927786b05bec75d949fbb2e", size = 3293131, upload-time = "2025-12-09T22:13:52.626Z" }, + { url = "https://files.pythonhosted.org/packages/da/4c/13dab31266fc9904f7609a5dc308a2432a066141d65b857760c3bef97e69/sqlalchemy-2.0.45-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:470daea2c1ce73910f08caf10575676a37159a6d16c4da33d0033546bddebc9b", size = 3225389, upload-time = "2025-12-09T22:11:08.093Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/891b5c2e9f83589de202e7abaf24cd4e4fa59e1837d64d528829ad6cc107/sqlalchemy-2.0.45-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9c6378449e0940476577047150fd09e242529b761dc887c9808a9a937fe990c8", size = 3266054, upload-time = "2025-12-09T22:13:54.262Z" }, + { url = "https://files.pythonhosted.org/packages/f1/24/fc59e7f71b0948cdd4cff7a286210e86b0443ef1d18a23b0d83b87e4b1f7/sqlalchemy-2.0.45-cp313-cp313-win32.whl", hash = "sha256:4b6bec67ca45bc166c8729910bd2a87f1c0407ee955df110d78948f5b5827e8a", size = 2110299, upload-time = "2025-12-09T21:39:33.486Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c5/d17113020b2d43073412aeca09b60d2009442420372123b8d49cc253f8b8/sqlalchemy-2.0.45-cp313-cp313-win_amd64.whl", hash = "sha256:afbf47dc4de31fa38fd491f3705cac5307d21d4bb828a4f020ee59af412744ee", size = 2136264, upload-time = "2025-12-09T21:39:36.801Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8d/bb40a5d10e7a5f2195f235c0b2f2c79b0bf6e8f00c0c223130a4fbd2db09/sqlalchemy-2.0.45-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83d7009f40ce619d483d26ac1b757dfe3167b39921379a8bd1b596cf02dab4a6", size = 3521998, upload-time = "2025-12-09T22:13:28.622Z" }, + { url = "https://files.pythonhosted.org/packages/75/a5/346128b0464886f036c039ea287b7332a410aa2d3fb0bb5d404cb8861635/sqlalchemy-2.0.45-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d8a2ca754e5415cde2b656c27900b19d50ba076aa05ce66e2207623d3fe41f5a", size = 3473434, upload-time = "2025-12-09T22:13:30.188Z" }, + { url = "https://files.pythonhosted.org/packages/cc/64/4e1913772646b060b025d3fc52ce91a58967fe58957df32b455de5a12b4f/sqlalchemy-2.0.45-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f46ec744e7f51275582e6a24326e10c49fbdd3fc99103e01376841213028774", size = 3272404, upload-time = "2025-12-09T22:11:09.662Z" }, + { url = "https://files.pythonhosted.org/packages/b3/27/caf606ee924282fe4747ee4fd454b335a72a6e018f97eab5ff7f28199e16/sqlalchemy-2.0.45-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:883c600c345123c033c2f6caca18def08f1f7f4c3ebeb591a63b6fceffc95cce", size = 3277057, upload-time = "2025-12-09T22:13:56.213Z" }, + { url = "https://files.pythonhosted.org/packages/85/d0/3d64218c9724e91f3d1574d12eb7ff8f19f937643815d8daf792046d88ab/sqlalchemy-2.0.45-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2c0b74aa79e2deade948fe8593654c8ef4228c44ba862bb7c9585c8e0db90f33", size = 3222279, upload-time = "2025-12-09T22:11:11.1Z" }, + { url = "https://files.pythonhosted.org/packages/24/10/dd7688a81c5bc7690c2a3764d55a238c524cd1a5a19487928844cb247695/sqlalchemy-2.0.45-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a420169cef179d4c9064365f42d779f1e5895ad26ca0c8b4c0233920973db74", size = 3244508, upload-time = "2025-12-09T22:13:57.932Z" }, + { url = "https://files.pythonhosted.org/packages/aa/41/db75756ca49f777e029968d9c9fee338c7907c563267740c6d310a8e3f60/sqlalchemy-2.0.45-cp314-cp314-win32.whl", hash = "sha256:e50dcb81a5dfe4b7b4a4aa8f338116d127cb209559124f3694c70d6cd072b68f", size = 2113204, upload-time = "2025-12-09T21:39:38.365Z" }, + { url = "https://files.pythonhosted.org/packages/89/a2/0e1590e9adb292b1d576dbcf67ff7df8cf55e56e78d2c927686d01080f4b/sqlalchemy-2.0.45-cp314-cp314-win_amd64.whl", hash = "sha256:4748601c8ea959e37e03d13dcda4a44837afcd1b21338e637f7c935b8da06177", size = 2138785, upload-time = "2025-12-09T21:39:39.503Z" }, + { url = "https://files.pythonhosted.org/packages/42/39/f05f0ed54d451156bbed0e23eb0516bcad7cbb9f18b3bf219c786371b3f0/sqlalchemy-2.0.45-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd337d3526ec5298f67d6a30bbbe4ed7e5e68862f0bf6dd21d289f8d37b7d60b", size = 3522029, upload-time = "2025-12-09T22:13:32.09Z" }, + { url = "https://files.pythonhosted.org/packages/54/0f/d15398b98b65c2bce288d5ee3f7d0a81f77ab89d9456994d5c7cc8b2a9db/sqlalchemy-2.0.45-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9a62b446b7d86a3909abbcd1cd3cc550a832f99c2bc37c5b22e1925438b9367b", size = 3475142, upload-time = "2025-12-09T22:13:33.739Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e1/3ccb13c643399d22289c6a9786c1a91e3dcbb68bce4beb44926ac2c557bf/sqlalchemy-2.0.45-py3-none-any.whl", hash = "sha256:5225a288e4c8cc2308dbdd874edad6e7d0fd38eac1e9e5f23503425c8eee20d0", size = 1936672, upload-time = "2025-12-09T21:54:52.608Z" }, +] + +[package.optional-dependencies] +postgresql-psycopgbinary = [ + { name = "psycopg", extra = ["binary"] }, +] + +[[package]] +name = "sqlmodel" +version = "0.0.27" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "sqlalchemy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/5a/693d90866233e837d182da76082a6d4c2303f54d3aaaa5c78e1238c5d863/sqlmodel-0.0.27.tar.gz", hash = "sha256:ad1227f2014a03905aef32e21428640848ac09ff793047744a73dfdd077ff620", size = 118053, upload-time = "2025-10-08T16:39:11.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/92/c35e036151fe53822893979f8a13e6f235ae8191f4164a79ae60a95d66aa/sqlmodel-0.0.27-py3-none-any.whl", hash = "sha256:667fe10aa8ff5438134668228dc7d7a08306f4c5c4c7e6ad3ad68defa0e7aa49", size = 29131, upload-time = "2025-10-08T16:39:10.917Z" }, +] + +[[package]] +name = "sse-starlette" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/8d/00d280c03ffd39aaee0e86ec81e2d3b9253036a0f93f51d10503adef0e65/sse_starlette-3.2.0.tar.gz", hash = "sha256:8127594edfb51abe44eac9c49e59b0b01f1039d0c7461c6fd91d4e03b70da422", size = 27253, upload-time = "2026-01-17T13:11:05.62Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/7f/832f015020844a8b8f7a9cbc103dd76ba8e3875004c41e08440ea3a2b41a/sse_starlette-3.2.0-py3-none-any.whl", hash = "sha256:5876954bd51920fc2cd51baee47a080eb88a37b5b784e615abb0b283f801cdbf", size = 12763, upload-time = "2026-01-17T13:11:03.775Z" }, +] + +[[package]] +name = "starlette" +version = "0.50.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/b8/73a0e6a6e079a9d9cfa64113d771e421640b6f679a52eeb9b32f72d871a1/starlette-0.50.0.tar.gz", hash = "sha256:a2a17b22203254bcbc2e1f926d2d55f3f9497f769416b3190768befe598fa3ca", size = 2646985, upload-time = "2025-11-01T15:25:27.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/52/1064f510b141bd54025f9b55105e26d1fa970b9be67ad766380a3c9b74b0/starlette-0.50.0-py3-none-any.whl", hash = "sha256:9e5391843ec9b6e472eed1365a78c8098cfceb7a74bfd4d6b1c0c0095efb3bca", size = 74033, upload-time = "2025-11-01T15:25:25.461Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, +] + +[[package]] +name = "tiktoken" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931", size = 37806, upload-time = "2025-10-06T20:22:45.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/61/441588ee21e6b5cdf59d6870f86beb9789e532ee9718c251b391b70c68d6/tiktoken-0.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:775c2c55de2310cc1bc9a3ad8826761cbdc87770e586fd7b6da7d4589e13dab3", size = 1050802, upload-time = "2025-10-06T20:22:00.96Z" }, + { url = "https://files.pythonhosted.org/packages/1f/05/dcf94486d5c5c8d34496abe271ac76c5b785507c8eae71b3708f1ad9b45a/tiktoken-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a01b12f69052fbe4b080a2cfb867c4de12c704b56178edf1d1d7b273561db160", size = 993995, upload-time = "2025-10-06T20:22:02.788Z" }, + { url = "https://files.pythonhosted.org/packages/a0/70/5163fe5359b943f8db9946b62f19be2305de8c3d78a16f629d4165e2f40e/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:01d99484dc93b129cd0964f9d34eee953f2737301f18b3c7257bf368d7615baa", size = 1128948, upload-time = "2025-10-06T20:22:03.814Z" }, + { url = "https://files.pythonhosted.org/packages/0c/da/c028aa0babf77315e1cef357d4d768800c5f8a6de04d0eac0f377cb619fa/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4a1a4fcd021f022bfc81904a911d3df0f6543b9e7627b51411da75ff2fe7a1be", size = 1151986, upload-time = "2025-10-06T20:22:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/a0/5a/886b108b766aa53e295f7216b509be95eb7d60b166049ce2c58416b25f2a/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:981a81e39812d57031efdc9ec59fa32b2a5a5524d20d4776574c4b4bd2e9014a", size = 1194222, upload-time = "2025-10-06T20:22:06.265Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f8/4db272048397636ac7a078d22773dd2795b1becee7bc4922fe6207288d57/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9baf52f84a3f42eef3ff4e754a0db79a13a27921b457ca9832cf944c6be4f8f3", size = 1255097, upload-time = "2025-10-06T20:22:07.403Z" }, + { url = "https://files.pythonhosted.org/packages/8e/32/45d02e2e0ea2be3a9ed22afc47d93741247e75018aac967b713b2941f8ea/tiktoken-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b8a0cd0c789a61f31bf44851defbd609e8dd1e2c8589c614cc1060940ef1f697", size = 879117, upload-time = "2025-10-06T20:22:08.418Z" }, + { url = "https://files.pythonhosted.org/packages/ce/76/994fc868f88e016e6d05b0da5ac24582a14c47893f4474c3e9744283f1d5/tiktoken-0.12.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d5f89ea5680066b68bcb797ae85219c72916c922ef0fcdd3480c7d2315ffff16", size = 1050309, upload-time = "2025-10-06T20:22:10.939Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b8/57ef1456504c43a849821920d582a738a461b76a047f352f18c0b26c6516/tiktoken-0.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b4e7ed1c6a7a8a60a3230965bdedba8cc58f68926b835e519341413370e0399a", size = 993712, upload-time = "2025-10-06T20:22:12.115Z" }, + { url = "https://files.pythonhosted.org/packages/72/90/13da56f664286ffbae9dbcfadcc625439142675845baa62715e49b87b68b/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:fc530a28591a2d74bce821d10b418b26a094bf33839e69042a6e86ddb7a7fb27", size = 1128725, upload-time = "2025-10-06T20:22:13.541Z" }, + { url = "https://files.pythonhosted.org/packages/05/df/4f80030d44682235bdaecd7346c90f67ae87ec8f3df4a3442cb53834f7e4/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:06a9f4f49884139013b138920a4c393aa6556b2f8f536345f11819389c703ebb", size = 1151875, upload-time = "2025-10-06T20:22:14.559Z" }, + { url = "https://files.pythonhosted.org/packages/22/1f/ae535223a8c4ef4c0c1192e3f9b82da660be9eb66b9279e95c99288e9dab/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:04f0e6a985d95913cabc96a741c5ffec525a2c72e9df086ff17ebe35985c800e", size = 1194451, upload-time = "2025-10-06T20:22:15.545Z" }, + { url = "https://files.pythonhosted.org/packages/78/a7/f8ead382fce0243cb625c4f266e66c27f65ae65ee9e77f59ea1653b6d730/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ee8f9ae00c41770b5f9b0bb1235474768884ae157de3beb5439ca0fd70f3e25", size = 1253794, upload-time = "2025-10-06T20:22:16.624Z" }, + { url = "https://files.pythonhosted.org/packages/93/e0/6cc82a562bc6365785a3ff0af27a2a092d57c47d7a81d9e2295d8c36f011/tiktoken-0.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:dc2dd125a62cb2b3d858484d6c614d136b5b848976794edfb63688d539b8b93f", size = 878777, upload-time = "2025-10-06T20:22:18.036Z" }, + { url = "https://files.pythonhosted.org/packages/72/05/3abc1db5d2c9aadc4d2c76fa5640134e475e58d9fbb82b5c535dc0de9b01/tiktoken-0.12.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a90388128df3b3abeb2bfd1895b0681412a8d7dc644142519e6f0a97c2111646", size = 1050188, upload-time = "2025-10-06T20:22:19.563Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7b/50c2f060412202d6c95f32b20755c7a6273543b125c0985d6fa9465105af/tiktoken-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:da900aa0ad52247d8794e307d6446bd3cdea8e192769b56276695d34d2c9aa88", size = 993978, upload-time = "2025-10-06T20:22:20.702Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/bf795595a2b897e271771cd31cb847d479073497344c637966bdf2853da1/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:285ba9d73ea0d6171e7f9407039a290ca77efcdb026be7769dccc01d2c8d7fff", size = 1129271, upload-time = "2025-10-06T20:22:22.06Z" }, + { url = "https://files.pythonhosted.org/packages/f5/de/9341a6d7a8f1b448573bbf3425fa57669ac58258a667eb48a25dfe916d70/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:d186a5c60c6a0213f04a7a802264083dea1bbde92a2d4c7069e1a56630aef830", size = 1151216, upload-time = "2025-10-06T20:22:23.085Z" }, + { url = "https://files.pythonhosted.org/packages/75/0d/881866647b8d1be4d67cb24e50d0c26f9f807f994aa1510cb9ba2fe5f612/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:604831189bd05480f2b885ecd2d1986dc7686f609de48208ebbbddeea071fc0b", size = 1194860, upload-time = "2025-10-06T20:22:24.602Z" }, + { url = "https://files.pythonhosted.org/packages/b3/1e/b651ec3059474dab649b8d5b69f5c65cd8fcd8918568c1935bd4136c9392/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8f317e8530bb3a222547b85a58583238c8f74fd7a7408305f9f63246d1a0958b", size = 1254567, upload-time = "2025-10-06T20:22:25.671Z" }, + { url = "https://files.pythonhosted.org/packages/80/57/ce64fd16ac390fafde001268c364d559447ba09b509181b2808622420eec/tiktoken-0.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:399c3dd672a6406719d84442299a490420b458c44d3ae65516302a99675888f3", size = 921067, upload-time = "2025-10-06T20:22:26.753Z" }, + { url = "https://files.pythonhosted.org/packages/ac/a4/72eed53e8976a099539cdd5eb36f241987212c29629d0a52c305173e0a68/tiktoken-0.12.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c2c714c72bc00a38ca969dae79e8266ddec999c7ceccd603cc4f0d04ccd76365", size = 1050473, upload-time = "2025-10-06T20:22:27.775Z" }, + { url = "https://files.pythonhosted.org/packages/e6/d7/0110b8f54c008466b19672c615f2168896b83706a6611ba6e47313dbc6e9/tiktoken-0.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cbb9a3ba275165a2cb0f9a83f5d7025afe6b9d0ab01a22b50f0e74fee2ad253e", size = 993855, upload-time = "2025-10-06T20:22:28.799Z" }, + { url = "https://files.pythonhosted.org/packages/5f/77/4f268c41a3957c418b084dd576ea2fad2e95da0d8e1ab705372892c2ca22/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:dfdfaa5ffff8993a3af94d1125870b1d27aed7cb97aa7eb8c1cefdbc87dbee63", size = 1129022, upload-time = "2025-10-06T20:22:29.981Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2b/fc46c90fe5028bd094cd6ee25a7db321cb91d45dc87531e2bdbb26b4867a/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:584c3ad3d0c74f5269906eb8a659c8bfc6144a52895d9261cdaf90a0ae5f4de0", size = 1150736, upload-time = "2025-10-06T20:22:30.996Z" }, + { url = "https://files.pythonhosted.org/packages/28/c0/3c7a39ff68022ddfd7d93f3337ad90389a342f761c4d71de99a3ccc57857/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:54c891b416a0e36b8e2045b12b33dd66fb34a4fe7965565f1b482da50da3e86a", size = 1194908, upload-time = "2025-10-06T20:22:32.073Z" }, + { url = "https://files.pythonhosted.org/packages/ab/0d/c1ad6f4016a3968c048545f5d9b8ffebf577774b2ede3e2e352553b685fe/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5edb8743b88d5be814b1a8a8854494719080c28faaa1ccbef02e87354fe71ef0", size = 1253706, upload-time = "2025-10-06T20:22:33.385Z" }, + { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, +] + +[[package]] +name = "types-defusedxml" +version = "0.7.0.20250822" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/4a/5b997ae87bf301d1796f72637baa4e0e10d7db17704a8a71878a9f77f0c0/types_defusedxml-0.7.0.20250822.tar.gz", hash = "sha256:ba6c395105f800c973bba8a25e41b215483e55ec79c8ca82b6fe90ba0bc3f8b2", size = 10590, upload-time = "2025-08-22T03:02:59.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/73/8a36998cee9d7c9702ed64a31f0866c7f192ecffc22771d44dbcc7878f18/types_defusedxml-0.7.0.20250822-py3-none-any.whl", hash = "sha256:5ee219f8a9a79c184773599ad216123aedc62a969533ec36737ec98601f20dcf", size = 13430, upload-time = "2025-08-22T03:02:58.466Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/a7/c202b344c5ca7daf398f3b8a477eeb205cf3b6f32e7ec3a6bac0629ca975/tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7", size = 196772, upload-time = "2025-12-13T17:45:35.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] + +[[package]] +name = "uuid-utils" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/8a/17b11768dcb473d3a255c02ffdd94fbd1b345c906efea0a39124dcbaed52/uuid_utils-0.13.0.tar.gz", hash = "sha256:4c17df6427a9e23a4cd7fb9ee1efb53b8abb078660b9bdb2524ca8595022dfe1", size = 21921, upload-time = "2026-01-08T15:48:10.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/b8/d40848ca22781f206c60a1885fc737d2640392bd6b5792d455525accd89c/uuid_utils-0.13.0-cp39-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:83628283e977fb212e756bc055df8fdd2f9f589a2e539ba1abe755b8ce8df7a4", size = 602130, upload-time = "2026-01-08T15:47:34.877Z" }, + { url = "https://files.pythonhosted.org/packages/40/b9/00a944b8096632ea12638181f8e294abcde3e3b8b5e29b777f809896f6ae/uuid_utils-0.13.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c47638ed6334ab19d80f73664f153b04bbb04ab8ce4298d10da6a292d4d21c47", size = 304213, upload-time = "2026-01-08T15:47:36.807Z" }, + { url = "https://files.pythonhosted.org/packages/da/d7/07b36c33aef683b81c9afff3ec178d5eb39d325447a68c3c68a62e4abb32/uuid_utils-0.13.0-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:b276b538c57733ed406948584912da422a604313c71479654848b84b9e19c9b0", size = 340624, upload-time = "2026-01-08T15:47:38.821Z" }, + { url = "https://files.pythonhosted.org/packages/7d/55/fcff2fff02a27866cb1a6614c9df2b3ace721f0a0aab2b7b8f5a7d4e4221/uuid_utils-0.13.0-cp39-abi3-manylinux_2_24_armv7l.whl", hash = "sha256:bdaf2b77e34b199cf04cde28399495fd1ed951de214a4ece1f3919b2f945bb06", size = 346705, upload-time = "2026-01-08T15:47:40.397Z" }, + { url = "https://files.pythonhosted.org/packages/41/48/67438506c2bb8bee1b4b00d7c0b3ff866401b4790849bf591d654d4ea0bc/uuid_utils-0.13.0-cp39-abi3-manylinux_2_24_i686.whl", hash = "sha256:eb2f0baf81e82f9769a7684022dca8f3bf801ca1574a3e94df1876e9d6f9271e", size = 366023, upload-time = "2026-01-08T15:47:42.662Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d7/2d91ce17f62fd764d593430de296b70843cc25229c772453f7261de9e6a8/uuid_utils-0.13.0-cp39-abi3-manylinux_2_24_ppc64le.whl", hash = "sha256:6be6c4d11275f5cc402a4fdba6c2b1ce45fd3d99bb78716cd1cc2cbf6802b2ce", size = 471149, upload-time = "2026-01-08T15:47:44.963Z" }, + { url = "https://files.pythonhosted.org/packages/2e/9a/aa0756186073ba84daf5704c150d41ede10eb3185d510e02532e2071550e/uuid_utils-0.13.0-cp39-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:77621cf6ceca7f42173a642a01c01c216f9eaec3b7b65d093d2d6a433ca0a83d", size = 342130, upload-time = "2026-01-08T15:47:46.331Z" }, + { url = "https://files.pythonhosted.org/packages/74/b4/3191789f4dc3bed59d79cec90559821756297a25d7dc34d1bf7781577a75/uuid_utils-0.13.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9a5a9eb06c2bb86dd876cd7b2fe927fc8543d14c90d971581db6ffda4a02526f", size = 524128, upload-time = "2026-01-08T15:47:47.628Z" }, + { url = "https://files.pythonhosted.org/packages/b2/30/29839210a8fff9fc219bfa7c8d8cd115324e92618cba0cda090d54d3d321/uuid_utils-0.13.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:775347c6110fb71360df17aac74132d8d47c1dbe71233ac98197fc872a791fd2", size = 615872, upload-time = "2026-01-08T15:47:50.61Z" }, + { url = "https://files.pythonhosted.org/packages/99/ed/15000c96a8bd8f5fd8efd622109bf52549ea0b366f8ce71c45580fa55878/uuid_utils-0.13.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf95f6370ad1a0910ee7b5ad5228fd19c4ae32fe3627389006adaf519408c41e", size = 581023, upload-time = "2026-01-08T15:47:52.776Z" }, + { url = "https://files.pythonhosted.org/packages/67/c8/3f809fa2dc2ca4bd331c792a3c7d3e45ae2b709d85847a12b8b27d1d5f19/uuid_utils-0.13.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5a88e23e0b2f4203fefe2ccbca5736ee06fcad10e61b5e7e39c8d7904bc13300", size = 546715, upload-time = "2026-01-08T15:47:54.415Z" }, + { url = "https://files.pythonhosted.org/packages/f5/80/4f7c7efd734d1494397c781bd3d421688e9c187ae836e3174625b1ddf8b0/uuid_utils-0.13.0-cp39-abi3-win32.whl", hash = "sha256:3e4f2cc54e6a99c0551158100ead528479ad2596847478cbad624977064ffce3", size = 177650, upload-time = "2026-01-08T15:47:55.679Z" }, + { url = "https://files.pythonhosted.org/packages/6c/94/d05ab68622e66ad787a241dfe5ccc649b3af09f30eae977b9ee8f7046aaa/uuid_utils-0.13.0-cp39-abi3-win_amd64.whl", hash = "sha256:046cb2756e1597b3de22d24851b769913e192135830486a0a70bf41327f0360c", size = 183211, upload-time = "2026-01-08T15:47:57.604Z" }, + { url = "https://files.pythonhosted.org/packages/69/37/674b3ce25cd715b831ea8ebbd828b74c40159f04c95d1bb963b2c876fe79/uuid_utils-0.13.0-cp39-abi3-win_arm64.whl", hash = "sha256:5447a680df6ef8a5a353976aaf4c97cc3a3a22b1ee13671c44227b921e3ae2a9", size = 183518, upload-time = "2026-01-08T15:47:59.148Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/d1/8f3c683c9561a4e6689dd3b1d345c815f10f86acd044ee1fb9a4dcd0b8c5/uvicorn-0.40.0.tar.gz", hash = "sha256:839676675e87e73694518b5574fd0f24c9d97b46bea16df7b8c05ea1a51071ea", size = 81761, upload-time = "2025-12-21T14:16:22.45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/d8/2083a1daa7439a66f3a48589a57d576aa117726762618f6bb09fe3798796/uvicorn-0.40.0-py3-none-any.whl", hash = "sha256:c6c8f55bc8bf13eb6fa9ff87ad62308bbbc33d0b67f84293151efe87e0d5f2ee", size = 68502, upload-time = "2025-12-21T14:16:21.041Z" }, +] + +[[package]] +name = "virtualenv" +version = "20.35.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/28/e6f1a6f655d620846bd9df527390ecc26b3805a0c5989048c210e22c5ca9/virtualenv-20.35.4.tar.gz", hash = "sha256:643d3914d73d3eeb0c552cbb12d7e82adf0e504dbf86a3182f8771a153a1971c", size = 6028799, upload-time = "2025-10-29T06:57:40.511Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/0c/c05523fa3181fdf0c9c52a6ba91a23fbf3246cc095f26f6516f9c60e6771/virtualenv-20.35.4-py3-none-any.whl", hash = "sha256:c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b", size = 6005095, upload-time = "2025-10-29T06:57:37.598Z" }, +] + +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, + { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, + { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] + +[[package]] +name = "win32-setctime" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, +] + +[[package]] +name = "xlrd" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/5a/377161c2d3538d1990d7af382c79f3b2372e880b65de21b01b1a2b78691e/xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9", size = 100167, upload-time = "2025-06-14T08:46:39.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555, upload-time = "2025-06-14T08:46:37.766Z" }, +] + +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + +[[package]] +name = "xxhash" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6", size = 85160, upload-time = "2025-10-02T14:37:08.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/76/35d05267ac82f53ae9b0e554da7c5e281ee61f3cad44c743f0fcd354f211/xxhash-3.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:599e64ba7f67472481ceb6ee80fa3bd828fd61ba59fb11475572cc5ee52b89ec", size = 32738, upload-time = "2025-10-02T14:34:55.839Z" }, + { url = "https://files.pythonhosted.org/packages/31/a8/3fbce1cd96534a95e35d5120637bf29b0d7f5d8fa2f6374e31b4156dd419/xxhash-3.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d8b8aaa30fca4f16f0c84a5c8d7ddee0e25250ec2796c973775373257dde8f1", size = 30821, upload-time = "2025-10-02T14:34:57.219Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ea/d387530ca7ecfa183cb358027f1833297c6ac6098223fd14f9782cd0015c/xxhash-3.6.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d597acf8506d6e7101a4a44a5e428977a51c0fadbbfd3c39650cca9253f6e5a6", size = 194127, upload-time = "2025-10-02T14:34:59.21Z" }, + { url = "https://files.pythonhosted.org/packages/ba/0c/71435dcb99874b09a43b8d7c54071e600a7481e42b3e3ce1eb5226a5711a/xxhash-3.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:858dc935963a33bc33490128edc1c12b0c14d9c7ebaa4e387a7869ecc4f3e263", size = 212975, upload-time = "2025-10-02T14:35:00.816Z" }, + { url = "https://files.pythonhosted.org/packages/84/7a/c2b3d071e4bb4a90b7057228a99b10d51744878f4a8a6dd643c8bd897620/xxhash-3.6.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba284920194615cb8edf73bf52236ce2e1664ccd4a38fdb543506413529cc546", size = 212241, upload-time = "2025-10-02T14:35:02.207Z" }, + { url = "https://files.pythonhosted.org/packages/81/5f/640b6eac0128e215f177df99eadcd0f1b7c42c274ab6a394a05059694c5a/xxhash-3.6.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b54219177f6c6674d5378bd862c6aedf64725f70dd29c472eaae154df1a2e89", size = 445471, upload-time = "2025-10-02T14:35:03.61Z" }, + { url = "https://files.pythonhosted.org/packages/5e/1e/3c3d3ef071b051cc3abbe3721ffb8365033a172613c04af2da89d5548a87/xxhash-3.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c36dd7dbad2f5238950c377fcbf6811b1cdb1c444fab447960030cea60504d", size = 193936, upload-time = "2025-10-02T14:35:05.013Z" }, + { url = "https://files.pythonhosted.org/packages/2c/bd/4a5f68381939219abfe1c22a9e3a5854a4f6f6f3c4983a87d255f21f2e5d/xxhash-3.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f22927652cba98c44639ffdc7aaf35828dccf679b10b31c4ad72a5b530a18eb7", size = 210440, upload-time = "2025-10-02T14:35:06.239Z" }, + { url = "https://files.pythonhosted.org/packages/eb/37/b80fe3d5cfb9faff01a02121a0f4d565eb7237e9e5fc66e73017e74dcd36/xxhash-3.6.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b45fad44d9c5c119e9c6fbf2e1c656a46dc68e280275007bbfd3d572b21426db", size = 197990, upload-time = "2025-10-02T14:35:07.735Z" }, + { url = "https://files.pythonhosted.org/packages/d7/fd/2c0a00c97b9e18f72e1f240ad4e8f8a90fd9d408289ba9c7c495ed7dc05c/xxhash-3.6.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6f2580ffab1a8b68ef2b901cde7e55fa8da5e4be0977c68f78fc80f3c143de42", size = 210689, upload-time = "2025-10-02T14:35:09.438Z" }, + { url = "https://files.pythonhosted.org/packages/93/86/5dd8076a926b9a95db3206aba20d89a7fc14dd5aac16e5c4de4b56033140/xxhash-3.6.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40c391dd3cd041ebc3ffe6f2c862f402e306eb571422e0aa918d8070ba31da11", size = 414068, upload-time = "2025-10-02T14:35:11.162Z" }, + { url = "https://files.pythonhosted.org/packages/af/3c/0bb129170ee8f3650f08e993baee550a09593462a5cddd8e44d0011102b1/xxhash-3.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f205badabde7aafd1a31e8ca2a3e5a763107a71c397c4481d6a804eb5063d8bd", size = 191495, upload-time = "2025-10-02T14:35:12.971Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3a/6797e0114c21d1725e2577508e24006fd7ff1d8c0c502d3b52e45c1771d8/xxhash-3.6.0-cp313-cp313-win32.whl", hash = "sha256:2577b276e060b73b73a53042ea5bd5203d3e6347ce0d09f98500f418a9fcf799", size = 30620, upload-time = "2025-10-02T14:35:14.129Z" }, + { url = "https://files.pythonhosted.org/packages/86/15/9bc32671e9a38b413a76d24722a2bf8784a132c043063a8f5152d390b0f9/xxhash-3.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:757320d45d2fbcce8f30c42a6b2f47862967aea7bf458b9625b4bbe7ee390392", size = 31542, upload-time = "2025-10-02T14:35:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/39/c5/cc01e4f6188656e56112d6a8e0dfe298a16934b8c47a247236549a3f7695/xxhash-3.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:457b8f85dec5825eed7b69c11ae86834a018b8e3df5e77783c999663da2f96d6", size = 27880, upload-time = "2025-10-02T14:35:16.315Z" }, + { url = "https://files.pythonhosted.org/packages/f3/30/25e5321c8732759e930c555176d37e24ab84365482d257c3b16362235212/xxhash-3.6.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a42e633d75cdad6d625434e3468126c73f13f7584545a9cf34e883aa1710e702", size = 32956, upload-time = "2025-10-02T14:35:17.413Z" }, + { url = "https://files.pythonhosted.org/packages/9f/3c/0573299560d7d9f8ab1838f1efc021a280b5ae5ae2e849034ef3dee18810/xxhash-3.6.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:568a6d743219e717b07b4e03b0a828ce593833e498c3b64752e0f5df6bfe84db", size = 31072, upload-time = "2025-10-02T14:35:18.844Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1c/52d83a06e417cd9d4137722693424885cc9878249beb3a7c829e74bf7ce9/xxhash-3.6.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bec91b562d8012dae276af8025a55811b875baace6af510412a5e58e3121bc54", size = 196409, upload-time = "2025-10-02T14:35:20.31Z" }, + { url = "https://files.pythonhosted.org/packages/e3/8e/c6d158d12a79bbd0b878f8355432075fc82759e356ab5a111463422a239b/xxhash-3.6.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78e7f2f4c521c30ad5e786fdd6bae89d47a32672a80195467b5de0480aa97b1f", size = 215736, upload-time = "2025-10-02T14:35:21.616Z" }, + { url = "https://files.pythonhosted.org/packages/bc/68/c4c80614716345d55071a396cf03d06e34b5f4917a467faf43083c995155/xxhash-3.6.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ed0df1b11a79856df5ffcab572cbd6b9627034c1c748c5566fa79df9048a7c5", size = 214833, upload-time = "2025-10-02T14:35:23.32Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e9/ae27c8ffec8b953efa84c7c4a6c6802c263d587b9fc0d6e7cea64e08c3af/xxhash-3.6.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e4edbfc7d420925b0dd5e792478ed393d6e75ff8fc219a6546fb446b6a417b1", size = 448348, upload-time = "2025-10-02T14:35:25.111Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6b/33e21afb1b5b3f46b74b6bd1913639066af218d704cc0941404ca717fc57/xxhash-3.6.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fba27a198363a7ef87f8c0f6b171ec36b674fe9053742c58dd7e3201c1ab30ee", size = 196070, upload-time = "2025-10-02T14:35:26.586Z" }, + { url = "https://files.pythonhosted.org/packages/96/b6/fcabd337bc5fa624e7203aa0fa7d0c49eed22f72e93229431752bddc83d9/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:794fe9145fe60191c6532fa95063765529770edcdd67b3d537793e8004cabbfd", size = 212907, upload-time = "2025-10-02T14:35:28.087Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d3/9ee6160e644d660fcf176c5825e61411c7f62648728f69c79ba237250143/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:6105ef7e62b5ac73a837778efc331a591d8442f8ef5c7e102376506cb4ae2729", size = 200839, upload-time = "2025-10-02T14:35:29.857Z" }, + { url = "https://files.pythonhosted.org/packages/0d/98/e8de5baa5109394baf5118f5e72ab21a86387c4f89b0e77ef3e2f6b0327b/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f01375c0e55395b814a679b3eea205db7919ac2af213f4a6682e01220e5fe292", size = 213304, upload-time = "2025-10-02T14:35:31.222Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1d/71056535dec5c3177eeb53e38e3d367dd1d16e024e63b1cee208d572a033/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:d706dca2d24d834a4661619dcacf51a75c16d65985718d6a7d73c1eeeb903ddf", size = 416930, upload-time = "2025-10-02T14:35:32.517Z" }, + { url = "https://files.pythonhosted.org/packages/dc/6c/5cbde9de2cd967c322e651c65c543700b19e7ae3e0aae8ece3469bf9683d/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5f059d9faeacd49c0215d66f4056e1326c80503f51a1532ca336a385edadd033", size = 193787, upload-time = "2025-10-02T14:35:33.827Z" }, + { url = "https://files.pythonhosted.org/packages/19/fa/0172e350361d61febcea941b0cc541d6e6c8d65d153e85f850a7b256ff8a/xxhash-3.6.0-cp313-cp313t-win32.whl", hash = "sha256:1244460adc3a9be84731d72b8e80625788e5815b68da3da8b83f78115a40a7ec", size = 30916, upload-time = "2025-10-02T14:35:35.107Z" }, + { url = "https://files.pythonhosted.org/packages/ad/e6/e8cf858a2b19d6d45820f072eff1bea413910592ff17157cabc5f1227a16/xxhash-3.6.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b1e420ef35c503869c4064f4a2f2b08ad6431ab7b229a05cce39d74268bca6b8", size = 31799, upload-time = "2025-10-02T14:35:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/56/15/064b197e855bfb7b343210e82490ae672f8bc7cdf3ddb02e92f64304ee8a/xxhash-3.6.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ec44b73a4220623235f67a996c862049f375df3b1052d9899f40a6382c32d746", size = 28044, upload-time = "2025-10-02T14:35:37.195Z" }, + { url = "https://files.pythonhosted.org/packages/7e/5e/0138bc4484ea9b897864d59fce9be9086030825bc778b76cb5a33a906d37/xxhash-3.6.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a40a3d35b204b7cc7643cbcf8c9976d818cb47befcfac8bbefec8038ac363f3e", size = 32754, upload-time = "2025-10-02T14:35:38.245Z" }, + { url = "https://files.pythonhosted.org/packages/18/d7/5dac2eb2ec75fd771957a13e5dda560efb2176d5203f39502a5fc571f899/xxhash-3.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a54844be970d3fc22630b32d515e79a90d0a3ddb2644d8d7402e3c4c8da61405", size = 30846, upload-time = "2025-10-02T14:35:39.6Z" }, + { url = "https://files.pythonhosted.org/packages/fe/71/8bc5be2bb00deb5682e92e8da955ebe5fa982da13a69da5a40a4c8db12fb/xxhash-3.6.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:016e9190af8f0a4e3741343777710e3d5717427f175adfdc3e72508f59e2a7f3", size = 194343, upload-time = "2025-10-02T14:35:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/e7/3b/52badfb2aecec2c377ddf1ae75f55db3ba2d321c5e164f14461c90837ef3/xxhash-3.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f6f72232f849eb9d0141e2ebe2677ece15adfd0fa599bc058aad83c714bb2c6", size = 213074, upload-time = "2025-10-02T14:35:42.29Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2b/ae46b4e9b92e537fa30d03dbc19cdae57ed407e9c26d163895e968e3de85/xxhash-3.6.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63275a8aba7865e44b1813d2177e0f5ea7eadad3dd063a21f7cf9afdc7054063", size = 212388, upload-time = "2025-10-02T14:35:43.929Z" }, + { url = "https://files.pythonhosted.org/packages/f5/80/49f88d3afc724b4ac7fbd664c8452d6db51b49915be48c6982659e0e7942/xxhash-3.6.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cd01fa2aa00d8b017c97eb46b9a794fbdca53fc14f845f5a328c71254b0abb7", size = 445614, upload-time = "2025-10-02T14:35:45.216Z" }, + { url = "https://files.pythonhosted.org/packages/ed/ba/603ce3961e339413543d8cd44f21f2c80e2a7c5cfe692a7b1f2cccf58f3c/xxhash-3.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0226aa89035b62b6a86d3c68df4d7c1f47a342b8683da2b60cedcddb46c4d95b", size = 194024, upload-time = "2025-10-02T14:35:46.959Z" }, + { url = "https://files.pythonhosted.org/packages/78/d1/8e225ff7113bf81545cfdcd79eef124a7b7064a0bba53605ff39590b95c2/xxhash-3.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c6e193e9f56e4ca4923c61238cdaced324f0feac782544eb4c6d55ad5cc99ddd", size = 210541, upload-time = "2025-10-02T14:35:48.301Z" }, + { url = "https://files.pythonhosted.org/packages/6f/58/0f89d149f0bad89def1a8dd38feb50ccdeb643d9797ec84707091d4cb494/xxhash-3.6.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9176dcaddf4ca963d4deb93866d739a343c01c969231dbe21680e13a5d1a5bf0", size = 198305, upload-time = "2025-10-02T14:35:49.584Z" }, + { url = "https://files.pythonhosted.org/packages/11/38/5eab81580703c4df93feb5f32ff8fa7fe1e2c51c1f183ee4e48d4bb9d3d7/xxhash-3.6.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c1ce4009c97a752e682b897aa99aef84191077a9433eb237774689f14f8ec152", size = 210848, upload-time = "2025-10-02T14:35:50.877Z" }, + { url = "https://files.pythonhosted.org/packages/5e/6b/953dc4b05c3ce678abca756416e4c130d2382f877a9c30a20d08ee6a77c0/xxhash-3.6.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:8cb2f4f679b01513b7adbb9b1b2f0f9cdc31b70007eaf9d59d0878809f385b11", size = 414142, upload-time = "2025-10-02T14:35:52.15Z" }, + { url = "https://files.pythonhosted.org/packages/08/a9/238ec0d4e81a10eb5026d4a6972677cbc898ba6c8b9dbaec12ae001b1b35/xxhash-3.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:653a91d7c2ab54a92c19ccf43508b6a555440b9be1bc8be553376778be7f20b5", size = 191547, upload-time = "2025-10-02T14:35:53.547Z" }, + { url = "https://files.pythonhosted.org/packages/f1/ee/3cf8589e06c2164ac77c3bf0aa127012801128f1feebf2a079272da5737c/xxhash-3.6.0-cp314-cp314-win32.whl", hash = "sha256:a756fe893389483ee8c394d06b5ab765d96e68fbbfe6fde7aa17e11f5720559f", size = 31214, upload-time = "2025-10-02T14:35:54.746Z" }, + { url = "https://files.pythonhosted.org/packages/02/5d/a19552fbc6ad4cb54ff953c3908bbc095f4a921bc569433d791f755186f1/xxhash-3.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:39be8e4e142550ef69629c9cd71b88c90e9a5db703fecbcf265546d9536ca4ad", size = 32290, upload-time = "2025-10-02T14:35:55.791Z" }, + { url = "https://files.pythonhosted.org/packages/b1/11/dafa0643bc30442c887b55baf8e73353a344ee89c1901b5a5c54a6c17d39/xxhash-3.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:25915e6000338999236f1eb68a02a32c3275ac338628a7eaa5a269c401995679", size = 28795, upload-time = "2025-10-02T14:35:57.162Z" }, + { url = "https://files.pythonhosted.org/packages/2c/db/0e99732ed7f64182aef4a6fb145e1a295558deec2a746265dcdec12d191e/xxhash-3.6.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c5294f596a9017ca5a3e3f8884c00b91ab2ad2933cf288f4923c3fd4346cf3d4", size = 32955, upload-time = "2025-10-02T14:35:58.267Z" }, + { url = "https://files.pythonhosted.org/packages/55/f4/2a7c3c68e564a099becfa44bb3d398810cc0ff6749b0d3cb8ccb93f23c14/xxhash-3.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1cf9dcc4ab9cff01dfbba78544297a3a01dafd60f3bde4e2bfd016cf7e4ddc67", size = 31072, upload-time = "2025-10-02T14:35:59.382Z" }, + { url = "https://files.pythonhosted.org/packages/c6/d9/72a29cddc7250e8a5819dad5d466facb5dc4c802ce120645630149127e73/xxhash-3.6.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:01262da8798422d0685f7cef03b2bd3f4f46511b02830861df548d7def4402ad", size = 196579, upload-time = "2025-10-02T14:36:00.838Z" }, + { url = "https://files.pythonhosted.org/packages/63/93/b21590e1e381040e2ca305a884d89e1c345b347404f7780f07f2cdd47ef4/xxhash-3.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51a73fb7cb3a3ead9f7a8b583ffd9b8038e277cdb8cb87cf890e88b3456afa0b", size = 215854, upload-time = "2025-10-02T14:36:02.207Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b8/edab8a7d4fa14e924b29be877d54155dcbd8b80be85ea00d2be3413a9ed4/xxhash-3.6.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b9c6df83594f7df8f7f708ce5ebeacfc69f72c9fbaaababf6cf4758eaada0c9b", size = 214965, upload-time = "2025-10-02T14:36:03.507Z" }, + { url = "https://files.pythonhosted.org/packages/27/67/dfa980ac7f0d509d54ea0d5a486d2bb4b80c3f1bb22b66e6a05d3efaf6c0/xxhash-3.6.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:627f0af069b0ea56f312fd5189001c24578868643203bca1abbc2c52d3a6f3ca", size = 448484, upload-time = "2025-10-02T14:36:04.828Z" }, + { url = "https://files.pythonhosted.org/packages/8c/63/8ffc2cc97e811c0ca5d00ab36604b3ea6f4254f20b7bc658ca825ce6c954/xxhash-3.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa912c62f842dfd013c5f21a642c9c10cd9f4c4e943e0af83618b4a404d9091a", size = 196162, upload-time = "2025-10-02T14:36:06.182Z" }, + { url = "https://files.pythonhosted.org/packages/4b/77/07f0e7a3edd11a6097e990f6e5b815b6592459cb16dae990d967693e6ea9/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b465afd7909db30168ab62afe40b2fcf79eedc0b89a6c0ab3123515dc0df8b99", size = 213007, upload-time = "2025-10-02T14:36:07.733Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d8/bc5fa0d152837117eb0bef6f83f956c509332ce133c91c63ce07ee7c4873/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a881851cf38b0a70e7c4d3ce81fc7afd86fbc2a024f4cfb2a97cf49ce04b75d3", size = 200956, upload-time = "2025-10-02T14:36:09.106Z" }, + { url = "https://files.pythonhosted.org/packages/26/a5/d749334130de9411783873e9b98ecc46688dad5db64ca6e04b02acc8b473/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9b3222c686a919a0f3253cfc12bb118b8b103506612253b5baeaac10d8027cf6", size = 213401, upload-time = "2025-10-02T14:36:10.585Z" }, + { url = "https://files.pythonhosted.org/packages/89/72/abed959c956a4bfc72b58c0384bb7940663c678127538634d896b1195c10/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:c5aa639bc113e9286137cec8fadc20e9cd732b2cc385c0b7fa673b84fc1f2a93", size = 417083, upload-time = "2025-10-02T14:36:12.276Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b3/62fd2b586283b7d7d665fb98e266decadf31f058f1cf6c478741f68af0cb/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5c1343d49ac102799905e115aee590183c3921d475356cb24b4de29a4bc56518", size = 193913, upload-time = "2025-10-02T14:36:14.025Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/c19c42c5b3f5a4aad748a6d5b4f23df3bed7ee5445accc65a0fb3ff03953/xxhash-3.6.0-cp314-cp314t-win32.whl", hash = "sha256:5851f033c3030dd95c086b4a36a2683c2ff4a799b23af60977188b057e467119", size = 31586, upload-time = "2025-10-02T14:36:15.603Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/4cc450345be9924fd5dc8c590ceda1db5b43a0a889587b0ae81a95511360/xxhash-3.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0444e7967dac37569052d2409b00a8860c2135cff05502df4da80267d384849f", size = 32526, upload-time = "2025-10-02T14:36:16.708Z" }, + { url = "https://files.pythonhosted.org/packages/0f/c9/7243eb3f9eaabd1a88a5a5acadf06df2d83b100c62684b7425c6a11bcaa8/xxhash-3.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bb79b1e63f6fd84ec778a4b1916dfe0a7c3fdb986c06addd5db3a0d413819d95", size = 28898, upload-time = "2025-10-02T14:36:17.843Z" }, +] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +]