chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,42 @@
|
||||
# Git
|
||||
.git
|
||||
.gitignore
|
||||
.github
|
||||
|
||||
# Rust
|
||||
target/
|
||||
|
||||
# Documentation
|
||||
*.md
|
||||
docs/
|
||||
CHANGELOG.md
|
||||
SECURITY.md
|
||||
CODE_OF_CONDUCT.md
|
||||
CONTRIBUTING.md
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Docker
|
||||
docker/
|
||||
Dockerfile*
|
||||
docker-compose*.yml
|
||||
.dockerignore
|
||||
|
||||
# Test artifacts
|
||||
*.log
|
||||
*.tmp
|
||||
|
||||
# Build artifacts
|
||||
*.o
|
||||
*.so
|
||||
*.dylib
|
||||
*.dll
|
||||
@@ -0,0 +1,24 @@
|
||||
# EditorConfig helps maintain consistent coding styles
|
||||
# https://editorconfig.org
|
||||
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
|
||||
[*.rs]
|
||||
indent_size = 4
|
||||
|
||||
[*.{yml,yaml}]
|
||||
indent_size = 2
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
@@ -0,0 +1,28 @@
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
|
||||
# Rust source files
|
||||
*.rs text eol=lf
|
||||
*.toml text eol=lf
|
||||
|
||||
# Documentation
|
||||
*.md text eol=lf
|
||||
*.txt text eol=lf
|
||||
|
||||
# Scripts
|
||||
*.sh text eol=lf
|
||||
|
||||
# Binary files
|
||||
*.mv2 binary
|
||||
*.mv2e binary
|
||||
*.pdf binary
|
||||
*.png binary
|
||||
*.jpg binary
|
||||
*.jpeg binary
|
||||
*.gif binary
|
||||
*.ico binary
|
||||
*.woff binary
|
||||
*.woff2 binary
|
||||
|
||||
# Prevent showing Cargo.lock diffs
|
||||
Cargo.lock linguist-generated=true
|
||||
@@ -0,0 +1,3 @@
|
||||
# These are supported funding model platforms
|
||||
|
||||
github: [memvid]
|
||||
@@ -0,0 +1,45 @@
|
||||
---
|
||||
name: Bug Report
|
||||
about: Report a bug to help us improve Memvid
|
||||
title: '[BUG] '
|
||||
labels: 'bug'
|
||||
assignees: ''
|
||||
---
|
||||
|
||||
## Bug Description
|
||||
A clear and concise description of the bug.
|
||||
|
||||
## Steps to Reproduce
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
## Expected Behavior
|
||||
What you expected to happen.
|
||||
|
||||
## Actual Behavior
|
||||
What actually happened.
|
||||
|
||||
## Environment
|
||||
- **OS**: [e.g., macOS 14.0, Ubuntu 22.04, Windows 11]
|
||||
- **Rust Version**: [e.g., 1.85.0]
|
||||
- **Memvid Version**: [e.g., 2.0.0]
|
||||
- **Features Enabled**: [e.g., lex, vec, clip]
|
||||
|
||||
## Minimal Reproducible Example
|
||||
```rust
|
||||
// Code to reproduce the issue
|
||||
```
|
||||
|
||||
## Error Output
|
||||
```
|
||||
// Paste any error messages or stack traces here
|
||||
```
|
||||
|
||||
## Additional Context
|
||||
Any other context about the problem (screenshots, logs, etc.)
|
||||
|
||||
## Checklist
|
||||
- [ ] I have searched existing issues for duplicates
|
||||
- [ ] I have tested with the latest version
|
||||
- [ ] I can reproduce this consistently
|
||||
@@ -0,0 +1,11 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Questions & Discussions
|
||||
url: https://github.com/memvid/memvid/discussions
|
||||
about: Ask questions and discuss ideas with the community
|
||||
- name: Documentation
|
||||
url: https://docs.memvid.com
|
||||
about: Read the official documentation
|
||||
- name: Discord Community
|
||||
url: https://discord.gg/2mynS7fcK7
|
||||
about: Join our Discord for real-time chat
|
||||
@@ -0,0 +1,32 @@
|
||||
---
|
||||
name: Feature Request
|
||||
about: Suggest a new feature for Memvid
|
||||
title: '[FEATURE] '
|
||||
labels: 'enhancement'
|
||||
assignees: ''
|
||||
---
|
||||
|
||||
## Feature Description
|
||||
A clear and concise description of the feature you'd like.
|
||||
|
||||
## Problem Statement
|
||||
What problem does this feature solve? Why is it needed?
|
||||
|
||||
## Proposed Solution
|
||||
Describe how you envision this feature working.
|
||||
|
||||
## Example Usage
|
||||
```rust
|
||||
// How would this feature be used?
|
||||
```
|
||||
|
||||
## Alternatives Considered
|
||||
What alternative solutions or features have you considered?
|
||||
|
||||
## Additional Context
|
||||
Any other context, mockups, or examples.
|
||||
|
||||
## Checklist
|
||||
- [ ] I have searched existing issues/discussions for similar requests
|
||||
- [ ] This feature aligns with Memvid's goals (portable, single-file AI memory)
|
||||
- [ ] I am willing to help implement this feature
|
||||
@@ -0,0 +1,42 @@
|
||||
## Description
|
||||
Brief description of the changes in this PR.
|
||||
|
||||
## Related Issue
|
||||
Fixes #(issue number)
|
||||
|
||||
## Type of Change
|
||||
- [ ] Bug fix (non-breaking change that fixes an issue)
|
||||
- [ ] New feature (non-breaking change that adds functionality)
|
||||
- [ ] Breaking change (fix or feature that would cause existing functionality to change)
|
||||
- [ ] Documentation update
|
||||
- [ ] Performance improvement
|
||||
- [ ] Code refactoring (no functional changes)
|
||||
|
||||
## Changes Made
|
||||
-
|
||||
-
|
||||
-
|
||||
|
||||
## Testing
|
||||
- [ ] I have added tests that prove my fix/feature works
|
||||
- [ ] All existing tests pass (`cargo test`)
|
||||
- [ ] I have tested on my local machine
|
||||
|
||||
## Documentation
|
||||
- [ ] I have updated relevant documentation
|
||||
- [ ] I have added doc comments for new public APIs
|
||||
- [ ] CHANGELOG.md has been updated (if applicable)
|
||||
|
||||
## Checklist
|
||||
- [ ] My code follows the project's style guidelines
|
||||
- [ ] I have performed a self-review of my code
|
||||
- [ ] I have commented my code where necessary
|
||||
- [ ] My changes generate no new warnings
|
||||
- [ ] I have run `cargo clippy` and addressed any issues
|
||||
- [ ] I have run `cargo fmt` to format my code
|
||||
|
||||
## Screenshots (if applicable)
|
||||
Add screenshots to help explain your changes.
|
||||
|
||||
## Additional Notes
|
||||
Any additional information reviewers should know.
|
||||
@@ -0,0 +1,31 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: cargo
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown: # applies only to version-updates (not security-updates)
|
||||
default-days: 7
|
||||
ignore:
|
||||
- dependency-name: "bincode"
|
||||
versions: [">=3.0.0"] # unsupported since v3
|
||||
- package-ecosystem: docker
|
||||
directories:
|
||||
- docker/cli
|
||||
- docker/core
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
- package-ecosystem: docker-compose
|
||||
directory: docker/core
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown: # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns
|
||||
default-days: 7
|
||||
@@ -0,0 +1,63 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
rust: [stable]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: ${{ matrix.rust }}
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
|
||||
|
||||
- name: Build
|
||||
run: cargo build --verbose
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --verbose ${{ runner.os == 'Windows' && '-- --test-threads=1' || '' }}
|
||||
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Install components
|
||||
run: rustup component add rustfmt clippy
|
||||
|
||||
- name: Check formatting
|
||||
run: cargo fmt --all -- --check
|
||||
|
||||
- name: Clippy
|
||||
run: cargo clippy -- -D warnings -A clippy::non_std_lazy_statics
|
||||
@@ -0,0 +1,50 @@
|
||||
name: Docker Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: docker.io
|
||||
IMAGE_NAME: memvid/cli
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract version
|
||||
id: version
|
||||
run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: ./docker/cli
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.IMAGE_NAME }}:latest
|
||||
${{ env.IMAGE_NAME }}:${{ steps.version.outputs.VERSION }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
doc.mv2
|
||||
*.mv2
|
||||
*.mv2-wal
|
||||
*.mv2-shm
|
||||
*.mv2-lock
|
||||
*.mv2-journal
|
||||
*.mv2.*
|
||||
*.mv2-wal.*
|
||||
*.mv2-shm.*
|
||||
*.mv2-lock.*
|
||||
*.mv2-journal.*
|
||||
|
||||
.DS_Store
|
||||
*.bk
|
||||
.idea
|
||||
|
||||
|
||||
/target
|
||||
**/*.rs.bk
|
||||
# Cargo.lock - now committed for reproducible CI builds
|
||||
.idea
|
||||
|
||||
|
||||
entry2.json
|
||||
entry1.txt
|
||||
vector_entry.txt
|
||||
vector_embedding.txt
|
||||
|
||||
*.pdf
|
||||
*.txt
|
||||
# Exception: SymSpell dictionary files needed for PDF text cleanup
|
||||
!data/*.txt
|
||||
*.docx
|
||||
*.doc
|
||||
*.pptx
|
||||
*.ppt
|
||||
*.xlsx
|
||||
*.xls
|
||||
*.csv
|
||||
*.json
|
||||
|
||||
models/
|
||||
*.DS_Store
|
||||
|
||||
issue_overview.md
|
||||
|
||||
feature_100x_memvid.md
|
||||
|
||||
plan_model_robust.md
|
||||
@@ -0,0 +1,50 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to Memvid will be documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- Initial public release of Memvid core library
|
||||
- Single-file `.mv2` format for portable AI memory
|
||||
- Full-text search with BM25 ranking (Tantivy)
|
||||
- Vector similarity search with HNSW
|
||||
- PDF, DOCX, XLSX document ingestion
|
||||
- CLIP visual embeddings for image search
|
||||
- Whisper audio transcription
|
||||
- Timeline queries for chronological browsing
|
||||
- Crash-safe WAL-based writes
|
||||
- Blake3 checksums for data integrity
|
||||
- Ed25519 signatures for authenticity
|
||||
- Optional AES-256-GCM encryption
|
||||
|
||||
### Security
|
||||
- Embedded WAL prevents data corruption
|
||||
- Atomic commits ensure consistency
|
||||
- File locking prevents concurrent write conflicts
|
||||
|
||||
## [2.0.0] - 2026-01-05
|
||||
|
||||
### Added
|
||||
- Complete rewrite in Rust for performance and safety
|
||||
- New `.mv2` file format (single-file, no sidecars)
|
||||
- Append-only frame-based architecture
|
||||
- Built-in full-text and vector search
|
||||
- Cross-platform support (macOS, Linux, Windows)
|
||||
|
||||
### Changed
|
||||
- Migrated from Python to Rust
|
||||
- New API design focused on simplicity
|
||||
- Improved memory efficiency
|
||||
|
||||
### Removed
|
||||
- Legacy Python implementation
|
||||
- QR code video encoding (replaced with efficient binary format)
|
||||
|
||||
---
|
||||
|
||||
[Unreleased]: https://github.com/memvid/memvid/compare/v2.0.0...HEAD
|
||||
[2.0.0]: https://github.com/memvid/memvid/releases/tag/v2.0.0
|
||||
@@ -0,0 +1,119 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code and other AI assistants working with this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
Memvid is a Rust library that provides a single-file memory layer for AI agents. It packages documents, embeddings, search indices, and metadata into a portable `.mv2` file.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Core Components
|
||||
|
||||
```
|
||||
src/
|
||||
├── lib.rs # Public API exports
|
||||
├── memvid/ # Main Memvid struct and operations
|
||||
│ ├── mod.rs # Memvid implementation
|
||||
│ ├── mutation.rs # Write operations (put, commit, delete)
|
||||
│ ├── search/ # Search implementations
|
||||
│ └── ask.rs # RAG query handling
|
||||
├── io/ # File I/O layer
|
||||
│ ├── header.rs # File header (4KB)
|
||||
│ ├── wal.rs # Write-ahead log
|
||||
│ └── time_index.rs # Chronological index
|
||||
├── lex.rs # Full-text search (Tantivy)
|
||||
├── vec.rs # Vector search (HNSW)
|
||||
├── clip.rs # CLIP embeddings
|
||||
├── whisper.rs # Audio transcription
|
||||
└── types/ # Type definitions
|
||||
```
|
||||
|
||||
### File Format (.mv2)
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │
|
||||
├────────────────────────────┤
|
||||
│ Lex Index (Tantivy) │
|
||||
├────────────────────────────┤
|
||||
│ Vec Index (HNSW) │
|
||||
├────────────────────────────┤
|
||||
│ Time Index │
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## Development Commands
|
||||
|
||||
```bash
|
||||
# Build
|
||||
cargo build
|
||||
cargo build --release
|
||||
|
||||
# Test
|
||||
cargo test
|
||||
cargo test --test lifecycle
|
||||
cargo test -- --nocapture
|
||||
|
||||
# Lint
|
||||
cargo clippy
|
||||
cargo fmt
|
||||
|
||||
# Run examples
|
||||
cargo run --example basic_usage
|
||||
cargo run --example pdf_ingestion
|
||||
|
||||
# Benchmarks
|
||||
cargo bench
|
||||
```
|
||||
|
||||
## Key APIs
|
||||
|
||||
```rust
|
||||
// Create/Open
|
||||
let mut mem = Memvid::create("file.mv2")?;
|
||||
let mut mem = Memvid::open("file.mv2")?;
|
||||
|
||||
// Write
|
||||
mem.put_bytes(content)?;
|
||||
mem.put_bytes_with_options(content, options)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
mem.search(SearchRequest { query, top_k, .. })?;
|
||||
mem.timeline(TimelineQuery::default())?;
|
||||
|
||||
// Verify
|
||||
Memvid::verify("file.mv2", deep)?;
|
||||
```
|
||||
|
||||
## Feature Flags
|
||||
|
||||
| Feature | Purpose |
|
||||
|---------|---------|
|
||||
| `lex` | Full-text search (default) |
|
||||
| `vec` | Vector similarity search |
|
||||
| `clip` | CLIP image embeddings |
|
||||
| `whisper` | Audio transcription |
|
||||
| `encryption` | AES-256-GCM encryption |
|
||||
|
||||
## Code Style
|
||||
|
||||
- Follow Rust idioms and best practices
|
||||
- Use `thiserror` for error types
|
||||
- Use `tracing` for logging
|
||||
- Add doc comments for public APIs
|
||||
- Write tests for new functionality
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Single-file design: Never create sidecar files
|
||||
- Crash safety: All writes go through WAL
|
||||
- Append-only: Frames are immutable once committed
|
||||
- No async: Library is synchronous for simplicity
|
||||
@@ -0,0 +1,45 @@
|
||||
# Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
**Examples of behavior that contributes to a positive environment:**
|
||||
|
||||
- Using welcoming and inclusive language
|
||||
- Being respectful of differing viewpoints and experiences
|
||||
- Gracefully accepting constructive criticism
|
||||
- Focusing on what is best for the community
|
||||
- Showing empathy towards other community members
|
||||
|
||||
**Examples of unacceptable behavior:**
|
||||
|
||||
- The use of sexualized language or imagery, and sexual attention or advances of any kind
|
||||
- Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
- Public or private harassment
|
||||
- Publishing others' private information without explicit permission
|
||||
- Other conduct which could reasonably be considered inappropriate in a professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at:
|
||||
|
||||
**conduct@memvid.com**
|
||||
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1.
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
# Contributing to Memvid
|
||||
|
||||
Thank you for your interest in contributing to Memvid! We welcome contributions from everyone.
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- **Rust 1.85.0+** — Install from [rustup.rs](https://rustup.rs)
|
||||
- **Git** — For version control
|
||||
|
||||
### Setup
|
||||
|
||||
1. **Fork the repository** on GitHub
|
||||
|
||||
2. **Clone your fork**:
|
||||
```bash
|
||||
git clone https://github.com/YOUR_USERNAME/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
3. **Build the project**:
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
4. **Run tests**:
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
## Development Workflow
|
||||
|
||||
### Creating a Branch
|
||||
|
||||
```bash
|
||||
git checkout -b feature/your-feature-name
|
||||
# or
|
||||
git checkout -b fix/your-bug-fix
|
||||
```
|
||||
|
||||
### Making Changes
|
||||
|
||||
1. Write your code following the [code style guidelines](#code-style)
|
||||
2. Add tests for new functionality
|
||||
3. Ensure all tests pass: `cargo test`
|
||||
4. Run clippy: `cargo clippy`
|
||||
5. Format code: `cargo fmt`
|
||||
|
||||
### Committing
|
||||
|
||||
Write clear, concise commit messages:
|
||||
|
||||
```bash
|
||||
git commit -m "feat: add support for XYZ"
|
||||
git commit -m "fix: resolve issue with ABC"
|
||||
git commit -m "docs: update README examples"
|
||||
```
|
||||
|
||||
### Submitting a Pull Request
|
||||
|
||||
1. Push to your fork:
|
||||
```bash
|
||||
git push origin feature/your-feature-name
|
||||
```
|
||||
|
||||
2. Open a Pull Request on GitHub
|
||||
|
||||
3. Fill out the PR template completely
|
||||
|
||||
4. Wait for review and address feedback
|
||||
|
||||
## Code Style
|
||||
|
||||
### Rust Guidelines
|
||||
|
||||
- Follow standard Rust idioms and conventions
|
||||
- Use `rustfmt` for formatting (`cargo fmt`)
|
||||
- Use `clippy` for linting (`cargo clippy`). We maintain a **zero-warning policy**.
|
||||
- Prefer explicit types for public APIs
|
||||
- Use `thiserror` for error definitions
|
||||
|
||||
### Linting & Safety
|
||||
|
||||
We enforce strict linting to ensure safety and portability:
|
||||
|
||||
1. **Zero Warnings**: CI will fail on any warning. Run `cargo clippy --workspace --all-targets -- -D warnings` locally.
|
||||
2. **No Panics**: `unwrap()` and `expect()` are **denied** in library code. Use `Result` propagation (`?`) or graceful error handling. They are allowed in `tests/`.
|
||||
3. **No Truncation**: `cast_possible_truncation` is denied. Use `try_from` when converting `u64` to `usize`/`u32`.
|
||||
4. **Exceptions**: We allow pragmatic lints (e.g., `cast_precision_loss` for ML math) in `src/lib.rs`. Do not add global `#![allow]` without discussion.
|
||||
|
||||
### Documentation
|
||||
|
||||
- Add doc comments (`///`) to all public functions, structs, and modules
|
||||
- Include examples in doc comments where helpful
|
||||
- Keep comments concise and up-to-date
|
||||
|
||||
### Testing
|
||||
|
||||
- Write unit tests for new functionality
|
||||
- Place tests in the same file using `#[cfg(test)]` module
|
||||
- Integration tests go in the `tests/` directory
|
||||
- Aim for high coverage of edge cases
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
memvid/
|
||||
├── src/ # Source code
|
||||
│ ├── lib.rs # Public API
|
||||
│ ├── memvid/ # Core implementation
|
||||
│ ├── io/ # File I/O
|
||||
│ └── types/ # Type definitions
|
||||
├── tests/ # Integration tests
|
||||
├── examples/ # Example code
|
||||
├── benchmarks/ # Benchmarks
|
||||
└── data/ # Required data files
|
||||
```
|
||||
|
||||
## Feature Flags
|
||||
|
||||
When adding new functionality, consider if it should be behind a feature flag:
|
||||
|
||||
```toml
|
||||
[features]
|
||||
my_feature = ["dep:some-dependency"]
|
||||
```
|
||||
|
||||
This keeps the default build lean and fast.
|
||||
|
||||
## Reporting Issues
|
||||
|
||||
When reporting bugs, please include:
|
||||
|
||||
- Rust version (`rustc --version`)
|
||||
- Operating system
|
||||
- Memvid version
|
||||
- Minimal code to reproduce
|
||||
- Expected vs actual behavior
|
||||
|
||||
## Translations
|
||||
|
||||
Interested in translating Memvid's documentation? See [Contributing Translations](docs/i18n/CONTRIBUTING_TRANSLATIONS.md) for guidelines on translating the README and other documentation.
|
||||
|
||||
## Getting Help
|
||||
|
||||
- Open a [Discussion](https://github.com/memvid/memvid/discussions) for questions
|
||||
- Check existing [Issues](https://github.com/memvid/memvid/issues) for similar problems
|
||||
- Email: contact@memvid.com
|
||||
|
||||
## Recognition
|
||||
|
||||
Contributors are:
|
||||
- Listed in release notes
|
||||
- Part of the Memvid community
|
||||
|
||||
---
|
||||
|
||||
**Thank you for making Memvid better!**
|
||||
Generated
+6837
File diff suppressed because it is too large
Load Diff
+164
@@ -0,0 +1,164 @@
|
||||
[package]
|
||||
name = "memvid-core"
|
||||
version = "2.0.140"
|
||||
edition = "2024"
|
||||
rust-version = "1.85.0"
|
||||
license = "Apache-2.0"
|
||||
description = "Core library for Memvid v2, a crash-safe, deterministic, single-file AI memory."
|
||||
repository = "https://github.com/memvid/memvid"
|
||||
documentation = "https://docs.memvid.com"
|
||||
readme = "README.md"
|
||||
keywords = ["ai", "memory", "search", "vector", "embeddings"]
|
||||
categories = ["database", "data-structures"]
|
||||
include = ["src/**/*", "data/**/*", "Cargo.toml", "README.md"]
|
||||
|
||||
[dependencies]
|
||||
once_cell = "1.19.0"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
bincode = { version = "2.0.1", features = ["serde"] }
|
||||
blake3 = "1.5.1"
|
||||
uuid = { version = "1.10.0", features = ["v4", "serde"] }
|
||||
log = "0.4.22"
|
||||
thiserror = "2.0.17"
|
||||
fs2 = "0.4.3"
|
||||
zstd = "0.13.1"
|
||||
lz4_flex = "0.12.0"
|
||||
tracing = "0.1.41"
|
||||
serde_json = "1.0.145"
|
||||
ed25519-dalek = { version = "2.2.0", features = ["std"] }
|
||||
base64 = "0.22.1"
|
||||
sha2 = "0.10.9"
|
||||
hex = "0.4.3"
|
||||
# extractous uses GraalVM native compilation which doesn't work on Windows ARM or WSL2 on ARM
|
||||
# Enable with --features extractous for full document extraction (PDF, DOCX, etc.)
|
||||
extractous = { version = "0.3", optional = true }
|
||||
regex = "1.11.1"
|
||||
time = { version = "0.3.36", features = ["formatting", "parsing"] }
|
||||
chrono = { version = "0.4.42", features = ["serde"] }
|
||||
interim = { version = "0.2.1", optional = true }
|
||||
lopdf = "0.39"
|
||||
# Pure Rust PDF text extraction - used as primary extractor when extractous is disabled
|
||||
pdf-extract = { version = "0.10", optional = true }
|
||||
# High-accuracy PDF text extraction with perfect word spacing (2025)
|
||||
pdf_oxide = { version = "0.3", optional = true }
|
||||
unicode-normalization = "0.1"
|
||||
unicode-segmentation = "1.11"
|
||||
zip = { version = "7.1", default-features = false, features = ["deflate"] }
|
||||
quick-xml = "0.31"
|
||||
calamine = "0.22"
|
||||
pdfium-render = { version = "0.8.28", optional = true }
|
||||
tempfile = "3.10.1"
|
||||
num_cpus = { version = "1.17", optional = true }
|
||||
crossbeam-channel = { version = "0.5.13", optional = true }
|
||||
memmap2 = "0.9"
|
||||
memchr = "2.7"
|
||||
same-file = "1.0"
|
||||
fs-err = "3.2"
|
||||
atomic-write-file = "0.3"
|
||||
dirs-next = "2.0"
|
||||
smallvec = { version = "1.13", features = ["serde", "union", "const_generics", "write"] }
|
||||
tantivy = { version = "0.25.0", optional = true, default-features = false, features = ["mmap"] }
|
||||
ort = { version = "=2.0.0-rc.10", optional = true }
|
||||
hnsw = { version = "0.11.0", optional = true, features = ["serde"] }
|
||||
jsonwebtoken = { version = "10.0.0", optional = true, features = ["rust_crypto"] }
|
||||
image = { version = "0.25", optional = true, default-features = false, features = ["jpeg", "png", "webp"] }
|
||||
ndarray = { version = "0.16", optional = true }
|
||||
rayon = { version = "1.10", optional = true }
|
||||
tokenizers = { version = "0.22", optional = true }
|
||||
symphonia = { version = "0.5.3", optional = true, default-features = false, features = ["aac", "mp3", "flac", "isomp4", "ogg", "wav", "pcm"] }
|
||||
rubato = { version = "0.15", optional = true }
|
||||
rustfft = { version = "6.2", optional = true }
|
||||
|
||||
# Encryption capsules (.mv2e) - feature-gated
|
||||
argon2 = { version = "0.5", optional = true }
|
||||
aes-gcm = { version = "0.10", optional = true }
|
||||
rand = { version = "0.8", optional = true, features = ["serde1"] }
|
||||
rand_pcg = { version = "0.3", optional = true, features = ["serde1"] }
|
||||
zeroize = { version = "1.7", optional = true }
|
||||
|
||||
# Candle ML framework for Whisper transcription
|
||||
candle-core = { version = "0.9", optional = true }
|
||||
candle-nn = { version = "0.9", optional = true }
|
||||
candle-transformers = { version = "0.9", optional = true }
|
||||
hf-hub = { version = "0.4", optional = true, features = ["tokio"] }
|
||||
byteorder = { version = "1.5", optional = true }
|
||||
|
||||
# SymSpell for PDF text cleanup - fixes broken word spacing from PDF extraction
|
||||
symspell = { version = "0.4", optional = true }
|
||||
# SIMD acceleration for vector distance calculations
|
||||
wide = { version = "1.1", optional = true }
|
||||
space = { version = "0.17", optional = true }
|
||||
|
||||
# HTTP client for API-based embedding providers (OpenAI, etc.)
|
||||
reqwest = { version = "0.12", optional = true, default-features = false, features = ["blocking", "json", "rustls-tls"] }
|
||||
|
||||
# Platform-specific: libc for stderr suppression on macOS
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
[features]
|
||||
default = ["lex", "pdf_extract", "simd"]
|
||||
# pdf_oxide disabled - cff-parser panics on ligature fonts (uniFB01/uniFB02)
|
||||
# symspell_cleanup - enables robust PDF text repair (requires `make download-models` for dictionaries)
|
||||
lex = ["dep:tantivy"]
|
||||
extractous = ["dep:extractous"]
|
||||
# Pure Rust PDF extraction - faster than lopdf for text extraction, cross-platform
|
||||
pdf_extract = ["dep:pdf-extract"]
|
||||
# High-accuracy PDF extraction with perfect word spacing (recommended for 2025+)
|
||||
pdf_oxide = ["dep:pdf_oxide"]
|
||||
vec = ["dep:ort", "dep:hnsw", "dep:ndarray", "dep:tokenizers", "dep:space", "dep:rand", "dep:rand_pcg"]
|
||||
clip = ["vec", "dep:image", "dep:ndarray", "dep:rayon", "dep:tokenizers"]
|
||||
mmap = []
|
||||
pdfium = ["dep:pdfium-render"]
|
||||
temporal_track = []
|
||||
temporal_enrich = ["dep:interim"]
|
||||
parallel_segments = ["dep:num_cpus", "dep:crossbeam-channel"]
|
||||
# Logic-Mesh: entity-relationship graph with NER extraction using DistilBERT-NER ONNX
|
||||
logic_mesh = ["dep:ort", "dep:ndarray", "dep:tokenizers"]
|
||||
# Whisper: audio transcription with Candle inference
|
||||
whisper = ["dep:symphonia", "dep:rubato", "dep:tokenizers", "dep:candle-core", "dep:candle-nn", "dep:candle-transformers", "dep:hf-hub", "dep:byteorder"]
|
||||
# GPU acceleration for Whisper (optional)
|
||||
metal = ["candle-core/metal", "candle-nn/metal", "candle-transformers/metal"]
|
||||
cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]
|
||||
accelerate = ["candle-core/accelerate", "candle-nn/accelerate", "candle-transformers/accelerate"]
|
||||
# Time-travel replay for agent sessions
|
||||
replay = []
|
||||
# Password-based encryption capsules (.mv2e)
|
||||
encryption = ["dep:argon2", "dep:aes-gcm", "dep:rand", "dep:zeroize"]
|
||||
# SymSpell-based PDF text cleanup - fixes broken word spacing
|
||||
symspell_cleanup = ["dep:symspell"]
|
||||
# API-based embedding providers (OpenAI, Anthropic, etc.) - requires network
|
||||
api_embed = ["dep:reqwest"]
|
||||
# SIMD acceleration for vector distance calculations
|
||||
simd = ["dep:wide"]
|
||||
hnsw_bench = ["dep:hnsw", "dep:rand", "dep:space", "dep:rand_pcg"]
|
||||
|
||||
[dev-dependencies]
|
||||
fastrand = "2.0"
|
||||
tempfile = "3.10.1"
|
||||
criterion = { version = "0.8.1", features = ["html_reports"] }
|
||||
|
||||
[[example]]
|
||||
name = "text_embedding"
|
||||
required-features = ["vec"]
|
||||
|
||||
[[example]]
|
||||
name = "text_embed_cache_bench"
|
||||
required-features = ["vec"]
|
||||
|
||||
[[example]]
|
||||
name = "openai_embedding"
|
||||
required-features = ["api_embed"]
|
||||
|
||||
[[example]]
|
||||
name = "simd_benchmark"
|
||||
required-features = ["simd"]
|
||||
|
||||
[[bench]]
|
||||
name = "search_precision_benchmark"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "vec_search_benchmark"
|
||||
harness = false
|
||||
required-features = ["hnsw_bench"]
|
||||
@@ -0,0 +1,190 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to the Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2026 Memvid
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
# MV2 File Format Specification
|
||||
|
||||
Version 2.1
|
||||
|
||||
## Overview
|
||||
|
||||
MV2 is a single-file format for AI memory storage. Everything lives in one file: header, write-ahead log, data segments, search indices, and metadata. No sidecar files.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ .mv2 FILE │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Header │ 4 KB │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Embedded WAL │ 1-64 MB (capacity-dependent) │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Data Segments │ Variable │
|
||||
│ - Frame payloads │
|
||||
│ - Compressed content │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Lex Index Segment │ Tantivy index (optional) │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Vec Index Segment │ HNSW vectors (optional) │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Time Index Segment │ Chronological ordering │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment catalog + checksums │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Header (4096 bytes)
|
||||
|
||||
The header occupies the first 4 KB of the file.
|
||||
|
||||
| Offset | Size | Field | Description |
|
||||
|--------|------|-------|-------------|
|
||||
| 0 | 4 | `magic` | `MV2\0` (0x4D 0x56 0x32 0x00) |
|
||||
| 4 | 2 | `version` | Format version (little-endian) |
|
||||
| 6 | 1 | `spec_major` | Spec major version (2) |
|
||||
| 7 | 1 | `spec_minor` | Spec minor version (1) |
|
||||
| 8 | 8 | `footer_offset` | Byte offset to TOC |
|
||||
| 16 | 8 | `wal_offset` | Byte offset to WAL (always 4096) |
|
||||
| 24 | 8 | `wal_size` | WAL region size in bytes |
|
||||
| 32 | 8 | `wal_checkpoint_pos` | Last checkpointed sequence |
|
||||
| 40 | 8 | `wal_sequence` | Current WAL sequence number |
|
||||
| 48 | 32 | `toc_checksum` | SHA-256 of TOC segment |
|
||||
| 80 | 4016 | reserved | Zero-filled, reserved for future use |
|
||||
|
||||
All multi-byte integers are little-endian.
|
||||
|
||||
## Write-Ahead Log (WAL)
|
||||
|
||||
The embedded WAL provides crash recovery. It starts at byte 4096 and has a capacity determined by the file's target size:
|
||||
|
||||
| File Capacity | WAL Size |
|
||||
|---------------|----------|
|
||||
| < 100 MB | 1 MB |
|
||||
| < 1 GB | 4 MB |
|
||||
| < 10 GB | 16 MB |
|
||||
| >= 10 GB | 64 MB |
|
||||
|
||||
### WAL Entry Format
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────┐
|
||||
│ sequence │ 8 bytes (u64 LE) │
|
||||
│ entry_type │ 1 byte │
|
||||
│ payload_len │ 4 bytes (u32 LE) │
|
||||
│ payload │ variable │
|
||||
│ checksum │ 4 bytes (CRC32) │
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Entry types:
|
||||
- `0x01` - Frame append
|
||||
- `0x02` - Frame update
|
||||
- `0x03` - Frame delete (tombstone)
|
||||
- `0x04` - Index update
|
||||
|
||||
### Checkpoint Behavior
|
||||
|
||||
- Checkpoint triggers at 75% WAL occupancy or every 1,000 transactions
|
||||
- Checkpoint flushes WAL entries to data segments
|
||||
- `seal()` forces immediate checkpoint
|
||||
- Recovery replays entries with `sequence > wal_checkpoint_pos`
|
||||
|
||||
## Frame Structure
|
||||
|
||||
Each frame represents a single piece of content.
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `frame_id` | u64 | Unique identifier (monotonic) |
|
||||
| `uri` | String | Hierarchical path (`mv2://path/to/doc`) |
|
||||
| `title` | String? | Optional display title |
|
||||
| `created_at` | u64 | Unix timestamp (seconds) |
|
||||
| `encoding` | u8 | Content encoding (see below) |
|
||||
| `payload` | bytes | Compressed content |
|
||||
| `payload_checksum` | [u8; 32] | SHA-256 of uncompressed payload |
|
||||
| `tags` | Map<String, String> | User-defined key-value pairs |
|
||||
| `status` | u8 | 0=active, 1=tombstoned |
|
||||
|
||||
### Encoding Types
|
||||
|
||||
| Value | Name | Description |
|
||||
|-------|------|-------------|
|
||||
| 0 | Raw | Uncompressed bytes |
|
||||
| 1 | Zstd | Zstandard compression |
|
||||
| 2 | Lz4 | LZ4 compression |
|
||||
|
||||
## Data Segments
|
||||
|
||||
Frames are grouped into segments for efficient storage and retrieval.
|
||||
|
||||
### Segment Header
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────┐
|
||||
│ magic │ 4 bytes │
|
||||
│ version │ 2 bytes │
|
||||
│ segment_type │ 1 byte │
|
||||
│ frame_count │ 4 bytes │
|
||||
│ compressed │ 1 byte (bool) │
|
||||
│ checksum │ 32 bytes │
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Segment types:
|
||||
- `0x01` - Data segment (frames)
|
||||
- `0x02` - Lex index segment
|
||||
- `0x03` - Vec index segment
|
||||
- `0x04` - Time index segment
|
||||
|
||||
## Time Index
|
||||
|
||||
The time index enables chronological queries and time-travel.
|
||||
|
||||
### Time Index Entry
|
||||
|
||||
| Field | Size | Description |
|
||||
|-------|------|-------------|
|
||||
| `frame_id` | 8 | Frame identifier |
|
||||
| `timestamp` | 8 | Unix timestamp |
|
||||
| `offset` | 8 | Byte offset in data segment |
|
||||
|
||||
Magic: `MVTI` (0x4D 0x56 0x54 0x49)
|
||||
|
||||
## Lex Index (Full-Text Search)
|
||||
|
||||
When the `lex` feature is enabled, the file contains a Tantivy index segment.
|
||||
|
||||
Indexed fields:
|
||||
- `body` - Full text content
|
||||
- `title` - Document title
|
||||
- `uri` - Document URI
|
||||
- `tags` - Flattened tag values
|
||||
|
||||
Supports:
|
||||
- BM25 ranking
|
||||
- Phrase queries
|
||||
- Boolean operators
|
||||
- Date range filters
|
||||
|
||||
## Vec Index (Vector Search)
|
||||
|
||||
When the `vec` feature is enabled, the file contains an HNSW index segment.
|
||||
|
||||
| Parameter | Value |
|
||||
|-----------|-------|
|
||||
| Dimensions | 384 (BGE-small) |
|
||||
| Distance | Cosine similarity |
|
||||
| M | 16 |
|
||||
| ef_construction | 200 |
|
||||
|
||||
## Table of Contents (TOC)
|
||||
|
||||
The TOC is the final segment, pointed to by `footer_offset` in the header.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────┐
|
||||
│ magic │ "MVTC" │
|
||||
│ version │ 2 bytes │
|
||||
│ segment_count │ 4 bytes │
|
||||
│ segments[] │ SegmentDescriptor[] │
|
||||
│ manifests │ IndexManifests │
|
||||
│ checksum │ 32 bytes │
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Segment Descriptor
|
||||
|
||||
| Field | Size | Description |
|
||||
|-------|------|-------------|
|
||||
| `segment_type` | 1 | Type identifier |
|
||||
| `offset` | 8 | Byte offset in file |
|
||||
| `length` | 8 | Segment size in bytes |
|
||||
| `checksum` | 32 | SHA-256 of segment |
|
||||
|
||||
## URI Scheme
|
||||
|
||||
All content is addressable via `mv2://` URIs:
|
||||
|
||||
```
|
||||
mv2://[track/][path/]name
|
||||
```
|
||||
|
||||
Examples:
|
||||
- `mv2://meetings/2024-01-15`
|
||||
- `mv2://docs/api/reference.md`
|
||||
- `mv2://media/photo.png`
|
||||
|
||||
## Invariants
|
||||
|
||||
1. **Single-file guarantee**: No `.wal`, `.shm`, `.lock`, or other sidecar files
|
||||
2. **Append-only frames**: Existing frames are never modified in place
|
||||
3. **Determinism**: Same API calls produce identical bytes
|
||||
4. **Crash safety**: WAL ensures durability across unexpected termination
|
||||
5. **Self-describing**: TOC contains all metadata needed to parse the file
|
||||
|
||||
## Version History
|
||||
|
||||
| Version | Changes |
|
||||
|---------|---------|
|
||||
| 2.1 | Current version. Embedded WAL, temporal track support |
|
||||
| 2.0 | Single-file format, removed external indices |
|
||||
| 1.x | Legacy format (deprecated) |
|
||||
@@ -0,0 +1,236 @@
|
||||
.PHONY: help build build-release test test-verbose clean fmt fmt-check clippy clippy-fix doc doc-open check install run-example docker-build docker-test docker-push pre-commit tree outdated bloat test-doc test-release package publish-dry-run coverage watch
|
||||
|
||||
# Default target
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
# Variables
|
||||
CARGO := cargo
|
||||
RUST_VERSION := 1.85.0
|
||||
FEATURES := lex,pdf_extract
|
||||
|
||||
# Colors for output
|
||||
CYAN := \033[0;36m
|
||||
GREEN := \033[0;32m
|
||||
YELLOW := \033[0;33m
|
||||
NC := \033[0m # No Color
|
||||
|
||||
help: ## Show this help message
|
||||
@echo "$(CYAN)Memvid Makefile Commands:$(NC)"
|
||||
@echo ""
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " $(GREEN)%-20s$(NC) %s\n", $$1, $$2}'
|
||||
@echo ""
|
||||
|
||||
install: ## Install Rust toolchain and dependencies
|
||||
@echo "$(CYAN)Installing Rust toolchain...$(NC)"
|
||||
@rustup toolchain install $(RUST_VERSION) || true
|
||||
@rustup default $(RUST_VERSION) || true
|
||||
@echo "$(CYAN)Installing cargo dependencies...$(NC)"
|
||||
@$(CARGO) fetch
|
||||
|
||||
download-models: ## Download required models and dictionaries
|
||||
@echo "$(CYAN)Downloading SymSpell dictionaries...$(NC)"
|
||||
@mkdir -p data
|
||||
@curl -L https://raw.githubusercontent.com/wolfgarbe/SymSpell/master/SymSpell/frequency_dictionary_en_82_765.txt -o data/frequency_dictionary_en_82_765.txt
|
||||
@curl -L https://raw.githubusercontent.com/wolfgarbe/SymSpell/master/SymSpell/frequency_bigramdictionary_en_243_342.txt -o data/frequency_bigramdictionary_en_243_342.txt
|
||||
|
||||
|
||||
check: ## Check code without building
|
||||
@echo "$(CYAN)Checking code...$(NC)"
|
||||
@$(CARGO) check --features $(FEATURES)
|
||||
|
||||
build: ## Build in debug mode
|
||||
@echo "$(CYAN)Building in debug mode...$(NC)"
|
||||
@$(CARGO) build --features $(FEATURES)
|
||||
|
||||
build-release: ## Build in release mode (optimized)
|
||||
@echo "$(CYAN)Building in release mode...$(NC)"
|
||||
@$(CARGO) build --release --features $(FEATURES)
|
||||
|
||||
build-verbose: ## Build with verbose output
|
||||
@echo "$(CYAN)Building with verbose output...$(NC)"
|
||||
@$(CARGO) build --verbose --features $(FEATURES)
|
||||
|
||||
build-release-verbose: ## Build release with verbose output
|
||||
@echo "$(CYAN)Building release with verbose output...$(NC)"
|
||||
@$(CARGO) build --release --verbose --features $(FEATURES)
|
||||
|
||||
build-all-features: ## Build with all features enabled
|
||||
@echo "$(CYAN)Building with all features...$(NC)"
|
||||
@$(CARGO) build --release --all-features
|
||||
|
||||
test: ## Run tests
|
||||
@echo "$(CYAN)Running tests...$(NC)"
|
||||
@$(CARGO) test --features $(FEATURES)
|
||||
|
||||
test-verbose: ## Run tests with output
|
||||
@echo "$(CYAN)Running tests with output...$(NC)"
|
||||
@$(CARGO) test --features $(FEATURES) -- --nocapture
|
||||
|
||||
test-release: ## Run tests in release mode
|
||||
@echo "$(CYAN)Running tests in release mode...$(NC)"
|
||||
@$(CARGO) test --release --features $(FEATURES)
|
||||
|
||||
test-doc: ## Run documentation tests only
|
||||
@echo "$(CYAN)Running documentation tests...$(NC)"
|
||||
@$(CARGO) test --doc --features $(FEATURES)
|
||||
|
||||
test-all-targets: ## Run all test targets (lib, bins, tests, examples)
|
||||
@echo "$(CYAN)Running all test targets...$(NC)"
|
||||
@$(CARGO) test --all-targets --features $(FEATURES)
|
||||
|
||||
test-no-fail-fast: ## Run all tests even if one fails
|
||||
@echo "$(CYAN)Running all tests (no fail fast)...$(NC)"
|
||||
@$(CARGO) test --features $(FEATURES) -- --no-fail-fast
|
||||
|
||||
test-integration: ## Run integration tests only
|
||||
@echo "$(CYAN)Running integration tests...$(NC)"
|
||||
@$(CARGO) test --test lifecycle --test search --test mutation --test crash_recovery --test doctor_recovery --test encryption_capsule --test replay_integrity --test single_file --features $(FEATURES)
|
||||
|
||||
test-unit: ## Run unit tests only
|
||||
@echo "$(CYAN)Running unit tests...$(NC)"
|
||||
@$(CARGO) test --lib --features $(FEATURES)
|
||||
|
||||
fmt: ## Format code
|
||||
@echo "$(CYAN)Formatting code...$(NC)"
|
||||
@$(CARGO) fmt --all
|
||||
|
||||
fmt-check: ## Check code formatting
|
||||
@echo "$(CYAN)Checking code formatting...$(NC)"
|
||||
@$(CARGO) fmt --all -- --check
|
||||
|
||||
clippy: ## Run clippy linter
|
||||
@echo "$(CYAN)Running clippy...$(NC)"
|
||||
@$(CARGO) clippy --all-targets --features $(FEATURES) -- -D warnings
|
||||
|
||||
clippy-fix: ## Run clippy and auto-fix issues
|
||||
@echo "$(CYAN)Running clippy with auto-fix...$(NC)"
|
||||
@$(CARGO) clippy --fix --all-targets --features $(FEATURES) -- -D warnings
|
||||
|
||||
doc: ## Generate documentation
|
||||
@echo "$(CYAN)Generating documentation...$(NC)"
|
||||
@$(CARGO) doc --features $(FEATURES) --no-deps
|
||||
|
||||
doc-open: ## Generate and open documentation
|
||||
@echo "$(CYAN)Generating and opening documentation...$(NC)"
|
||||
@$(CARGO) doc --features $(FEATURES) --no-deps --open
|
||||
|
||||
clean: ## Clean build artifacts
|
||||
@echo "$(CYAN)Cleaning build artifacts...$(NC)"
|
||||
@$(CARGO) clean
|
||||
|
||||
clean-all: clean ## Clean everything including target directory
|
||||
@echo "$(CYAN)Cleaning all artifacts...$(NC)"
|
||||
@rm -rf target/
|
||||
|
||||
run-example-basic: ## Run basic_usage example
|
||||
@echo "$(CYAN)Running basic_usage example...$(NC)"
|
||||
@$(CARGO) run --example basic_usage --features $(FEATURES)
|
||||
|
||||
run-example-pdf: ## Run pdf_ingestion example (usage: make run-example-pdf PDF_PATH=/path/to/pdf)
|
||||
@if [ -z "$(PDF_PATH)" ]; then \
|
||||
echo "$(YELLOW)Usage: make run-example-pdf PDF_PATH=/path/to/pdf$(NC)"; \
|
||||
echo "$(YELLOW)Example: make run-example-pdf PDF_PATH=examples/1706.03762v7.pdf$(NC)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "$(CYAN)Running pdf_ingestion example with $(PDF_PATH)...$(NC)"
|
||||
@$(CARGO) run --example pdf_ingestion --features $(FEATURES) -- $(PDF_PATH)
|
||||
|
||||
run-example-clip: ## Run clip_visual_search example (usage: make run-example-clip PDF_PATH=/path/to/pdf)
|
||||
@if [ -z "$(PDF_PATH)" ]; then \
|
||||
echo "$(YELLOW)Usage: make run-example-clip PDF_PATH=/path/to/pdf$(NC)"; \
|
||||
echo "$(YELLOW)Example: make run-example-clip PDF_PATH=examples/document.pdf$(NC)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "$(CYAN)Running clip_visual_search example with $(PDF_PATH)...$(NC)"
|
||||
@$(CARGO) run --example clip_visual_search --features $(FEATURES),clip -- $(PDF_PATH)
|
||||
|
||||
run-example-whisper: ## Run test_whisper example (usage: make run-example-whisper AUDIO_PATH=/path/to/audio)
|
||||
@if [ -z "$(AUDIO_PATH)" ]; then \
|
||||
echo "$(YELLOW)Usage: make run-example-whisper AUDIO_PATH=/path/to/audio$(NC)"; \
|
||||
echo "$(YELLOW)Example: make run-example-whisper AUDIO_PATH=examples/call_sale.mp3$(NC)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "$(CYAN)Running test_whisper example with $(AUDIO_PATH)...$(NC)"
|
||||
@$(CARGO) run --example test_whisper --features $(FEATURES),whisper -- $(AUDIO_PATH)
|
||||
|
||||
lint: fmt-check clippy ## Run all linting checks
|
||||
|
||||
verify: check lint test ## Run all verification checks (check, lint, test)
|
||||
|
||||
ci: verify build-release ## Run CI pipeline (verify + release build)
|
||||
|
||||
docker-build: ## Build Docker image
|
||||
@echo "$(CYAN)Building Docker image...$(NC)"
|
||||
@cd docker/cli && docker build -t memvid/cli:latest .
|
||||
|
||||
docker-test: ## Test Docker image
|
||||
@echo "$(CYAN)Testing Docker image...$(NC)"
|
||||
@cd docker/cli && ./test.sh
|
||||
|
||||
docker-push: ## Push Docker image to registry (requires login)
|
||||
@echo "$(CYAN)Pushing Docker image...$(NC)"
|
||||
@docker push memvid/cli:latest
|
||||
|
||||
docker-tag: ## Tag Docker image with version
|
||||
@echo "$(CYAN)Tagging Docker image...$(NC)"
|
||||
@VERSION=$$(grep "^version" Cargo.toml | cut -d'"' -f2); \
|
||||
docker tag memvid/cli:latest memvid/cli:$$VERSION; \
|
||||
echo "Tagged as memvid/cli:$$VERSION"
|
||||
|
||||
bench: ## Run benchmarks (if available)
|
||||
@echo "$(CYAN)Running benchmarks...$(NC)"
|
||||
@$(CARGO) bench --features $(FEATURES) || echo "$(YELLOW)No benchmarks found$(NC)"
|
||||
|
||||
update: ## Update dependencies
|
||||
@echo "$(CYAN)Updating dependencies...$(NC)"
|
||||
@$(CARGO) update
|
||||
|
||||
audit: ## Audit dependencies for security vulnerabilities
|
||||
@echo "$(CYAN)Auditing dependencies...$(NC)"
|
||||
@$(CARGO) audit || echo "$(YELLOW)cargo-audit not installed. Install with: cargo install cargo-audit$(NC)"
|
||||
|
||||
version: ## Show version information
|
||||
@echo "$(CYAN)Version Information:$(NC)"
|
||||
@$(CARGO) --version
|
||||
@rustc --version
|
||||
@echo ""
|
||||
@echo "$(CYAN)Project version:$(NC)"
|
||||
@grep "^version" Cargo.toml
|
||||
|
||||
tree: ## Show dependency tree
|
||||
@echo "$(CYAN)Dependency tree:$(NC)"
|
||||
@$(CARGO) tree --features $(FEATURES)
|
||||
|
||||
tree-duplicates: ## Show duplicate dependencies
|
||||
@echo "$(CYAN)Duplicate dependencies:$(NC)"
|
||||
@$(CARGO) tree --duplicates --features $(FEATURES)
|
||||
|
||||
outdated: ## Check for outdated dependencies
|
||||
@echo "$(CYAN)Checking for outdated dependencies...$(NC)"
|
||||
@$(CARGO) outdated || echo "$(YELLOW)cargo-outdated not installed. Install with: cargo install cargo-outdated$(NC)"
|
||||
|
||||
bloat: ## Analyze binary size bloat
|
||||
@echo "$(CYAN)Analyzing binary size...$(NC)"
|
||||
@$(CARGO) bloat --release --features $(FEATURES) || echo "$(YELLOW)cargo-bloat not installed. Install with: cargo install cargo-bloat$(NC)"
|
||||
|
||||
package: ## Create a package for publishing
|
||||
@echo "$(CYAN)Creating package...$(NC)"
|
||||
@$(CARGO) package
|
||||
|
||||
publish-dry-run: ## Dry run of publish (check if ready)
|
||||
@echo "$(CYAN)Running publish dry-run...$(NC)"
|
||||
@$(CARGO) publish --dry-run
|
||||
|
||||
coverage: ## Generate test coverage report
|
||||
@echo "$(CYAN)Generating test coverage...$(NC)"
|
||||
@$(CARGO) tarpaulin --features $(FEATURES) --out Html || echo "$(YELLOW)cargo-tarpaulin not installed. Install with: cargo install cargo-tarpaulin$(NC)"
|
||||
|
||||
watch: ## Watch for changes and run tests
|
||||
@echo "$(CYAN)Watching for changes...$(NC)"
|
||||
@$(CARGO) watch -x "test --features $(FEATURES)" || echo "$(YELLOW)cargo-watch not installed. Install with: cargo install cargo-watch$(NC)"
|
||||
|
||||
watch-build: ## Watch for changes and rebuild
|
||||
@echo "$(CYAN)Watching for changes and rebuilding...$(NC)"
|
||||
@$(CARGO) watch -x "build --features $(FEATURES)" || echo "$(YELLOW)cargo-watch not installed. Install with: cargo install cargo-watch$(NC)"
|
||||
|
||||
pre-commit: fmt-check clippy test ## Run pre-commit checks (fmt, clippy, test)
|
||||
@@ -0,0 +1,514 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)"
|
||||
src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<div style="height: 16px;"></div>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid is a single-file memory layer for AI agents with instant retrieval and long-term memory.</strong><br/>
|
||||
Persistent, versioned, and portable memory, without databases.
|
||||
</p>
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/vHvRHhJnHC"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
|
||||
## Benchmark Highlights
|
||||
|
||||
**🚀 Higher accuracy than any other memory system :** +35% SOTA on LoCoMo, best-in-class long-horizon conversational recall & reasoning
|
||||
|
||||
**🧠 Superior multi-hop & temporal reasoning:** +76% multi-hop, +56% temporal vs. the industry average
|
||||
|
||||
**⚡ Ultra-low latency at scale** 0.025ms P50 and 0.075ms P99, with 1,372× higher throughput than standard
|
||||
|
||||
**🔬 Fully reproducible benchmarks:** LoCoMo (10 × ~26K-token conversations), open-source eval, LLM-as-Judge
|
||||
|
||||
|
||||
## What is Memvid?
|
||||
|
||||
Memvid is a portable AI memory system that packages your data, embeddings, search structure, and metadata into a single file.
|
||||
|
||||
Instead of running complex RAG pipelines or server-based vector databases, Memvid enables fast retrieval directly from the file.
|
||||
|
||||
The result is a model-agnostic, infrastructure-free memory layer that gives AI agents persistent, long-term memory they can carry anywhere.
|
||||
|
||||
|
||||
## What are Smart Frames?
|
||||
|
||||
Memvid draws inspiration from video encoding, not to store video, but to **organize AI memory as an append-only, ultra-efficient sequence of Smart Frames.**
|
||||
|
||||
A Smart Frame is an immutable unit that stores content along with timestamps, checksums and basic metadata.
|
||||
Frames are grouped in a way that allows efficient compression, indexing, and parallel reads.
|
||||
|
||||
This frame-based design enables:
|
||||
|
||||
- Append-only writes without modifying or corrupting existing data
|
||||
- Queries over past memory states
|
||||
- Timeline-style inspection of how knowledge evolves
|
||||
- Crash safety through committed, immutable frames
|
||||
- Efficient compression using techniques adapted from video encoding
|
||||
|
||||
The result is a single file that behaves like a rewindable memory timeline for AI systems.
|
||||
|
||||
|
||||
## Core Concepts
|
||||
|
||||
- **Living Memory Engine**
|
||||
Continuously append, branch, and evolve memory across sessions.
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
Self-contained, shareable memory capsules with rules and expiry.
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
Rewind, replay, or branch any memory state.
|
||||
|
||||
- **Smart Recall**
|
||||
Sub-5ms local memory access with predictive caching.
|
||||
|
||||
- **Codec Intelligence**
|
||||
Auto-selects and upgrades compression over time.
|
||||
|
||||
|
||||
## Use Cases
|
||||
|
||||
Memvid is a portable, serverless memory layer that gives AI agents persistent memory and fast recall. Because it's model-agnostic, multi-modal, and works fully offline, developers are using Memvid across a wide range of real-world applications.
|
||||
|
||||
- Long-Running AI Agents
|
||||
- Enterprise Knowledge Bases
|
||||
- Offline-First AI Systems
|
||||
- Codebase Understanding
|
||||
- Customer Support Agents
|
||||
- Workflow Automation
|
||||
- Sales and Marketing Copilots
|
||||
- Personal Knowledge Assistants
|
||||
- Medical, Legal, and Financial Agents
|
||||
- Auditable and Debuggable AI Workflows
|
||||
- Custom Applications
|
||||
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
Use Memvid in your preferred language:
|
||||
|
||||
| Package | Install | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Installation (Rust)
|
||||
|
||||
### Requirements
|
||||
|
||||
- **Rust 1.85.0+** — Install from [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Add to Your Project
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Description |
|
||||
| ------------------- | ---------------------------------------------------------------- |
|
||||
| `lex` | Full-text search with BM25 ranking (Tantivy) |
|
||||
| `pdf_extract` | Pure Rust PDF text extraction |
|
||||
| `vec` | Vector similarity search (HNSW + local text embeddings via ONNX) |
|
||||
| `clip` | CLIP visual embeddings for image search |
|
||||
| `whisper` | Audio transcription with Whisper |
|
||||
| `api_embed` | Cloud API embeddings (OpenAI) |
|
||||
| `temporal_track` | Natural language date parsing ("last Tuesday") |
|
||||
| `parallel_segments` | Multi-threaded ingestion |
|
||||
| `encryption` | Password-based encryption capsules (.mv2e) |
|
||||
| `symspell_cleanup` | Robust PDF text repair (fixes "emp lo yee" -> "employee") |
|
||||
|
||||
Enable features as needed:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
|
||||
## Quick Start
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Build
|
||||
|
||||
Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
Build in debug mode:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Build in release mode (optimized):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Build with specific features:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Run Tests
|
||||
|
||||
Run all tests:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Run tests with output:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Run a specific test:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Run integration tests only:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Examples
|
||||
|
||||
The `examples/` directory contains working examples:
|
||||
|
||||
### Basic Usage
|
||||
|
||||
Demonstrates create, put, search, and timeline operations:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDF Ingestion
|
||||
|
||||
Ingest and search PDF documents (uses the "Attention Is All You Need" paper):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP Visual Search
|
||||
|
||||
Image search using CLIP embeddings (requires `clip` feature):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Whisper Transcription
|
||||
|
||||
Audio transcription (requires `whisper` feature):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper -- /path/to/audio.mp3
|
||||
```
|
||||
|
||||
**Available Models:**
|
||||
|
||||
| Model | Size | Speed | Use Case |
|
||||
| --------------------- | ------ | ------- | ----------------------------------- |
|
||||
| `whisper-small-en` | 244 MB | Slowest | Best accuracy (default) |
|
||||
| `whisper-tiny-en` | 75 MB | Fast | Balanced |
|
||||
| `whisper-tiny-en-q8k` | 19 MB | Fastest | Quick testing, resource-constrained |
|
||||
|
||||
**Model Selection:**
|
||||
|
||||
```bash
|
||||
# Default (FP32 small, highest accuracy)
|
||||
cargo run --example test_whisper --features whisper -- audio.mp3
|
||||
|
||||
# Quantized tiny (75% smaller, faster)
|
||||
MEMVID_WHISPER_MODEL=whisper-tiny-en-q8k cargo run --example test_whisper --features whisper -- audio.mp3
|
||||
```
|
||||
|
||||
**Programmatic Configuration:**
|
||||
|
||||
```rust
|
||||
use memvid_core::{WhisperConfig, WhisperTranscriber};
|
||||
|
||||
// Default FP32 small model
|
||||
let config = WhisperConfig::default();
|
||||
|
||||
// Quantized tiny model (faster, smaller)
|
||||
let config = WhisperConfig::with_quantization();
|
||||
|
||||
// Specific model
|
||||
let config = WhisperConfig::with_model("whisper-tiny-en-q8k");
|
||||
|
||||
let transcriber = WhisperTranscriber::new(&config)?;
|
||||
let result = transcriber.transcribe_file("audio.mp3")?;
|
||||
println!("{}", result.text);
|
||||
```
|
||||
|
||||
|
||||
## Text Embedding Models
|
||||
|
||||
The `vec` feature includes local text embedding support using ONNX models. Before using local text embeddings, you need to download the model files manually.
|
||||
|
||||
### Quick Start: BGE-small (Recommended)
|
||||
|
||||
Download the default BGE-small model (384 dimensions, fast and efficient):
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# Download ONNX model
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# Download tokenizer
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### Available Models
|
||||
|
||||
| Model | Dimensions | Size | Best For |
|
||||
| ----------------------- | ---------- | ------ | --------------- |
|
||||
| `bge-small-en-v1.5` | 384 | ~120MB | Default, fast |
|
||||
| `bge-base-en-v1.5` | 768 | ~420MB | Better quality |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530MB | Versatile tasks |
|
||||
| `gte-large` | 1024 | ~1.3GB | Highest quality |
|
||||
|
||||
### Other Models
|
||||
|
||||
**BGE-base** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large** (1024 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### Usage in Code
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Use default model (BGE-small)
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// Use different model
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
See `examples/text_embedding.rs` for a complete example with similarity computation and search ranking.
|
||||
|
||||
### Model Consistency
|
||||
|
||||
To prevent accidental model mixing (e.g., querying a BGE-small index with OpenAI embeddings), you can explicitly bind your Memvid instance to a specific model name:
|
||||
|
||||
```rust
|
||||
// Bind the index to a specific model.
|
||||
// If the index was previously created with a different model, this will return an error.
|
||||
mem.set_vec_model("bge-small-en-v1.5")?;
|
||||
```
|
||||
|
||||
This binding is persistent. Once set, future attempts to use a different model name will fail fast with a `ModelMismatch` error.
|
||||
|
||||
|
||||
|
||||
## API Embeddings (OpenAI)
|
||||
|
||||
The `api_embed` feature enables cloud-based embedding generation using OpenAI's API.
|
||||
|
||||
### Setup
|
||||
|
||||
Set your OpenAI API key:
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY="sk-..."
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
```rust
|
||||
use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Use default model (text-embedding-3-small)
|
||||
let config = OpenAIConfig::default();
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 1536);
|
||||
|
||||
// Use higher quality model
|
||||
let config = OpenAIConfig::large(); // text-embedding-3-large (3072 dims)
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
### Available Models
|
||||
|
||||
| Model | Dimensions | Best For |
|
||||
| ------------------------ | ---------- | -------------------------- |
|
||||
| `text-embedding-3-small` | 1536 | Default, fastest, cheapest |
|
||||
| `text-embedding-3-large` | 3072 | Highest quality |
|
||||
| `text-embedding-ada-002` | 1536 | Legacy model |
|
||||
|
||||
See `examples/openai_embedding.rs` for a complete example.
|
||||
|
||||
|
||||
|
||||
## File Format
|
||||
|
||||
Everything lives in a single `.mv2` file:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
No `.wal`, `.lock`, `.shm`, or sidecar files. Ever.
|
||||
|
||||
See [MV2_SPEC.md](MV2_SPEC.md) for the complete file format specification.
|
||||
|
||||
|
||||
|
||||
## Support
|
||||
|
||||
Have questions or feedback?
|
||||
Email: contact@memvid.com
|
||||
|
||||
**Drop a ⭐ to show support**
|
||||
|
||||
---
|
||||
|
||||
> **Memvid v1 (QR-based memory) is deprecated**
|
||||
>
|
||||
> If you are referencing QR codes, you are using outdated information.
|
||||
>
|
||||
> See: https://docs.memvid.com/memvid-v1-deprecation
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
Apache License 2.0 — see the [LICENSE](LICENSE) file for details.
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`memvid/memvid`
|
||||
- 原始仓库：https://github.com/memvid/memvid
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| 2.x | :white_check_mark: |
|
||||
| < 2.0 | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
We take security seriously at Memvid. If you discover a security vulnerability, please report it responsibly.
|
||||
|
||||
### How to Report
|
||||
|
||||
**Please do NOT open a public GitHub issue for security vulnerabilities.**
|
||||
|
||||
Instead, email us at: **security@memvid.com**
|
||||
|
||||
Include the following in your report:
|
||||
- Description of the vulnerability
|
||||
- Steps to reproduce
|
||||
- Potential impact
|
||||
- Suggested fix (if any)
|
||||
|
||||
### What to Expect
|
||||
|
||||
- **Acknowledgment**: We will acknowledge your report within 48 hours
|
||||
- **Assessment**: We will assess the vulnerability and determine its severity
|
||||
- **Fix Timeline**: Critical vulnerabilities will be addressed within 7 days
|
||||
- **Disclosure**: We will coordinate with you on public disclosure timing
|
||||
- **Credit**: We will credit you in our security advisories (unless you prefer anonymity)
|
||||
|
||||
### Scope
|
||||
|
||||
The following are in scope:
|
||||
- Memory corruption vulnerabilities
|
||||
- Data leakage from `.mv2` files
|
||||
- Encryption bypass (when using `encryption` feature)
|
||||
- Denial of service attacks
|
||||
- Path traversal vulnerabilities
|
||||
|
||||
### Safe Harbor
|
||||
|
||||
We consider security research conducted in good faith to be authorized. We will not pursue legal action against researchers who:
|
||||
- Act in good faith
|
||||
- Avoid privacy violations
|
||||
- Do not access or modify other users' data
|
||||
- Report vulnerabilities promptly
|
||||
- Give us reasonable time to fix issues before disclosure
|
||||
|
||||
## Security Best Practices
|
||||
|
||||
When using Memvid:
|
||||
|
||||
1. **File Permissions**: Set appropriate file permissions on `.mv2` files
|
||||
2. **Encryption**: Use the `encryption` feature for sensitive data
|
||||
3. **Validation**: Validate input before ingesting into memory
|
||||
4. **Updates**: Keep Memvid updated to the latest version
|
||||
|
||||
## Security Features
|
||||
|
||||
Memvid includes several security features:
|
||||
|
||||
- **Checksums**: Blake3 checksums for data integrity
|
||||
- **Signatures**: Ed25519 signatures for authenticity
|
||||
- **Encryption**: Optional AES-256-GCM encryption (`.mv2e` capsules)
|
||||
- **Crash Safety**: WAL-based recovery prevents corruption
|
||||
@@ -0,0 +1,176 @@
|
||||
//! Search precision benchmarks for implicit AND operator change.
|
||||
//!
|
||||
//! This benchmark suite measures the performance impact of changing the implicit
|
||||
//! query operator from OR to AND. It verifies that the precision improvement
|
||||
//! (33% → 100%) comes with no query latency regression.
|
||||
//!
|
||||
//! # Benchmarks
|
||||
//!
|
||||
//! - `query_two_words`: Measures latency for simple two-word queries
|
||||
//! - `precision_calculation`: Measures precision metrics and filtering overhead
|
||||
//! - `result_count`: Measures result set size impact
|
||||
//!
|
||||
//! # Running
|
||||
//!
|
||||
//! ```bash
|
||||
//! cargo bench --bench search_precision_benchmark --features lex
|
||||
//! ```
|
||||
|
||||
use criterion::{Criterion, criterion_group, criterion_main};
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Setup test corpus
|
||||
fn setup_corpus(size: usize) -> std::path::PathBuf {
|
||||
let temp_file = std::env::temp_dir().join(format!("bench_{}.mv2", size));
|
||||
let _ = std::fs::remove_file(&temp_file);
|
||||
|
||||
let mut mem = Memvid::create(&temp_file).unwrap();
|
||||
|
||||
let topics = [
|
||||
"machine learning neural networks",
|
||||
"python programming development",
|
||||
"machine learning with python",
|
||||
"rust systems programming",
|
||||
"web development javascript",
|
||||
];
|
||||
|
||||
for i in 0..size {
|
||||
let content = format!("Document {} about {}", i, topics[i % topics.len()]);
|
||||
mem.put_bytes_with_options(
|
||||
content.as_bytes(),
|
||||
PutOptions::builder().title(format!("Doc {}", i)).build(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
if (i + 1) % 100 == 0 {
|
||||
mem.commit().unwrap();
|
||||
}
|
||||
}
|
||||
mem.commit().unwrap();
|
||||
temp_file
|
||||
}
|
||||
|
||||
fn bench_query_latency(c: &mut Criterion) {
|
||||
let corpus_path = setup_corpus(1000);
|
||||
|
||||
c.bench_function("query_two_words", |b| {
|
||||
b.iter_custom(|iters| {
|
||||
let mut total = std::time::Duration::ZERO;
|
||||
for _ in 0..iters {
|
||||
let mut mem = Memvid::open(&corpus_path).unwrap(); // FIX: mut
|
||||
let start = Instant::now();
|
||||
let _results = mem
|
||||
.search(SearchRequest {
|
||||
query: "machine learning".to_string(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})
|
||||
.unwrap();
|
||||
total += start.elapsed();
|
||||
}
|
||||
total
|
||||
});
|
||||
});
|
||||
|
||||
std::fs::remove_file(&corpus_path).ok();
|
||||
}
|
||||
|
||||
fn bench_precision(c: &mut Criterion) {
|
||||
let corpus_path = setup_corpus(1000);
|
||||
|
||||
c.bench_function("precision_calculation", |b| {
|
||||
b.iter_custom(|iters| {
|
||||
let mut total = std::time::Duration::ZERO;
|
||||
for _ in 0..iters {
|
||||
let mut mem = Memvid::open(&corpus_path).unwrap(); // FIX: mut
|
||||
let start = Instant::now();
|
||||
let results = mem
|
||||
.search(SearchRequest {
|
||||
query: "machine python".to_string(),
|
||||
top_k: 100,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let _relevant = results
|
||||
.hits
|
||||
.iter()
|
||||
.filter(|hit| {
|
||||
let text = hit.text.to_lowercase();
|
||||
text.contains("machine") && text.contains("python")
|
||||
})
|
||||
.count();
|
||||
|
||||
total += start.elapsed();
|
||||
}
|
||||
total
|
||||
});
|
||||
});
|
||||
|
||||
std::fs::remove_file(&corpus_path).ok();
|
||||
}
|
||||
|
||||
fn bench_result_count(c: &mut Criterion) {
|
||||
let corpus_path = setup_corpus(1000);
|
||||
|
||||
c.bench_function("result_count", |b| {
|
||||
b.iter_custom(|iters| {
|
||||
let mut total = std::time::Duration::ZERO;
|
||||
for _ in 0..iters {
|
||||
let mut mem = Memvid::open(&corpus_path).unwrap(); // FIX: mut
|
||||
let start = Instant::now();
|
||||
let results = mem
|
||||
.search(SearchRequest {
|
||||
query: "machine learning".to_string(),
|
||||
top_k: 100,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})
|
||||
.unwrap();
|
||||
let _count = results.hits.len();
|
||||
total += start.elapsed();
|
||||
}
|
||||
total
|
||||
});
|
||||
});
|
||||
|
||||
std::fs::remove_file(&corpus_path).ok();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_query_latency,
|
||||
bench_precision,
|
||||
bench_result_count
|
||||
);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,145 @@
|
||||
use criterion::{Criterion, black_box, criterion_group, criterion_main};
|
||||
use memvid_core::types::FrameId;
|
||||
use memvid_core::vec::{VecDocument, VecIndex, VecIndexBuilder};
|
||||
|
||||
fn generate_vectors(count: usize, dim: usize) -> Vec<Vec<f32>> {
|
||||
let mut vectors = Vec::with_capacity(count);
|
||||
for _ in 0..count {
|
||||
let mut vec = Vec::with_capacity(dim);
|
||||
for _ in 0..dim {
|
||||
vec.push(fastrand::f32());
|
||||
}
|
||||
vectors.push(vec);
|
||||
}
|
||||
vectors
|
||||
}
|
||||
|
||||
fn bench_search_10k(c: &mut Criterion) {
|
||||
let count = 10_000;
|
||||
let dim = 128; // Smaller dimension for faster setup in benchmarks
|
||||
let vectors = generate_vectors(count, dim);
|
||||
let query = generate_vectors(1, dim).pop().unwrap();
|
||||
|
||||
// Build HNSW Index (via Builder which triggers HNSW for > 1000)
|
||||
let mut builder = VecIndexBuilder::new();
|
||||
for (i, vec) in vectors.iter().enumerate() {
|
||||
builder.add_document(i as FrameId, vec.clone());
|
||||
}
|
||||
let artifact = builder.finish().expect("finish hnsw");
|
||||
let hnsw_index = VecIndex::decode(&artifact.bytes).expect("decode hnsw");
|
||||
|
||||
// Build Brute Force Index (Force Uncompressed)
|
||||
let documents: Vec<VecDocument> = vectors
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, vec)| VecDocument {
|
||||
frame_id: i as FrameId,
|
||||
embedding: vec.clone(),
|
||||
})
|
||||
.collect();
|
||||
let brute_index = VecIndex::Uncompressed { documents };
|
||||
|
||||
let mut group = c.benchmark_group("search_10k");
|
||||
|
||||
group.bench_function("hnsw", |b| {
|
||||
b.iter(|| {
|
||||
hnsw_index.search(black_box(&query), black_box(10));
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("brute_force", |b| {
|
||||
b.iter(|| {
|
||||
brute_index.search(black_box(&query), black_box(10));
|
||||
})
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_search_50k(c: &mut Criterion) {
|
||||
let count = 50_000;
|
||||
let dim = 128;
|
||||
let vectors = generate_vectors(count, dim);
|
||||
let query = generate_vectors(1, dim).pop().unwrap();
|
||||
|
||||
let mut builder = VecIndexBuilder::new();
|
||||
for (i, vec) in vectors.iter().enumerate() {
|
||||
builder.add_document(i as FrameId, vec.clone());
|
||||
}
|
||||
let artifact = builder.finish().expect("finish hnsw");
|
||||
let hnsw_index = VecIndex::decode(&artifact.bytes).expect("decode hnsw");
|
||||
|
||||
let documents: Vec<VecDocument> = vectors
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, vec)| VecDocument {
|
||||
frame_id: i as FrameId,
|
||||
embedding: vec.clone(),
|
||||
})
|
||||
.collect();
|
||||
let brute_index = VecIndex::Uncompressed { documents };
|
||||
|
||||
let mut group = c.benchmark_group("search_50k");
|
||||
|
||||
group.bench_function("hnsw", |b| {
|
||||
b.iter(|| {
|
||||
hnsw_index.search(black_box(&query), black_box(10));
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("brute_force", |b| {
|
||||
b.iter(|| {
|
||||
brute_index.search(black_box(&query), black_box(10));
|
||||
})
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_search_100k(c: &mut Criterion) {
|
||||
let count = 100_000;
|
||||
let dim = 128;
|
||||
let vectors = generate_vectors(count, dim);
|
||||
let query = generate_vectors(1, dim).pop().unwrap();
|
||||
|
||||
let mut builder = VecIndexBuilder::new();
|
||||
for (i, vec) in vectors.iter().enumerate() {
|
||||
builder.add_document(i as FrameId, vec.clone());
|
||||
}
|
||||
let artifact = builder.finish().expect("finish hnsw");
|
||||
let hnsw_index = VecIndex::decode(&artifact.bytes).expect("decode hnsw");
|
||||
|
||||
let documents: Vec<VecDocument> = vectors
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, vec)| VecDocument {
|
||||
frame_id: i as FrameId,
|
||||
embedding: vec.clone(),
|
||||
})
|
||||
.collect();
|
||||
let brute_index = VecIndex::Uncompressed { documents };
|
||||
|
||||
let mut group = c.benchmark_group("search_100k");
|
||||
|
||||
group.bench_function("hnsw", |b| {
|
||||
b.iter(|| {
|
||||
let _ = hnsw_index.search(black_box(&query), black_box(10));
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("brute_force", |b| {
|
||||
b.iter(|| {
|
||||
let _ = brute_index.search(black_box(&query), black_box(10));
|
||||
})
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_search_10k,
|
||||
bench_search_50k,
|
||||
bench_search_100k
|
||||
);
|
||||
criterion_main!(benches);
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,82 @@
|
||||
# Docker Images for Memvid
|
||||
|
||||
This directory contains Docker configurations for Memvid components.
|
||||
|
||||
## Available Images
|
||||
|
||||
### Memvid Core (`core/`)
|
||||
|
||||
The Memvid Core Docker images provide containerized Rust development, testing, and build environments for the `memvid-core` library.
|
||||
|
||||
**Quick Start:**
|
||||
```bash
|
||||
# Development environment
|
||||
cd core
|
||||
docker-compose up -d dev
|
||||
docker-compose exec dev bash
|
||||
|
||||
# Run tests
|
||||
docker-compose run --rm test
|
||||
|
||||
# Build release
|
||||
docker-compose run --rm build
|
||||
```
|
||||
|
||||
For detailed usage, see [core/README.md](core/README.md).
|
||||
|
||||
### Memvid CLI (`cli/`)
|
||||
|
||||
The Memvid CLI Docker image provides a containerized version of the `memvid-cli` tool, allowing you to run Memvid commands without installing Node.js or dealing with platform-specific binaries.
|
||||
|
||||
**Quick Start:**
|
||||
|
||||
```bash
|
||||
# Pull the image
|
||||
docker pull memvid/cli
|
||||
|
||||
# Create a memory
|
||||
docker run --rm -v $(pwd):/data memvid/cli create my-memory.mv2
|
||||
|
||||
# Add documents
|
||||
docker run --rm -v $(pwd):/data memvid/cli put my-memory.mv2 --input doc.pdf
|
||||
|
||||
# Search
|
||||
docker run --rm -v $(pwd):/data memvid/cli find my-memory.mv2 --query "search"
|
||||
```
|
||||
|
||||
For detailed usage instructions, examples, and Docker Compose configurations, see [cli/README.md](cli/README.md).
|
||||
|
||||
## Building Images
|
||||
|
||||
### Build CLI Image Locally
|
||||
|
||||
```bash
|
||||
cd cli
|
||||
docker build -t memvid/cli:test .
|
||||
```
|
||||
|
||||
## Publishing
|
||||
|
||||
Docker images are automatically built and published to Docker Hub via GitHub Actions when tags are pushed. See `.github/workflows/docker-release.yml` for the CI/CD configuration.
|
||||
|
||||
**Image Registry:**
|
||||
- Docker Hub: `memvid/cli`
|
||||
- Tags: `latest`, `2.0.129`, and version-specific tags
|
||||
|
||||
## Architecture Support
|
||||
|
||||
The CLI image supports multi-architecture builds:
|
||||
- `linux/amd64`
|
||||
- `linux/arm64`
|
||||
|
||||
## Security
|
||||
|
||||
The CLI image runs as a non-root user (`memvid`) for improved security. When mounting volumes, ensure your host directories have appropriate permissions.
|
||||
|
||||
## Links
|
||||
|
||||
- [Core Documentation](core/README.md)
|
||||
- [CLI Documentation](cli/README.md)
|
||||
- [CLI Testing Guide](cli/TESTING.md)
|
||||
- [Main Project README](../README.md)
|
||||
- [Memvid Documentation](https://docs.memvid.com)
|
||||
@@ -0,0 +1,7 @@
|
||||
node_modules
|
||||
npm-debug.log
|
||||
.git
|
||||
.gitignore
|
||||
*.md
|
||||
.env
|
||||
.DS_Store
|
||||
@@ -0,0 +1,33 @@
|
||||
# Memvid CLI Docker Image
|
||||
FROM ubuntu:24.04
|
||||
|
||||
LABEL org.opencontainers.image.title="Memvid CLI"
|
||||
LABEL org.opencontainers.image.description="AI memory CLI with semantic search"
|
||||
LABEL org.opencontainers.image.url="https://memvid.com"
|
||||
LABEL org.opencontainers.image.source="https://github.com/memvid/memvid"
|
||||
LABEL org.opencontainers.image.vendor="Memvid"
|
||||
LABEL org.opencontainers.image.version="2.0.131"
|
||||
|
||||
# Install system deps + Node.js 24
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
libssl3 \
|
||||
&& curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
|
||||
&& apt-get install -y --no-install-recommends nodejs \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install memvid CLI (pinned version)
|
||||
RUN npm install -g memvid-cli@latest \
|
||||
&& npm cache clean --force
|
||||
|
||||
# Create non-root user (recommended)
|
||||
RUN useradd -m memvid
|
||||
USER memvid
|
||||
|
||||
# Working directory for memory files
|
||||
WORKDIR /data
|
||||
|
||||
# Entrypoint
|
||||
ENTRYPOINT ["memvid"]
|
||||
CMD ["--help"]
|
||||
@@ -0,0 +1,101 @@
|
||||
# Memvid CLI Docker Image
|
||||
|
||||
AI memory CLI with crash-safe, single-file storage and semantic search.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Pull the image
|
||||
docker pull memvid/cli
|
||||
|
||||
# Create a memory
|
||||
docker run --rm -v $(pwd):/data memvid/cli create my-memory.mv2
|
||||
|
||||
# Add documents
|
||||
docker run --rm -v $(pwd):/data memvid/cli put my-memory.mv2 --input doc.pdf
|
||||
|
||||
# Search
|
||||
docker run --rm -v $(pwd):/data memvid/cli find my-memory.mv2 --query "search"
|
||||
```
|
||||
|
||||
## Basic Commands
|
||||
|
||||
```bash
|
||||
# Show help
|
||||
docker run --rm memvid/cli
|
||||
|
||||
# Show version
|
||||
docker run --rm memvid/cli --version
|
||||
|
||||
# Create a memory file (mount local directory)
|
||||
docker run --rm -v $(pwd):/data memvid/cli create my-memory.mv2
|
||||
|
||||
# Ingest a document
|
||||
docker run --rm -v $(pwd):/data memvid/cli put my-memory.mv2 --input document.pdf
|
||||
|
||||
# Search the memory
|
||||
docker run --rm -v $(pwd):/data memvid/cli find my-memory.mv2 --query "search term"
|
||||
|
||||
# Ask questions (requires API key for LLM)
|
||||
docker run --rm -v $(pwd):/data \
|
||||
-e OPENAI_API_KEY="sk-..." \
|
||||
memvid/cli ask my-memory.mv2 "What is this about?" -m openai
|
||||
|
||||
# View stats
|
||||
docker run --rm -v $(pwd):/data memvid/cli stats my-memory.mv2
|
||||
```
|
||||
|
||||
## With API Keys
|
||||
|
||||
```bash
|
||||
# Pass Memvid API key for cloud features
|
||||
docker run --rm -v $(pwd):/data \
|
||||
-e MEMVID_API_KEY="mv2_..." \
|
||||
-e OPENAI_API_KEY="sk-..." \
|
||||
memvid/cli ask my-memory.mv2 "your question"
|
||||
```
|
||||
|
||||
## Shell Alias (Recommended)
|
||||
|
||||
Add to `~/.bashrc` or `~/.zshrc`:
|
||||
|
||||
```bash
|
||||
alias memvid='docker run --rm -v $(pwd):/data -e MEMVID_API_KEY -e OPENAI_API_KEY memvid/cli'
|
||||
```
|
||||
|
||||
Then use normally:
|
||||
|
||||
```bash
|
||||
memvid create my-memory.mv2
|
||||
memvid put my-memory.mv2 --input docs/
|
||||
memvid find my-memory.mv2 --query "hello"
|
||||
```
|
||||
|
||||
## Docker Compose Example
|
||||
|
||||
```yaml
|
||||
version: '3.8'
|
||||
services:
|
||||
memvid:
|
||||
image: memvid/cli:latest
|
||||
volumes:
|
||||
- ./data:/data
|
||||
environment:
|
||||
- MEMVID_API_KEY=${MEMVID_API_KEY}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||
entrypoint: ["memvid"]
|
||||
command: ["stats", "my-memory.mv2"]
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
- Single-file `.mv2` storage
|
||||
- Semantic + lexical search
|
||||
- RAG question answering
|
||||
- PDF, DOCX, images, audio support
|
||||
|
||||
## Links
|
||||
|
||||
- Website: https://memvid.com
|
||||
- Docs: https://docs.memvid.com
|
||||
- GitHub: https://github.com/memvid/memvid
|
||||
@@ -0,0 +1,108 @@
|
||||
# Testing Memvid CLI Docker Image
|
||||
|
||||
## Quick Test
|
||||
|
||||
### 1. Build the Image
|
||||
|
||||
```bash
|
||||
cd docker/cli
|
||||
docker build -t memvid/cli:test .
|
||||
```
|
||||
|
||||
### 2. Test Basic Commands
|
||||
|
||||
```bash
|
||||
# Test help command
|
||||
docker run --rm memvid/cli:test --help
|
||||
|
||||
# Test version (if available)
|
||||
docker run --rm memvid/cli:test --version
|
||||
```
|
||||
|
||||
### 3. Test with Volume Mount
|
||||
|
||||
```bash
|
||||
# Create a test directory
|
||||
mkdir -p /tmp/memvid-test
|
||||
cd /tmp/memvid-test
|
||||
|
||||
# Create a test document
|
||||
echo "This is a test document about AI and machine learning." > test.txt
|
||||
|
||||
# Create a memory file
|
||||
docker run --rm \
|
||||
-v $(pwd):/data \
|
||||
memvid/cli:test create test-memory.mv2
|
||||
|
||||
# Add the document
|
||||
docker run --rm \
|
||||
-v $(pwd):/data \
|
||||
memvid/cli:test put test-memory.mv2 --input test.txt
|
||||
|
||||
# Search the memory
|
||||
docker run --rm \
|
||||
-v $(pwd):/data \
|
||||
memvid/cli:test find test-memory.mv2 --query "AI"
|
||||
|
||||
# View stats
|
||||
docker run --rm \
|
||||
-v $(pwd):/data \
|
||||
memvid/cli:test stats test-memory.mv2
|
||||
```
|
||||
|
||||
### 4. Test with API Keys (if needed)
|
||||
|
||||
```bash
|
||||
docker run --rm \
|
||||
-v $(pwd):/data \
|
||||
-e OPENAI_API_KEY="sk-..." \
|
||||
memvid/cli:test ask test-memory.mv2 "What is this about?" -m openai
|
||||
```
|
||||
|
||||
## Automated Testing
|
||||
|
||||
Run the test script:
|
||||
|
||||
```bash
|
||||
cd docker/cli
|
||||
./test.sh
|
||||
```
|
||||
|
||||
## Multi-Architecture Testing
|
||||
|
||||
To test multi-arch builds locally (requires buildx):
|
||||
|
||||
```bash
|
||||
# Create builder
|
||||
docker buildx create --name memvid-builder --use
|
||||
|
||||
# Build for specific platform
|
||||
docker buildx build \
|
||||
--platform linux/amd64 \
|
||||
--tag memvid/cli:test-amd64 \
|
||||
--load \
|
||||
.
|
||||
|
||||
# Test the amd64 image
|
||||
docker run --rm memvid/cli:test-amd64 --help
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Image not found
|
||||
Make sure you've built the image:
|
||||
```bash
|
||||
docker build -t memvid/cli:test docker/cli
|
||||
```
|
||||
|
||||
### Permission errors
|
||||
Ensure Docker has permission to access the mounted directory:
|
||||
```bash
|
||||
chmod 755 /path/to/your/directory
|
||||
```
|
||||
|
||||
### CLI not found
|
||||
Verify the npm package exists:
|
||||
```bash
|
||||
docker run --rm memvid/cli:test which memvid
|
||||
```
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
# Test script for Memvid CLI Docker image
|
||||
|
||||
set -e
|
||||
|
||||
IMAGE_NAME="memvid/cli:test"
|
||||
DOCKERFILE_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
echo "🐳 Building Docker image..."
|
||||
docker build -t "$IMAGE_NAME" "$DOCKERFILE_DIR"
|
||||
|
||||
echo ""
|
||||
echo "✅ Build complete! Testing basic commands..."
|
||||
echo ""
|
||||
|
||||
echo "1️⃣ Testing help command..."
|
||||
docker run --rm "$IMAGE_NAME" --help | head -5
|
||||
|
||||
echo ""
|
||||
echo "2️⃣ Testing version command..."
|
||||
docker run --rm "$IMAGE_NAME" --version || echo "Version command not available"
|
||||
|
||||
echo ""
|
||||
echo "3️⃣ Testing with volume mount (create test memory)..."
|
||||
TEST_DIR=$(mktemp -d)
|
||||
cd "$TEST_DIR"
|
||||
|
||||
# Create a test document
|
||||
echo "This is a test document about artificial intelligence and machine learning." > test.txt
|
||||
|
||||
docker run --rm \
|
||||
-v "$TEST_DIR":/data \
|
||||
"$IMAGE_NAME" create test-memory.mv2 || echo "Create command may require additional setup"
|
||||
|
||||
echo ""
|
||||
echo "🧹 Cleaning up test directory..."
|
||||
rm -rf "$TEST_DIR"
|
||||
|
||||
echo ""
|
||||
echo "✅ Basic tests complete!"
|
||||
echo ""
|
||||
echo "To test manually, run:"
|
||||
echo " docker run --rm $IMAGE_NAME --help"
|
||||
echo " docker run --rm -v \$(pwd):/data $IMAGE_NAME create my-memory.mv2"
|
||||
@@ -0,0 +1,215 @@
|
||||
# Docker Setup for Memvid Core
|
||||
|
||||
This document describes how to use Docker with the Memvid Core Rust library.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Development Environment
|
||||
|
||||
```bash
|
||||
# From project root or docker/core directory
|
||||
cd docker/core
|
||||
docker-compose up -d dev
|
||||
|
||||
# Enter the container
|
||||
docker-compose exec dev bash
|
||||
|
||||
# Inside container, you can run:
|
||||
cargo build
|
||||
cargo test
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Run Tests
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
cd docker/core
|
||||
docker-compose run --rm test
|
||||
|
||||
# Or build test image manually (from project root)
|
||||
docker build -f docker/core/Dockerfile.test -t memvid-test .
|
||||
docker run --rm memvid-test
|
||||
```
|
||||
|
||||
### Build Release
|
||||
|
||||
```bash
|
||||
# Build release version
|
||||
cd docker/core
|
||||
docker-compose run --rm build
|
||||
|
||||
# Or build manually (from project root)
|
||||
docker build -f docker/core/Dockerfile -t memvid-core:latest .
|
||||
```
|
||||
|
||||
## Docker Images
|
||||
|
||||
### 1. Development Image (`Dockerfile.dev`)
|
||||
|
||||
Full development environment with all tools:
|
||||
|
||||
```bash
|
||||
# From project root
|
||||
docker build -f docker/core/Dockerfile.dev -t memvid-dev .
|
||||
docker run -it --rm -v $(pwd):/app memvid-dev bash
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Rust toolchain 1.92
|
||||
- All build dependencies
|
||||
- Cargo watch (optional)
|
||||
- Volume mounting for live development
|
||||
|
||||
### 2. Test Image (`Dockerfile.test`)
|
||||
|
||||
Optimized for running tests:
|
||||
|
||||
```bash
|
||||
# From project root
|
||||
docker build -f docker/core/Dockerfile.test -t memvid-test .
|
||||
docker run --rm memvid-test
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Rust toolchain 1.92
|
||||
- Test dependencies
|
||||
- Runs tests automatically
|
||||
|
||||
### 3. Production Build (`Dockerfile`)
|
||||
|
||||
Multi-stage build for optimized production image:
|
||||
|
||||
```bash
|
||||
# From project root
|
||||
docker build -f docker/core/Dockerfile -t memvid-core:latest .
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Multi-stage build (smaller final image)
|
||||
- Only runtime dependencies
|
||||
- Optimized release build
|
||||
|
||||
## Docker Compose
|
||||
|
||||
### Services
|
||||
|
||||
- **`dev`** - Development environment with live code mounting
|
||||
- **`test`** - Test runner
|
||||
- **`build`** - Release builder
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
# Start development environment
|
||||
docker-compose up -d dev
|
||||
|
||||
# Run tests
|
||||
docker-compose run --rm test
|
||||
|
||||
# Build release
|
||||
docker-compose run --rm build
|
||||
|
||||
# Stop all services
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Run Examples in Docker
|
||||
|
||||
```bash
|
||||
# Development container
|
||||
docker-compose exec dev cargo run --example basic_usage
|
||||
|
||||
# With features
|
||||
docker-compose exec dev cargo run --example pdf_ingestion --features lex,pdf_extract
|
||||
```
|
||||
|
||||
### Memory-Constrained Testing
|
||||
|
||||
Test OOM prevention with memory limits:
|
||||
|
||||
```bash
|
||||
# Test with memory limit (for OOM testing)
|
||||
docker run --rm --memory=150m --memory-swap=150m \
|
||||
-v $(pwd):/app \
|
||||
memvid-test cargo test --features encryption --test encryption_capsule
|
||||
```
|
||||
|
||||
### Build with Specific Features
|
||||
|
||||
```bash
|
||||
# Build with all features
|
||||
docker-compose exec dev cargo build --release --all-features
|
||||
|
||||
# Build with specific features
|
||||
docker-compose exec dev cargo build --release --features lex,vec,encryption
|
||||
```
|
||||
|
||||
## Volume Mounts
|
||||
|
||||
The docker-compose setup uses volumes for:
|
||||
- **Source code** - Live mounting for development
|
||||
- **Cargo cache** - Speeds up builds
|
||||
- **Target cache** - Preserves build artifacts
|
||||
|
||||
## CI/CD Integration
|
||||
|
||||
### GitHub Actions Example
|
||||
|
||||
```yaml
|
||||
name: Docker Build
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build Docker image
|
||||
run: docker build -t memvid-core:test .
|
||||
- name: Run tests
|
||||
run: docker run --rm memvid-core:test cargo test
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Build Fails
|
||||
|
||||
```bash
|
||||
# Clean build
|
||||
docker-compose down -v
|
||||
docker-compose build --no-cache
|
||||
```
|
||||
|
||||
### Permission Issues
|
||||
|
||||
```bash
|
||||
# Fix permissions
|
||||
sudo chown -R $USER:$USER .
|
||||
```
|
||||
|
||||
### Out of Memory
|
||||
|
||||
```bash
|
||||
# Increase Docker memory limit in Docker Desktop settings
|
||||
# Or use memory limits in docker run:
|
||||
docker run --memory=2g --memory-swap=2g ...
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Use docker-compose** for development
|
||||
2. **Cache volumes** for faster builds
|
||||
3. **Multi-stage builds** for production
|
||||
4. **Test in containers** to match CI/CD environment
|
||||
5. **Use .dockerignore** to exclude unnecessary files
|
||||
|
||||
## Related
|
||||
|
||||
- [CLI Docker Setup](../cli/README.md)
|
||||
- [Docker Overview](../README.md)
|
||||
- [Main Project README](../../README.md)
|
||||
- [Contributing Guide](../../CONTRIBUTING.md)
|
||||
@@ -0,0 +1,53 @@
|
||||
# Multi-stage Dockerfile for Memvid Core (Rust Library)
|
||||
# Build stage
|
||||
FROM rust:1.92-slim-trixie AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy dependency files first for better caching
|
||||
COPY Cargo.toml Cargo.lock ./
|
||||
COPY rust-toolchain.toml ./
|
||||
|
||||
# Create a dummy src/lib.rs to build dependencies
|
||||
RUN mkdir -p src && \
|
||||
echo "// Dummy file for dependency building" > src/lib.rs && \
|
||||
cargo build --release --features lex,pdf_extract && \
|
||||
rm -rf src
|
||||
|
||||
# Copy source code
|
||||
COPY src ./src
|
||||
COPY examples ./examples
|
||||
COPY tests ./tests
|
||||
|
||||
# Build the library with default features
|
||||
RUN cargo build --release --features lex,pdf_extract
|
||||
|
||||
# Runtime stage - minimal image for running examples/tests
|
||||
FROM debian:trixie-slim AS runtime
|
||||
|
||||
# Install runtime dependencies
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
libssl3 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy the built artifacts from builder
|
||||
COPY --from=builder /app/target/release/examples ./examples
|
||||
COPY --from=builder /app/target/release/deps ./deps
|
||||
COPY --from=builder /app/target/release/libmemvid_core*.rlib ./lib/
|
||||
|
||||
# Set environment variables
|
||||
ENV RUST_LOG=info
|
||||
ENV PATH="/app/examples:${PATH}"
|
||||
|
||||
# Default command
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# Development Dockerfile for Memvid Core
|
||||
FROM rust:1.92-slim-trixie AS builder
|
||||
|
||||
# Install development dependencies
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
ca-certificates \
|
||||
git \
|
||||
curl \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy dependency files
|
||||
COPY Cargo.toml Cargo.lock ./
|
||||
COPY rust-toolchain.toml ./
|
||||
|
||||
# Install cargo-watch for development (optional)
|
||||
RUN cargo install cargo-watch --locked || true
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Set environment variables
|
||||
ENV RUST_LOG=debug
|
||||
ENV CARGO_TARGET_DIR=/app/target
|
||||
|
||||
# Default command (can be overridden)
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,23 @@
|
||||
# Test Dockerfile for Memvid Core
|
||||
FROM rust:1.92-slim-trixie AS builder
|
||||
|
||||
# Install test dependencies
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
ca-certificates \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy everything
|
||||
COPY . .
|
||||
|
||||
# Set environment variables for testing
|
||||
ENV RUST_BACKTRACE=1
|
||||
ENV RUST_LOG=info
|
||||
ENV CARGO_TARGET_DIR=/app/target
|
||||
|
||||
# Run tests with default features
|
||||
CMD ["cargo", "test", "--features", "lex,pdf_extract", "--", "--nocapture"]
|
||||
@@ -0,0 +1,215 @@
|
||||
# Docker Setup for Memvid Core
|
||||
|
||||
This document describes how to use Docker with the Memvid Core Rust library.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Development Environment
|
||||
|
||||
```bash
|
||||
# From project root or docker/core directory
|
||||
cd docker/core
|
||||
docker-compose up -d dev
|
||||
|
||||
# Enter the container
|
||||
docker-compose exec dev bash
|
||||
|
||||
# Inside container, you can run:
|
||||
cargo build
|
||||
cargo test
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Run Tests
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
cd docker/core
|
||||
docker-compose run --rm test
|
||||
|
||||
# Or build test image manually (from project root)
|
||||
docker build -f docker/core/Dockerfile.test -t memvid-test .
|
||||
docker run --rm memvid-test
|
||||
```
|
||||
|
||||
### Build Release
|
||||
|
||||
```bash
|
||||
# Build release version
|
||||
cd docker/core
|
||||
docker-compose run --rm build
|
||||
|
||||
# Or build manually (from project root)
|
||||
docker build -f docker/core/Dockerfile -t memvid-core:latest .
|
||||
```
|
||||
|
||||
## Docker Images
|
||||
|
||||
### 1. Development Image (`Dockerfile.dev`)
|
||||
|
||||
Full development environment with all tools:
|
||||
|
||||
```bash
|
||||
# From project root
|
||||
docker build -f docker/core/Dockerfile.dev -t memvid-dev .
|
||||
docker run -it --rm -v $(pwd):/app memvid-dev bash
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Rust toolchain 1.92
|
||||
- All build dependencies
|
||||
- Cargo watch (optional)
|
||||
- Volume mounting for live development
|
||||
|
||||
### 2. Test Image (`Dockerfile.test`)
|
||||
|
||||
Optimized for running tests:
|
||||
|
||||
```bash
|
||||
# From project root
|
||||
docker build -f docker/core/Dockerfile.test -t memvid-test .
|
||||
docker run --rm memvid-test
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Rust toolchain 1.92
|
||||
- Test dependencies
|
||||
- Runs tests automatically
|
||||
|
||||
### 3. Production Build (`Dockerfile`)
|
||||
|
||||
Multi-stage build for optimized production image:
|
||||
|
||||
```bash
|
||||
# From project root
|
||||
docker build -f docker/core/Dockerfile -t memvid-core:latest .
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Multi-stage build (smaller final image)
|
||||
- Only runtime dependencies
|
||||
- Optimized release build
|
||||
|
||||
## Docker Compose
|
||||
|
||||
### Services
|
||||
|
||||
- **`dev`** - Development environment with live code mounting
|
||||
- **`test`** - Test runner
|
||||
- **`build`** - Release builder
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
# Start development environment
|
||||
docker-compose up -d dev
|
||||
|
||||
# Run tests
|
||||
docker-compose run --rm test
|
||||
|
||||
# Build release
|
||||
docker-compose run --rm build
|
||||
|
||||
# Stop all services
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Run Examples in Docker
|
||||
|
||||
```bash
|
||||
# Development container
|
||||
docker-compose exec dev cargo run --example basic_usage
|
||||
|
||||
# With features
|
||||
docker-compose exec dev cargo run --example pdf_ingestion --features lex,pdf_extract
|
||||
```
|
||||
|
||||
### Memory-Constrained Testing
|
||||
|
||||
Test OOM prevention with memory limits:
|
||||
|
||||
```bash
|
||||
# Test with memory limit (for OOM testing)
|
||||
docker run --rm --memory=150m --memory-swap=150m \
|
||||
-v $(pwd):/app \
|
||||
memvid-test cargo test --features encryption --test encryption_capsule
|
||||
```
|
||||
|
||||
### Build with Specific Features
|
||||
|
||||
```bash
|
||||
# Build with all features
|
||||
docker-compose exec dev cargo build --release --all-features
|
||||
|
||||
# Build with specific features
|
||||
docker-compose exec dev cargo build --release --features lex,vec,encryption
|
||||
```
|
||||
|
||||
## Volume Mounts
|
||||
|
||||
The docker-compose setup uses volumes for:
|
||||
- **Source code** - Live mounting for development
|
||||
- **Cargo cache** - Speeds up builds
|
||||
- **Target cache** - Preserves build artifacts
|
||||
|
||||
## CI/CD Integration
|
||||
|
||||
### GitHub Actions Example
|
||||
|
||||
```yaml
|
||||
name: Docker Build
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build Docker image
|
||||
run: docker build -t memvid-core:test .
|
||||
- name: Run tests
|
||||
run: docker run --rm memvid-core:test cargo test
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Build Fails
|
||||
|
||||
```bash
|
||||
# Clean build
|
||||
docker-compose down -v
|
||||
docker-compose build --no-cache
|
||||
```
|
||||
|
||||
### Permission Issues
|
||||
|
||||
```bash
|
||||
# Fix permissions
|
||||
sudo chown -R $USER:$USER .
|
||||
```
|
||||
|
||||
### Out of Memory
|
||||
|
||||
```bash
|
||||
# Increase Docker memory limit in Docker Desktop settings
|
||||
# Or use memory limits in docker run:
|
||||
docker run --memory=2g --memory-swap=2g ...
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Use docker-compose** for development
|
||||
2. **Cache volumes** for faster builds
|
||||
3. **Multi-stage builds** for production
|
||||
4. **Test in containers** to match CI/CD environment
|
||||
5. **Use .dockerignore** to exclude unnecessary files
|
||||
|
||||
## Related
|
||||
|
||||
- [CLI Docker Setup](../cli/README.md)
|
||||
- [Docker Overview](../README.md)
|
||||
- [Main Project README](../../README.md)
|
||||
- [Contributing Guide](../../CONTRIBUTING.md)
|
||||
@@ -0,0 +1,46 @@
|
||||
services:
|
||||
# Development environment
|
||||
dev:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: docker/core/Dockerfile.dev
|
||||
volumes:
|
||||
- ../../:/app
|
||||
- cargo-cache:/usr/local/cargo/registry
|
||||
- target-cache:/app/target
|
||||
environment:
|
||||
- RUST_LOG=debug
|
||||
- CARGO_TARGET_DIR=/app/target
|
||||
stdin_open: true
|
||||
tty: true
|
||||
command: /bin/bash
|
||||
|
||||
# Test runner
|
||||
test:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: docker/core/Dockerfile.test
|
||||
volumes:
|
||||
- ../../:/app
|
||||
- cargo-cache:/usr/local/cargo/registry
|
||||
- target-cache:/app/target
|
||||
environment:
|
||||
- RUST_BACKTRACE=1
|
||||
- RUST_LOG=info
|
||||
command: cargo test --features lex,pdf_extract -- --nocapture
|
||||
|
||||
# Build release
|
||||
build:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: docker/core/Dockerfile
|
||||
target: builder
|
||||
volumes:
|
||||
- ../../:/app
|
||||
- cargo-cache:/usr/local/cargo/registry
|
||||
- target-cache:/app/target
|
||||
command: cargo build --release --features lex,pdf_extract
|
||||
|
||||
volumes:
|
||||
cargo-cache:
|
||||
target-cache:
|
||||
@@ -0,0 +1,125 @@
|
||||
# Contributing Translations (i18n)
|
||||
|
||||
Thank you for helping translate Memvid’s documentation and make the project accessible to a global audience.
|
||||
|
||||
This guide explains how to contribute translations for the main `README.md`.
|
||||
|
||||
---
|
||||
|
||||
## What Can Be Translated
|
||||
|
||||
- The main repository `README.md`
|
||||
- Translations are stored in `docs/i18n/`
|
||||
- Each language has a single translation file
|
||||
|
||||
---
|
||||
|
||||
## Translation Workflow
|
||||
|
||||
### 1. Check Existing Issues
|
||||
|
||||
Before starting:
|
||||
- Check open issues to see if your language is already in progress
|
||||
- If an issue exists, comment on it to indicate you want to work on it
|
||||
- Only one contributor should work on a language at a time
|
||||
|
||||
---
|
||||
|
||||
### 2. Create a Translation Issue (If Needed)
|
||||
|
||||
If no issue exists for your language, open a new one using this format:
|
||||
|
||||
**Title:**
|
||||
```
|
||||
README translation: <Language> (<code>)
|
||||
```
|
||||
|
||||
**Labels to apply:**
|
||||
- `i18n`
|
||||
- `documentation`
|
||||
- `help wanted` or `good first issue`
|
||||
|
||||
---
|
||||
|
||||
### 3. Create the Translation File
|
||||
|
||||
1. Copy the main `README.md` from the repository root
|
||||
2. Create a new file in `docs/i18n/` using this format:
|
||||
|
||||
```
|
||||
README.<language-code>.md
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
- `README.es.md`
|
||||
- `README.zh-CN.md`
|
||||
- `README.pt-BR.md`
|
||||
|
||||
Use standard ISO language codes (include region where applicable).
|
||||
|
||||
---
|
||||
|
||||
## Translation Guidelines
|
||||
|
||||
- **Preserve structure**
|
||||
Keep headings, section order, and formatting identical to the original README
|
||||
|
||||
- **Preserve links and badges**
|
||||
Do not modify URLs, badges, or shields
|
||||
|
||||
- **Preserve code blocks**
|
||||
Keep code examples, commands, flags, and API names in English
|
||||
|
||||
- **Maintain technical accuracy**
|
||||
Translate naturally, but do not change meaning or behavior
|
||||
|
||||
- **Language quality matters**
|
||||
Native or fluent speakers are strongly preferred
|
||||
|
||||
---
|
||||
|
||||
## Submitting Your Translation
|
||||
|
||||
1. **Create a new branch:**
|
||||
```bash
|
||||
git checkout -b docs/i18n-<language-code>
|
||||
```
|
||||
|
||||
2. **Add your translation file** under `docs/i18n/`
|
||||
|
||||
3. **Commit your changes:**
|
||||
```bash
|
||||
git commit -m "docs(i18n): add <Language> README translation"
|
||||
```
|
||||
|
||||
4. **Push to your fork** and open a Pull Request
|
||||
|
||||
5. **Reference the translation issue** in your PR description
|
||||
|
||||
---
|
||||
|
||||
## Review Process
|
||||
|
||||
- Maintainers may request:
|
||||
- Clarifications
|
||||
- Formatting fixes
|
||||
- Review by another native speaker
|
||||
- Once approved, the PR will be merged
|
||||
- The corresponding issue will be closed
|
||||
|
||||
---
|
||||
|
||||
## Keeping Translations Up to Date
|
||||
|
||||
When the main `README.md` changes significantly:
|
||||
|
||||
- Existing translations may need updates
|
||||
- Contributors are encouraged to help keep translations in sync
|
||||
|
||||
---
|
||||
|
||||
## Getting Help
|
||||
|
||||
- Open an issue if you have questions about translations
|
||||
- Use GitHub Discussions for coordination
|
||||
- Contact: [contact@memvid.com](mailto:contact@memvid.com)
|
||||
@@ -0,0 +1,160 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
# المساهمة في Memvid (الترجمة العربية)
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid هي طبقة ذاكرة مكونة من ملف واحد لوكلاء الذكاء الاصطناعي (AI Agents)، توفر استرجاعاً فورياً وذاكرة طويلة المدى.</strong>
|
||||
|
||||
ذاكرة دائمة، مؤرشفة، وقابلة للنقل، دون الحاجة إلى قواعد بيانات.
|
||||
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ اترك نجمة (STAR) لدعم المشروع ⭐️</h2>
|
||||
|
||||
---
|
||||
|
||||
## ما هو Memvid؟
|
||||
|
||||
Memvid هو نظام ذاكرة محمول للذكاء الاصطناعي يقوم بتغليف بياناتك، والمتجهات (Embeddings)، وهيكل البحث، والبيانات الوصفية في **ملف واحد فقط**.
|
||||
|
||||
بدلاً من تشغيل خطوط أنابيب RAG معقدة أو قواعد بيانات متجهة تعتمد على الخادم، يتيح Memvid استرجاعاً سريعاً للبيانات مباشرة من الملف.
|
||||
|
||||
النتيجة هي طبقة ذاكرة مستقلة عن النموذج (Model-agnostic) ولا تحتاج إلى بنية تحتية، مما يمنح وكلاء الذكاء الاصطناعي ذاكرة دائمة وطويلة المدى يمكنهم حملها في أي مكان.
|
||||
|
||||
---
|
||||
|
||||
## لماذا "إطارات الفيديو" (Video Frames)؟
|
||||
|
||||
يستلهم Memvid فكرته من ترميز الفيديو، ليس لتخزين الفيديو، بل **لتنظيم ذاكرة الذكاء الاصطناعي كمتسلسلة فائقة الكفاءة من "الإطارات الذكية" (Smart Frames) التي تُضاف باستمرار.**
|
||||
|
||||
"الإطار الذكي" هو وحدة غير قابلة للتغيير تخزن المحتوى مع الطوابع الزمنية، والتحقق من البيانات (Checksums)، والبيانات الوصفية الأساسية. يتم تجميع هذه الإطارات بطريقة تسمح بضغط البيانات وفهرستها والقراءة المتوازية بكفاءة عالية.
|
||||
|
||||
يسمح هذا التصميم بـ:
|
||||
|
||||
- **إضافة البيانات فقط:** الكتابة دون تعديل أو إفساد البيانات الموجودة.
|
||||
- **الاستعلام عبر الزمن:** البحث في حالات الذاكرة السابقة.
|
||||
- **جدول زمني للمعرفة:** فحص كيفية تطور المعرفة بمرور الوقت.
|
||||
- **سلامة البيانات:** ضمان عدم فقدان البيانات عند التعطل بفضل الإطارات الثابتة.
|
||||
|
||||
---
|
||||
|
||||
## المفاهيم الأساسية
|
||||
|
||||
- **محرك الذاكرة الحية:** إضافة وتطوير الذاكرة باستمرار عبر الجلسات.
|
||||
- **كبسولة السياق (`.mv2`):** كبسولات ذاكرة ذاتية الاحتواء وقابلة للمشاركة مع قواعد وصلاحية محددة.
|
||||
- **تصحيح السفر عبر الزمن:** إرجاع أو إعادة تشغيل أو تفريع أي حالة من حالات الذاكرة.
|
||||
- **الاستدعاء الذكي:** وصول محلي للذاكرة في أقل من 5 ملي ثانية مع ذاكرة تخزين مؤقت تنبؤية.
|
||||
- **ذكاء الترميز:** يختار ويحدث تقنيات الضغط تلقائياً بمرور الوقت.
|
||||
|
||||
---
|
||||
|
||||
## حالات الاستخدام
|
||||
|
||||
نظرًا لأن Memvid يعمل دون اتصال بالإنترنت ومستقل عن النماذج، فإنه يُستخدم في:
|
||||
|
||||
- وكلاء الذكاء الاصطناعي طويلي الأمد.
|
||||
- قواعد المعرفة للمؤسسات.
|
||||
- أنظمة الذكاء الاصطناعي التي تعمل "بدون إنترنت أولاً".
|
||||
- فهم الأكواد البرمجية (Codebases).
|
||||
- المساعدين الشخصيين والأنظمة الطبية والقانونية والمالية.
|
||||
|
||||
---
|
||||
|
||||
## أدوات المطورين (SDKs)
|
||||
|
||||
| الحزمة | طريقة التثبيت |
|
||||
| ----------------------------- | --------------------------- |
|
||||
| **واجهة السطر البرمجي (CLI)** | `npm install -g memvid-cli` |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` |
|
||||
| **Python SDK** | `pip install memvid-sdk` |
|
||||
| **Rust** | `cargo add memvid-core` |
|
||||
|
||||
---
|
||||
|
||||
## هيكل الملف
|
||||
|
||||
كل شيء يعيش داخل ملف واحد بصيغة `.mv2`:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ العنوان: النسخة والقدرة الاستيعابية
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ سجل العمليات للتعافي من الأعطال
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ أجزاء البيانات: الإطارات المضغوطة
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ الفهرس اللغوي: البحث النصي الكامل
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ الفهرس المتجهي: البحث بالمتجهات (HNSW)
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ الفهرس الزمني: الترتيب الزمني
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ جدول المحتويات: مواقع الأجزاء
|
||||
└────────────────────────────┘
|
||||
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## الدعم
|
||||
|
||||
هل لديك أسئلة؟
|
||||
البريد الإلكتروني: contact@memvid.com
|
||||
|
||||
**لا تنسَ ترك ⭐ لدعم المشروع!**
|
||||
|
||||
---
|
||||
|
||||
## الترخيص
|
||||
|
||||
رخصة Apache 2.0 — راجع ملف [LICENSE](../../LICENSE) لمزيد من التفاصيل.
|
||||
@@ -0,0 +1,379 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
|
||||
# মেমভিড-এ অবদান (বাংলা অনুবাদ)
|
||||
|
||||
<p align="center">
|
||||
<strong>মেমভিড হল এআই এজেন্টদের জন্য একটি একক-ফাইল মেমরি স্তর যার তাৎক্ষণিক পুনরুদ্ধার এবং দীর্ঘমেয়াদী মেমরি রয়েছে।</strong><br/>
|
||||
ডাটাবেস ছাড়াই স্থায়ী, সংস্করণযুক্ত এবং পোর্টেবল মেমরি।
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">ওয়েবসাইট</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">স্যান্ডবক্সি চেষ্টা করে দেখুন</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">ডক্স</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">আলোচনা</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/</a>
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ প্রকল্পটি সমর্থন করার জন্য একটি তারকা দিন। ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## মেমভিড কী?
|
||||
|
||||
মেমভিড হল একটি পোর্টেবল এআই মেমরি সিস্টেম যা আপনার ডেটা, এম্বেডিং, অনুসন্ধান কাঠামো এবং মেটাডেটা একটি একক ফাইলে প্যাক করে।
|
||||
|
||||
জটিল RAG পাইপলাইন বা সার্ভার-ভিত্তিক ভেক্টর ডাটাবেস চালানোর পরিবর্তে, Memvid ফাইল থেকে সরাসরি দ্রুত ডেটা পুনরুদ্ধারে সহায়তা করে।
|
||||
|
||||
ফলাফল হল একটি মডেল-অজ্ঞেয়বাদী, অবকাঠামো-মুক্ত মেমোরি স্তর যা AI এজেন্টদের স্থায়ী, দীর্ঘমেয়াদী মেমোরি দেয় যা তারা যেকোনো জায়গায় নিতে পারে।
|
||||
---
|
||||
|
||||
## ভিডিও ফ্রেম কেন?
|
||||
|
||||
মেমভিড ভিডিও এনকোডিং থেকে অনুপ্রেরণা নেয়, ভিডিও সংরক্ষণের জন্য নয়, বরং এআই মেমোরিকে অ্যাপেন্ড-ওনলি, স্মার্ট ফ্রেমের অতি-দক্ষ সিকোয়েন্স হিসেবে সংগঠিত করার জন্য।
|
||||
|
||||
একটি স্মার্ট ফ্রেম হল একটি অ-পরিবর্তনযোগ্য ইউনিট যা টাইমস্ট্যাম্প, চেকসাম এবং মৌলিক মেটাডেটা সহ বিষয়বস্তু সংরক্ষণ করে।
|
||||
ফ্রেমগুলিকে এমনভাবে গোষ্ঠীভুক্ত করা হয় যা দক্ষ কম্প্রেশন, ইনডেক্সিং এবং সমান্তরাল পঠনের সুযোগ করে দেয়।
|
||||
|
||||
এই ফ্রেম-ভিত্তিক নকশাটি সক্ষম করে:
|
||||
|
||||
- বিদ্যমান ডেটা পরিবর্তন বা দূষিত না করে কেবল ডেটা যোগ করা
|
||||
- পূর্ববর্তী মেমরি অবস্থা অনুসন্ধান করা
|
||||
- জ্ঞান কীভাবে বিকশিত হয় তার টাইমলাইন-স্টাইল তদন্ত
|
||||
- প্রতিশ্রুতিবদ্ধ, অপরিবর্তনীয় ফ্রেমের মাধ্যমে ক্র্যাশ সুরক্ষা
|
||||
- ভিডিও এনকোডিং থেকে গৃহীত কৌশল ব্যবহার করে দক্ষ কম্প্রেশন।
|
||||
|
||||
ফলাফল হল একটি একক ফাইল যা AI সিস্টেমের জন্য একটি রিওয়াইন্ডেবল মেমরি টাইমলাইন হিসাবে কাজ করে।
|
||||
|
||||
---
|
||||
|
||||
## মূল ধারণা
|
||||
|
||||
- **লিভিং মেমোরি ইঞ্জিন**
|
||||
|
||||
একটি সেশনের সময় স্থায়ী মেমোরি যোগ করুন, শাখা করুন এবং বিকশিত করুন।
|
||||
|
||||
- **ক্যাপসুল রেফারেন্স (`.mv2`)**
|
||||
|
||||
নিয়ম এবং মেয়াদোত্তীর্ণতা সহ স্বয়ংসম্পূর্ণ, শেয়ারযোগ্য মেমোরি ক্যাপসুল।
|
||||
|
||||
- **টাইম-ট্রাভেল ডিবাগিং**
|
||||
|
||||
যেকোন মেমোরি স্টেট রিওয়াইন্ড, রিপ্লে বা শাখা করুন।
|
||||
|
||||
- **স্মার্ট রিকল**
|
||||
|
||||
প্রেডিক্টিভ ক্যাশিং সহ সাব-5ms স্থানীয় মেমোরি অ্যাক্সেস।
|
||||
|
||||
- **কোডেক ইন্টেলিজেন্স**
|
||||
|
||||
সময়ের সাথে সাথে স্বয়ংক্রিয়ভাবে কম্প্রেশন নির্বাচন এবং আপগ্রেড করে।
|
||||
|
||||
---
|
||||
|
||||
## ব্যবহারের ক্ষেত্রে
|
||||
|
||||
মেমভিড হল একটি পোর্টেবল, সার্ভারলেস মেমরি স্তর যা এআই এজেন্টদের স্থায়ী মেমরি এবং দ্রুত রিকল দেয়। যেহেতু এটি মডেল-অ্যাগনস্টিক, মাল্টি-মডেল এবং সম্পূর্ণ অফলাইনে কাজ করে, তাই ডেভেলপাররা বিভিন্ন বাস্তব-বিশ্ব অ্যাপ্লিকেশনে মেমভিড ব্যবহার করছে।
|
||||
|
||||
- দীর্ঘমেয়াদী এআই এজেন্ট
|
||||
- এন্টারপ্রাইজ জ্ঞান ভিত্তি
|
||||
- অফলাইন-প্রথম এআই সিস্টেম
|
||||
- কোডবেস বোঝা
|
||||
- গ্রাহক সহায়তা এজেন্ট
|
||||
- ওয়ার্কফ্লো অটোমেশন
|
||||
- বিক্রয় এবং বিপণন সহ-পাইলট
|
||||
- ব্যক্তিগত জ্ঞান সহকারী
|
||||
- চিকিৎসা, আইনি এবং আর্থিক এজেন্ট
|
||||
- নিরীক্ষণযোগ্য এবং ডিবাগযোগ্য এআই কর্মপ্রবাহ
|
||||
- কাস্টম অ্যাপ্লিকেশন
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
আপনার পছন্দের ভাষায় Memvid ব্যবহার করুন:
|
||||
|
||||
| Package | Install | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Installation (Rust)
|
||||
|
||||
### আবশ্যকতা
|
||||
|
||||
- **Rust 1.85.0+** — Install from [rustup.rs](https://rustup.rs)
|
||||
|
||||
### আপনার প্রকল্পে যোগ করুন
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Description |
|
||||
| ------------------- | ---------------------------------------------- |
|
||||
| `lex` | Full-text search with BM25 ranking (Tantivy) |
|
||||
| `pdf_extract` | Pure Rust PDF text extraction |
|
||||
| `vec` | Vector similarity search (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings for image search |
|
||||
| `whisper` | Audio transcription with Whisper |
|
||||
| `temporal_track` | Natural language date parsing ("last Tuesday") |
|
||||
| `parallel_segments` | Multi-threaded ingestion |
|
||||
| `encryption` | Password-based encryption capsules (.mv2e) |
|
||||
|
||||
Enable features as needed:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## দ্রুত শুরু
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## তৈরি করুন
|
||||
|
||||
রিপোজিটরিটি ক্লোন করুন:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
ডিবাগ মোডে তৈরি করুন:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
রিলিজ মোডে তৈরি করুন (অপ্টিমাইজ করা):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
স্বতন্ত্র বৈশিষ্ট্য সহ তৈরি করুন:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
## পরীক্ষাগুলি চালান
|
||||
|
||||
সকল পরীক্ষা চালান:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
নিম্নলিখিত আউটপুট দিয়ে পরীক্ষাটি চালান:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
একটি নির্দিষ্ট পরীক্ষা চালান:
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
শুধুমাত্র ইন্টিগ্রেশন পরীক্ষা চালান:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## উদাহরণ
|
||||
|
||||
The `examples/` কার্যকরী ডিরেক্টরিগুলির উদাহরণ হল:
|
||||
|
||||
### মৌলিক ব্যবহার
|
||||
|
||||
এটি তৈরি, পুট, অনুসন্ধান এবং টাইমলাইন ক্রিয়াকলাপগুলি দেখায়:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### পিডিএফ ইনজেকশন
|
||||
|
||||
পিডিএফ ডকুমেন্টগুলি গ্রহণ করুন এবং অনুসন্ধান করুন ("Attention Is All You Need" পেপার ব্যবহার করে):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP ভিজ্যুয়াল সার্চ
|
||||
|
||||
CLIP এম্বেডিং ব্যবহার করে ছবি সার্চ (`ক্লিপ` বৈশিষ্ট্য প্রয়োজন):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### হুইস্পার ট্রান্সক্রিপশন
|
||||
|
||||
অডিও ট্রান্সক্রিপশন (`হুইস্পার` বৈশিষ্ট্য প্রয়োজন):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ফাইল ফরম্যাট
|
||||
|
||||
সবকিছু একটি একক `.mv2` ফাইলের মধ্যে রয়েছে:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
কোন `.wal`, `.lock`, `.shm`, অথবা সাইডকার ফাইল নেই। কখনও না।
|
||||
|
||||
সম্পূর্ণ ফাইল ফর্ম্যাট স্পেসিফিকেশনের জন্য [MV2_SPEC.md](MV2_SPEC.md) দেখুন।
|
||||
|
||||
---
|
||||
|
||||
## সহায়তা
|
||||
|
||||
আপনার কি কোন প্রশ্ন বা প্রতিক্রিয়া আছে?
|
||||
|
||||
ইমেল: contact@memvid.com
|
||||
|
||||
**সমর্থন দেখানোর জন্য ⭐ দিন**
|
||||
|
||||
---
|
||||
|
||||
## লাইসেন্স
|
||||
|
||||
Apache License 2.0 — আরও তথ্যের জন্য [LICENSE](LICENSE) ফাইলটি দেখুন।
|
||||
@@ -0,0 +1,424 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid je jednosouborová paměťová vrstva pro AI agenty s okamžitým vyhledáváním a dlouhodobou pamětí.</strong><br/>
|
||||
Trvalá, verzovaná a přenosná paměť, bez databází.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Zanechte hvězdičku na podporu projektu ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Co je Memvid?
|
||||
|
||||
Memvid je systém pro tvorbu AI pamětí, který balí vaše data, embeddingy, strukturu vyhledávání a metadata do jediného souboru.
|
||||
|
||||
Místo spouštění složitých RAG řešení nebo serverových vektorových databází umožňuje Memvid rychlé vyhledávání přímo ze souboru.
|
||||
|
||||
Výsledkem je modelově nezávislá paměťová vrstva bez infrastruktury, která poskytuje agentům AI trvalou, dlouhodobou paměť, kterou lze přenášet kamkoli.
|
||||
|
||||
---
|
||||
|
||||
## Co jsou inteligentní rámce?
|
||||
|
||||
Memvid čerpá inspiraci z enkódování videa, nikoli za účelem ukládání videa, ale za účelem **organizace paměti AI jako ultraefektivní sekvence inteligentních rámců, do kterých lze data pouze přidávat.**
|
||||
|
||||
Inteligentní rámec je neměnná jednotka, která ukládá obsah spolu s časovými značkami, kontrolními součty a základními metadaty.
|
||||
Rámce jsou seskupeny tak, aby umožňovaly efektivní kompresi, indexování a paralelní čtení.
|
||||
|
||||
Tento design založený na rámcích umožňuje:
|
||||
|
||||
- Pouze zápisy bez úpravy nebo poškození existujících dat
|
||||
- Dotazy na minulé stavy paměti
|
||||
- Kontrolu vývoje znalostí ve stylu časové osy
|
||||
- Bezpečnost proti selhání díky závazným, neměnným rámcům
|
||||
- Efektivní kompresi pomocí technik převzatých z kódování videa
|
||||
|
||||
Výsledkem je jediný soubor, který se chová jako časová osa paměti pro systémy AI, ve které lze snadno hledat.
|
||||
|
||||
---
|
||||
|
||||
## Základní koncepty
|
||||
|
||||
- **Living Memory Engine**
|
||||
Kontinuální přidávání, rozvětvování a vývoj paměti, napříč relacemi.
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
Samostatné, sdílené paměťové kapsle s pravidly a dobou platnosti.
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
Převíjení, přehrávání nebo rozvětvování libovolného stavu paměti.
|
||||
|
||||
- **Smart Recall**
|
||||
Přístup k lokální paměti za méně než 5 ms s prediktivním ukládáním do mezipaměti.
|
||||
|
||||
- **Codec Intelligence**
|
||||
Automaticky vybírá a vylepšuje kompresi v průběhu času.
|
||||
|
||||
---
|
||||
|
||||
## Případy použití
|
||||
|
||||
Memvid je přenosná paměťová vrstva bez serveru, která poskytuje agentům AI trvalou paměť a rychlé vyvolání. Protože je modelově nezávislá, multimodální a funguje zcela offline, vývojáři používají Memvid v široké škále reálných aplikací.
|
||||
|
||||
- Dlouhodobě běžící AI agenti
|
||||
- Firemní znalostní báze
|
||||
- Offline-First AI systémy
|
||||
- Porozumění kódu
|
||||
- Agenti zákaznické podpory
|
||||
- Automatizace pracovních postupů
|
||||
- Asistenti prodeje a marketingu
|
||||
- Osobní znalostní asistenti
|
||||
- Lékařští, právní a finanční agenti
|
||||
- Auditovatelné a laditelné AI pracovní postupy
|
||||
- Vlastní aplikace
|
||||
|
||||
---
|
||||
|
||||
## SDK a CLI
|
||||
|
||||
Používejte Memvid ve svém preferovaném jazyce:
|
||||
|
||||
| Balíček | Instalace | Odkazy |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Instalace (Rust)
|
||||
|
||||
### Požadavky
|
||||
|
||||
- **Rust 1.85.0+** — Instalace z [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Přidejte do svého projektu
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Funkční příznaky
|
||||
|
||||
| Funkce | Popis |
|
||||
| ------------------- | ---------------------------------------------------------- |
|
||||
| `lex` | Fulltextové vyhledávání s hodnocením BM25 (Tantivy) |
|
||||
| `pdf_extract` | Čistá extrakce textu z PDF v Rustu |
|
||||
| `vec` | Vektorové vyhledávání podobnosti (HNSW + lokální vkládání textu přes ONNX) |
|
||||
| `clip` | Vizuální vkládání CLIP pro vyhledávání obrázků |
|
||||
| `whisper` | Přepis zvuku pomocí Whisper |
|
||||
| `temporal_track` | Analýza datumu v přirozeném jazyce ("minulé úterý") |
|
||||
| `parallel_segments` | Vícevláknové načítání |
|
||||
| `encryption` | Kapsle šifrované pomocí hesla (.mv2e) |
|
||||
|
||||
Povolte funkce podle potřeby:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Rychlý start
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Vytvoř nový paměťový soubor
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Přidej dokumenty s metadaty
|
||||
let opts = PutOptions::builder()
|
||||
.title("Zápis jednání")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 plánované diskuze...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Vyhledávání
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Sestavení
|
||||
|
||||
Klonujte repozitář:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
Sestavení v režimu developkment:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Sestavení v režimu production (optimalizované):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Sestavení s konkrétními funkcemi:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Spuštění testů
|
||||
|
||||
Spuštění všech testů:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Spuštění testů s výstupem:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Spuštění konkrétního testu:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Spuštění pouze integračních testů:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Příklady
|
||||
|
||||
Adresář `examples/` obsahuje funkční příklady:
|
||||
|
||||
### Základní použití
|
||||
|
||||
Demonstruje operace vytváření, vkládání, vyhledávání a práci s časovou osou:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Načítání PDF
|
||||
|
||||
Načítání a vyhledávání v dokumentech PDF (demo používá dokument "Attention Is All You Need"):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### Vizuální vyhledávání CLIP
|
||||
|
||||
Vyhledávání obrázků pomocí vložení CLIP (vyžaduje funkci `clip`):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Přepis Whisper
|
||||
|
||||
Přepis zvuku (vyžaduje funkci `whisper`):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Modely vkládání textu
|
||||
|
||||
Funkce `vec` zahrnuje podporu lokálního vkládání textu pomocí modelů ONNX. Před použitím lokálního vkládání textu je nutné ručně stáhnout soubory modelů.
|
||||
|
||||
### Rychlý start: BGE-small (doporučeno)
|
||||
|
||||
Stáhněte si výchozí model BGE-small (384 dimenzí, rychlý a efektivní):
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# Download ONNX model
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# Download tokenizer
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### Dostupné modely
|
||||
|
||||
| Model | Rozměry | Velikost | Nejvhodnější pro |
|
||||
| ----------------------- | ---------- | ------- | --------------------- |
|
||||
| `bge-small-en-v1.5` | 384 | ~120 MB | Výchozí, rychlý |
|
||||
| `bge-base-en-v1.5` | 768 | ~420 MB | Lepší kvalita |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530 MB | Univerzální úkoly |
|
||||
| `gte-large` | 1024 | ~1,3 GB | Nejvyšší kvalita |
|
||||
|
||||
### Další modely
|
||||
|
||||
**BGE-base** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large** (1024 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### Použití v kódu
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Použít výchozí model (BGE-small)
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// Použijte jiný model
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
Kompletní příklad s výpočtem podobnosti a hodnocením vyhledávání najdete v souboru `examples/text_embedding.rs`.
|
||||
|
||||
---
|
||||
|
||||
## Formát souboru
|
||||
|
||||
Vše je uloženo v jediném souboru `.mv2`:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Záhlaví (4 KB) │ Magie, verze, kapacita
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64 MB) │ Obnova po selhání
|
||||
├────────────────────────────┤
|
||||
│ Datové segmenty │ Komprimované rámce
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ Vektory HNSW
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronologické řazení
|
||||
├────────────────────────────┤
|
||||
│ TOC (zápatí) │ Segmentové posuny
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Žádné další soubory, jako je `.wal`, `.lock`, `.shm` nejsou potřeba. Nikdy.
|
||||
|
||||
Kompletní specifikace formátu souboru najdete v [MV2_SPEC.md](MV2_SPEC.md).
|
||||
|
||||
---
|
||||
|
||||
## Podpora
|
||||
|
||||
Máte dotazy nebo připomínky?
|
||||
E-mail: contact@memvid.com
|
||||
|
||||
**Dejte ⭐ a projevte svou podporu**
|
||||
|
||||
---
|
||||
|
||||
## Licence
|
||||
|
||||
Apache License 2.0 — podrobnosti najdete v souboru [LICENSE](LICENSE).
|
||||
@@ -0,0 +1,500 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid es una capa de memoria de un solo archivo para agentes de IA, con recuperación instantánea y memoria a largo plazo.</strong><br/>
|
||||
Memoria persistente, versionada y portable, sin bases de datos.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Deja una STAR para apoyar el proyecto ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Lo más destacado de los benchmarks
|
||||
|
||||
**🚀 Mayor precisión que cualquier otro sistema de memoria:** +35% SOTA en LoCoMo, con recall y razonamiento conversacional de largo horizonte de primer nivel.
|
||||
|
||||
**🧠 Mejor razonamiento multi-hop y temporal:** +76% en multi-hop y +56% en temporal frente al promedio de la industria.
|
||||
|
||||
**⚡ Latencia ultra baja a escala:** 0.025 ms P50 y 0.075 ms P99, con 1,372× más throughput que los enfoques estándar.
|
||||
|
||||
**🔬 Benchmarks totalmente reproducibles:** LoCoMo (10 conversaciones de ~26K tokens), evaluación open source y LLM-as-Judge.
|
||||
|
||||
---
|
||||
|
||||
## ¿Qué es Memvid?
|
||||
|
||||
Memvid es un sistema de memoria portable para IA que empaqueta tus datos, embeddings, estructura de búsqueda y metadatos en un solo archivo.
|
||||
|
||||
En lugar de ejecutar pipelines RAG complejos o bases de datos vectoriales basadas en servidor, Memvid permite una recuperación rápida directamente desde el archivo.
|
||||
|
||||
El resultado es una capa de memoria agnóstica al modelo, sin infraestructura, que da a los agentes de IA una memoria persistente y a largo plazo que pueden llevar a cualquier parte.
|
||||
|
||||
---
|
||||
|
||||
## ¿Por qué fotogramas de vídeo?
|
||||
|
||||
Memvid se inspira en la codificación de vídeo, no para almacenar vídeo, sino para **organizar la memoria de IA como una secuencia de Smart Frames ultrarrápida y append-only.**
|
||||
|
||||
Un Smart Frame es una unidad inmutable que almacena contenido junto con marcas de tiempo (timestamps), checksums y metadatos básicos.
|
||||
Los frames se agrupan de una forma que permite una compresión, indexación y lecturas paralelas eficientes.
|
||||
|
||||
Este diseño basado en frames permite:
|
||||
|
||||
- Escrituras append-only sin modificar ni corromper los datos existentes
|
||||
- Consultas sobre estados pasados de la memoria
|
||||
- Inspección estilo línea temporal (timeline) de cómo evoluciona el conocimiento
|
||||
- Seguridad ante fallos (crash safety) mediante frames confirmados e inmutables
|
||||
- Compresión eficiente usando técnicas adaptadas de la codificación de vídeo
|
||||
|
||||
El resultado es un único archivo que se comporta como una línea temporal de memoria “rebobinable” para sistemas de IA.
|
||||
|
||||
---
|
||||
|
||||
## Conceptos principales
|
||||
|
||||
- **Living Memory Engine**
|
||||
Añade, ramifica (branch) y evoluciona la memoria de forma continua entre sesiones.
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
Cápsulas de memoria autocontenidas y compartibles, con reglas y caducidad.
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
Rebobina, reproduce (replay) o ramifica cualquier estado de memoria.
|
||||
|
||||
- **Smart Recall**
|
||||
Acceso local a memoria en menos de 5ms con caché predictiva.
|
||||
|
||||
- **Codec Intelligence**
|
||||
Selecciona y actualiza la compresión automáticamente con el tiempo.
|
||||
|
||||
---
|
||||
|
||||
## Casos de uso
|
||||
|
||||
Memvid es una capa de memoria portable y serverless que da a los agentes de IA memoria persistente y recuerdo rápido. Como es agnóstica al modelo, multi-modal y funciona totalmente offline, los desarrolladores están usando Memvid en una amplia gama de aplicaciones reales.
|
||||
|
||||
- Agentes de IA de larga duración
|
||||
- Bases de conocimiento empresariales
|
||||
- Sistemas de IA offline-first
|
||||
- Comprensión de codebases
|
||||
- Agentes de soporte al cliente
|
||||
- Automatización de flujos de trabajo
|
||||
- Copilotos de ventas y marketing
|
||||
- Asistentes de conocimiento personal
|
||||
- Agentes médicos, legales y financieros
|
||||
- Flujos de trabajo de IA auditables y depurables
|
||||
- Aplicaciones personalizadas
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
Usa Memvid en tu lenguaje preferido:
|
||||
|
||||
| Package | Install | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Instalación (Rust)
|
||||
|
||||
### Requisitos
|
||||
|
||||
- **Rust 1.85.0+** — Instálalo desde [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Añadir a tu proyecto
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Descripción |
|
||||
| ------------------- | ------------------------------------------------------------------ |
|
||||
| `lex` | Búsqueda full-text con ranking BM25 (Tantivy) |
|
||||
| `pdf_extract` | Extracción de texto PDF 100% en Rust |
|
||||
| `vec` | Búsqueda por similitud vectorial (HNSW + embeddings locales vía ONNX) |
|
||||
| `clip` | Embeddings visuales CLIP para búsqueda de imágenes |
|
||||
| `whisper` | Transcripción de audio con Whisper |
|
||||
| `api_embed` | Embeddings en la nube mediante API (OpenAI) |
|
||||
| `temporal_track` | Interpretación de fechas en lenguaje natural ("el martes pasado") |
|
||||
| `parallel_segments` | Ingesta multi-hilo |
|
||||
| `encryption` | Cápsulas cifradas con contraseña (.mv2e) |
|
||||
| `symspell_cleanup` | Reparación robusta de texto PDF (corrige "emp lo yee" -> "employee") |
|
||||
|
||||
Activa las features según lo necesites:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Inicio rápido
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Build
|
||||
|
||||
Clona el repositorio:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
Compila en modo debug:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Compila en modo release (optimizado):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Compila con features específicas:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ejecutar tests
|
||||
|
||||
Ejecuta todos los tests:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Ejecuta los tests con salida:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Ejecuta un test específico:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Ejecuta solo tests de integración:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ejemplos
|
||||
|
||||
El directorio `examples/` contiene ejemplos funcionales:
|
||||
|
||||
### Uso básico
|
||||
|
||||
Demuestra operaciones de create, put, search y timeline:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Ingesta de PDF
|
||||
|
||||
Ingiere y busca documentos PDF (usa el paper “Attention Is All You Need”):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### Búsqueda visual con CLIP
|
||||
|
||||
Búsqueda de imágenes usando embeddings de CLIP (requiere la feature `clip`):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Transcripción con Whisper
|
||||
|
||||
Transcripción de audio (requiere la feature `whisper`):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Modelos de embeddings de texto
|
||||
|
||||
La feature `vec` incluye soporte para embeddings de texto locales usando modelos ONNX. Antes de usar embeddings locales, necesitas descargar manualmente los archivos del modelo.
|
||||
|
||||
### Inicio rápido: BGE-small (recomendado)
|
||||
|
||||
Descarga el modelo BGE-small por defecto (384 dimensiones, rápido y eficiente):
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# Descargar modelo ONNX
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# Descargar tokenizer
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### Modelos disponibles
|
||||
|
||||
| Modelo | Dimensiones | Tamaño | Mejor para |
|
||||
| ----------------------- | ----------- | ------ | --------------------- |
|
||||
| `bge-small-en-v1.5` | 384 | ~120MB | Opción por defecto, rápido |
|
||||
| `bge-base-en-v1.5` | 768 | ~420MB | Mejor calidad |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530MB | Tareas versátiles |
|
||||
| `gte-large` | 1024 | ~1.3GB | Máxima calidad |
|
||||
|
||||
### Otros modelos
|
||||
|
||||
**BGE-base** (768 dimensiones):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic** (768 dimensiones):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large** (1024 dimensiones):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### Uso en código
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Usar el modelo por defecto (BGE-small)
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// Usar un modelo distinto
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
Consulta `examples/text_embedding.rs` para ver un ejemplo completo con cálculo de similitud y ranking de búsqueda.
|
||||
|
||||
### Consistencia del modelo
|
||||
|
||||
Para evitar mezclar modelos por accidente, por ejemplo consultar un índice BGE-small con embeddings de OpenAI, puedes asociar explícitamente tu instancia de Memvid a un nombre de modelo:
|
||||
|
||||
```rust
|
||||
// Vincula el índice a un modelo concreto.
|
||||
// Si el índice ya fue creado con otro modelo, devolverá un error.
|
||||
mem.set_vec_model("bge-small-en-v1.5")?;
|
||||
```
|
||||
|
||||
Esta vinculación es persistente. Una vez definida, cualquier intento futuro de usar otro nombre de modelo fallará de inmediato con un error `ModelMismatch`.
|
||||
|
||||
---
|
||||
|
||||
## Embeddings por API (OpenAI)
|
||||
|
||||
La feature `api_embed` habilita la generación de embeddings en la nube usando la API de OpenAI.
|
||||
|
||||
### Configuración
|
||||
|
||||
Define tu clave de API de OpenAI:
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY="sk-..."
|
||||
```
|
||||
|
||||
### Uso
|
||||
|
||||
```rust
|
||||
use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Usar el modelo por defecto (text-embedding-3-small)
|
||||
let config = OpenAIConfig::default();
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 1536);
|
||||
|
||||
// Usar un modelo de mayor calidad
|
||||
let config = OpenAIConfig::large(); // text-embedding-3-large (3072 dims)
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
### Modelos disponibles
|
||||
|
||||
| Modelo | Dimensiones | Mejor para |
|
||||
| ------------------------ | ----------- | -------------------------------- |
|
||||
| `text-embedding-3-small` | 1536 | Por defecto, más rápido y económico |
|
||||
| `text-embedding-3-large` | 3072 | Máxima calidad |
|
||||
| `text-embedding-ada-002` | 1536 | Modelo heredado |
|
||||
|
||||
Consulta `examples/openai_embedding.rs` para ver un ejemplo completo.
|
||||
|
||||
---
|
||||
|
||||
## Formato de archivo
|
||||
|
||||
Todo vive en un único archivo `.mv2`:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Sin archivos `.wal`, `.lock`, `.shm` ni sidecars. Nunca.
|
||||
|
||||
Consulta [MV2_SPEC.md](MV2_SPEC.md) para la especificación completa del formato de archivo.
|
||||
|
||||
---
|
||||
|
||||
## Soporte
|
||||
|
||||
¿Tienes preguntas o feedback?
|
||||
Email: contact@memvid.com
|
||||
|
||||
**Deja una ⭐ para mostrar apoyo**
|
||||
|
||||
---
|
||||
|
||||
> **Memvid v1 (memoria basada en QR) está obsoleto**
|
||||
>
|
||||
> Si estás viendo referencias a códigos QR, estás usando información desactualizada.
|
||||
>
|
||||
> Consulta: https://docs.memvid.com/memvid-v1-deprecation
|
||||
|
||||
---
|
||||
|
||||
## Licencia
|
||||
|
||||
Apache License 2.0 — consulta el archivo [LICENSE](LICENSE) para más detalles.
|
||||
@@ -0,0 +1,346 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid est une couche mémoire à fichier unique pour agents IA, avec récupération instantanée et mémoire long terme.</strong><br/>
|
||||
Mémoire persistante, versionnée et portable, sans bases de données.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Laissez une STAR pour soutenir le projet ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Qu'est-ce que Memvid ?
|
||||
|
||||
Memvid est un système de mémoire IA portable qui regroupe vos données, embeddings, structure de recherche et métadonnées dans un seul fichier.
|
||||
|
||||
Au lieu d'exécuter des pipelines RAG complexes ou des bases de données vectorielles côté serveur, Memvid permet une récupération rapide directement depuis le fichier.
|
||||
|
||||
Le résultat est une couche mémoire agnostique au modèle, sans infrastructure, qui donne aux agents IA une mémoire persistante et longue durée qu'ils peuvent emporter partout.
|
||||
|
||||
---
|
||||
|
||||
## Pourquoi des images vidéo ?
|
||||
|
||||
Memvid s'inspire de l'encodage vidéo, non pas pour stocker de la vidéo, mais pour **organiser la mémoire IA en une séquence append-only ultra-efficace de Smart Frames.**
|
||||
|
||||
Une Smart Frame est une unité immuable qui stocke le contenu avec des horodatages, des checksums et des métadonnées de base.
|
||||
Les frames sont regroupées d'une manière qui permet une compression, une indexation et des lectures parallèles efficaces.
|
||||
|
||||
Ce design basé sur les frames permet :
|
||||
|
||||
- Écritures append-only sans modifier ni corrompre les données existantes
|
||||
- Requêtes sur des états mémoire passés
|
||||
- Inspection type timeline de l'évolution des connaissances
|
||||
- Sécurité en cas de crash via des frames immuables et validées
|
||||
- Compression efficace grâce à des techniques adaptées de l'encodage vidéo
|
||||
|
||||
Le résultat est un fichier unique qui se comporte comme une timeline mémoire rembobinable pour les systèmes IA.
|
||||
|
||||
---
|
||||
|
||||
## Concepts de base
|
||||
|
||||
- **Moteur de mémoire vivant**
|
||||
Ajoutez, branchez et faites évoluer la mémoire en continu entre les sessions.
|
||||
|
||||
- **Capsule de Contexte (`.mv2`)**
|
||||
Capsules mémoire autonomes et partageables avec règles et expiration.
|
||||
|
||||
- **Débogage par 'voyage temporel'**
|
||||
Rembobinez, rejouez ou branchez n'importe quel état mémoire.
|
||||
|
||||
- **Rappel intelligent**
|
||||
Accès mémoire local en moins de 5 ms avec cache prédictif.
|
||||
|
||||
- **Intelligence du codec**
|
||||
Sélection et mise à niveau automatiques de la compression au fil du temps.
|
||||
|
||||
---
|
||||
|
||||
## Cas d'usage
|
||||
Memvid est une couche mémoire portable et sans serveur qui donne aux agents IA une mémoire persistante et un rappel rapide. Parce qu'il est agnostique au modèle, multimodal et fonctionne entièrement hors ligne, les développeurs utilisent Memvid pour un large éventail d'applications réelles.
|
||||
|
||||
- Agents IA longue durée
|
||||
- Bases de connaissances d'entreprise
|
||||
- Systèmes IA offline-first
|
||||
- Compréhension de codebase
|
||||
- Agents de support client
|
||||
- Automatisation des workflows
|
||||
- Copilotes ventes et marketing
|
||||
- Assistants de connaissance personnels
|
||||
- Agents médicaux, juridiques et financiers
|
||||
- Workflows IA auditables et débogables
|
||||
- Applications sur mesure
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
Utilisez Memvid dans votre langage préféré :
|
||||
|
||||
| Package | Installation | Liens |
|
||||
|---------|---------|-------|
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Installation (Rust)
|
||||
|
||||
### Prérequis
|
||||
|
||||
- **Rust 1.85.0+** — Installer depuis [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Ajouter à votre projet
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| `lex` | Full-text search with BM25 ranking (Tantivy) |
|
||||
| `pdf_extract` | Pure Rust PDF text extraction |
|
||||
| `vec` | Vector similarity search (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings for image search |
|
||||
| `whisper` | Audio transcription with Whisper |
|
||||
| `temporal_track` | Natural language date parsing ("last Tuesday") |
|
||||
| `parallel_segments` | Multi-threaded ingestion |
|
||||
| `encryption` | Password-based encryption capsules (.mv2e) |
|
||||
|
||||
Activez les features selon vos besoins :
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Démarrage rapide
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Créer un nouveau fichier de mémoire
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Ajouter des documents avec des métadonnées
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Rechercher
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Compiler
|
||||
|
||||
Cloner le repository :
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
Compiler en mode debug :
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Compiler en mode release (optimisé) :
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Compiler avec des features spécifiques :
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Exécuter les tests
|
||||
|
||||
Exécuter tous les tests :
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Exécuter les tests avec sortie :
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Exécuter un test spécifique :
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Exécuter uniquement les tests d'intégration :
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Exemples
|
||||
|
||||
Le répertoire `examples/` contient des exemples fonctionnels :
|
||||
|
||||
### Utilisation de base
|
||||
|
||||
Démontre create, put, search et les opérations de timeline :
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Ingestion de PDF
|
||||
|
||||
Ingérer et rechercher des documents PDF (utilise l'article "Attention Is All You Need") :
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### Recherche visuelle CLIP
|
||||
|
||||
Recherche d'images à l'aide d'embeddings CLIP (nécessite la feature `clip`) :
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Transcription Whisper
|
||||
|
||||
Transcription audio (nécessite la feature `whisper`) :
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Format de fichier
|
||||
|
||||
Tout est dans un seul fichier `.mv2` :
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Pas de `.wal`, `.lock`, `.shm` ou fichiers auxiliaires. Jamais.
|
||||
|
||||
Voir [MV2_SPEC.md](MV2_SPEC.md) pour la spécification complète du format de fichier.
|
||||
|
||||
---
|
||||
|
||||
## Support
|
||||
|
||||
Vous avez des questions ou des retours ?
|
||||
Email : contact@memvid.com
|
||||
|
||||
**Laissez une ⭐ pour montrer votre soutien**
|
||||
|
||||
---
|
||||
|
||||
## Licence
|
||||
|
||||
Apache License 2.0 — voir le fichier [LICENSE](LICENSE) pour plus de détails.
|
||||
@@ -0,0 +1,347 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>मेमविड AI एजेंट के लिए एक सिंगल-फाइल मेमोरी लेयर है जिसमें तुरंत रिट्रीवल और लॉन्ग-टर्म मेमोरी होती है।</strong><br/>
|
||||
बिना डेटाबेस के, स्थायी, वर्शन वाली और पोर्टेबल मेमोरी।
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ प्रोजेक्ट को सपोर्ट करने के लिए एक स्टार दें। ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## What is Memvid?
|
||||
|
||||
मेमविड एक पोर्टेबल AI मेमोरी सिस्टम है जो आपके डेटा, एम्बेडिंग, सर्च स्ट्रक्चर और मेटाडेटा को एक ही फ़ाइल में पैक करता है।
|
||||
|
||||
जटिल RAG पाइपलाइन या सर्वर-आधारित वेक्टर डेटाबेस चलाने के बजाय, Memvid सीधे फ़ाइल से तेज़ी से डेटा रिट्रीव करने में मदद करता है।
|
||||
|
||||
इसका नतीजा एक मॉडल-एग्नोस्टिक, इंफ्रास्ट्रक्चर-फ्री मेमोरी लेयर है जो AI एजेंट को परमानेंट, लॉन्ग-टर्म मेमोरी देती है जिसे वे कहीं भी ले जा सकते हैं।
|
||||
|
||||
---
|
||||
|
||||
## वीडियो फ़्रेम क्यों?
|
||||
|
||||
मेमविड वीडियो एन्कोडिंग से प्रेरणा लेता है, वीडियो स्टोर करने के लिए नहीं, बल्कि**AI मेमोरी को स्मार्ट फ्रेम्स के अपेंड-ओनली, अल्ट्रा-एफ़िशिएंट सीक्वेंस के तौर पर ऑर्गनाइज़ करें।**
|
||||
|
||||
स्मार्ट फ्रेम एक ऐसा यूनिट है जिसे बदला नहीं जा सकता, जो टाइमस्टैम्प, चेकसम और बेसिक मेटाडेटा के साथ कंटेंट स्टोर करता है।
|
||||
फ्रेम को इस तरह से ग्रुप किया जाता है जिससे कुशल कम्प्रेशन, इंडेक्सिंग और पैरेलल रीड संभव हो सके।
|
||||
|
||||
यह फ्रेम-आधारित डिज़ाइन इन चीज़ों को संभव बनाता है:
|
||||
|
||||
- मौजूदा डेटा को संशोधित या दूषित किए बिना केवल डेटा जोड़ना
|
||||
- पिछली मेमोरी स्थितियों पर प्रश्न
|
||||
- ज्ञान कैसे विकसित होता है, इसकी टाइमलाइन-शैली में जांच
|
||||
- प्रतिबद्ध, अपरिवर्तनीय फ्रेम के माध्यम से क्रैश सुरक्षा
|
||||
- वीडियो एन्कोडिंग से अपनाई गई तकनीकों का उपयोग करके कुशल कम्प्रेशन।
|
||||
|
||||
इसका नतीजा एक सिंगल फ़ाइल होती है जो AI सिस्टम के लिए रिवाइंड करने लायक मेमोरी टाइमलाइन की तरह काम करती है।
|
||||
|
||||
---
|
||||
|
||||
## मुख्य अवधारणाएँ
|
||||
|
||||
- **लिविंग मेमोरी इंजन**
|
||||
सेशन के दौरान लगातार मेमोरी को जोड़ें, ब्रांच करें और विकसित करें।
|
||||
|
||||
- **कैप्सूल संदर्भ (`.mv2`)**
|
||||
नियमों और एक्सपायरी के साथ सेल्फ-कंटेन्ड, शेयर करने लायक मेमोरी कैप्सूल।
|
||||
|
||||
- **टाइम-ट्रैवल डिबगिंग**
|
||||
किसी भी मेमोरी स्टेट को रिवाइंड, रिप्ले या ब्रांच करें।
|
||||
|
||||
- **स्मार्ट रिकॉल**
|
||||
प्रेडिक्टिव कैशिंग के साथ सब-5ms लोकल मेमोरी एक्सेस।
|
||||
|
||||
- **कोडेक इंटेलिजेंस**
|
||||
यह समय के साथ कम्प्रेशन को ऑटो-सेलेक्ट और अपग्रेड करता है।
|
||||
|
||||
---
|
||||
|
||||
## उपयोग के मामले
|
||||
|
||||
मेमविड एक पोर्टेबल, सर्वरलेस मेमोरी लेयर है जो AI एजेंट को परमानेंट मेमोरी और तेज़ रिकॉल देता है। क्योंकि यह मॉडल-एग्नोस्टिक, मल्टी-मॉडल है, और पूरी तरह से ऑफ़लाइन काम करता है, इसलिए डेवलपर्स मेमविड का इस्तेमाल कई तरह के रियल-वर्ल्ड एप्लीकेशन में कर रहे हैं।
|
||||
|
||||
- लंबे समय तक चलने वाले AI एजेंट
|
||||
- एंटरप्राइज़ नॉलेज बेस
|
||||
- ऑफ़लाइन-फ़र्स्ट AI सिस्टम
|
||||
- कोडबेस को समझना
|
||||
- कस्टमर सपोर्ट एजेंट
|
||||
- वर्कफ़्लो ऑटोमेशन
|
||||
- सेल्स और मार्केटिंग कोपायलट
|
||||
- पर्सनल नॉलेज असिस्टेंट
|
||||
- मेडिकल, लीगल और फाइनेंशियल एजेंट
|
||||
- ऑडिट करने योग्य और डीबग करने योग्य AI वर्कफ़्लो
|
||||
- कस्टम एप्लीकेशन
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
मेमविड को अपनी पसंदीदा भाषा में इस्तेमाल करें:
|
||||
|
||||
| Package | Install | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Installation (Rust)
|
||||
|
||||
### आवश्यकताएं
|
||||
|
||||
- **Rust 1.85.0+** — Install from [rustup.rs](https://rustup.rs)
|
||||
|
||||
### अपने प्रोजेक्ट में जोड़ें
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Description |
|
||||
| ------------------- | ---------------------------------------------- |
|
||||
| `lex` | Full-text search with BM25 ranking (Tantivy) |
|
||||
| `pdf_extract` | Pure Rust PDF text extraction |
|
||||
| `vec` | Vector similarity search (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings for image search |
|
||||
| `whisper` | Audio transcription with Whisper |
|
||||
| `temporal_track` | Natural language date parsing ("last Tuesday") |
|
||||
| `parallel_segments` | Multi-threaded ingestion |
|
||||
| `encryption` | Password-based encryption capsules (.mv2e) |
|
||||
|
||||
Enable features as needed:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## त्वरित प्रारंभ
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## निर्माण
|
||||
|
||||
रिपॉजिटरी को क्लोन करें:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
डीबग मोड में बिल्ड करें:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
रिलीज़ मोड में बिल्ड करें (ऑप्टिमाइज़्ड):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
विशिष्ट विशेषताओं के साथ बनाएं:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## टेस्ट चलाएँ
|
||||
|
||||
सभी टेस्ट चलाएँ:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
आउटपुट के साथ टेस्ट चलाएँ:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
एक विशिष्ट टेस्ट चलाएँ:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
केवल इंटीग्रेशन टेस्ट चलाएँ:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## उदाहरण
|
||||
|
||||
The `examples/` डायरेक्टरी में काम करने वाले उदाहरण हैं:
|
||||
|
||||
### बेसिक उपयोग
|
||||
|
||||
यह क्रिएट, पुट, सर्च और टाइमलाइन ऑपरेशन दिखाता है:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDF इन्जेक्शन
|
||||
|
||||
PDF डॉक्यूमेंट्स को इन्जेस्ट करें और सर्च करें ("अटेंशन इज़ ऑल यू नीड" पेपर का इस्तेमाल करता है):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP विज़ुअल सर्च
|
||||
|
||||
CLIP एम्बेडिंग का इस्तेमाल करके इमेज सर्च (इसके लिए `clip` फ़ीचर ज़रूरी है):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### व्हिस्पर ट्रांसक्रिप्शन
|
||||
|
||||
ऑडियो ट्रांसक्रिप्शन (`whisper` फीचर ज़रूरी है):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## फ़ाइल फ़ॉर्मेट
|
||||
|
||||
सब कुछ एक ही `.mv2` फ़ाइल में होता है:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
कोई `.wal`, `.lock`, `.shm`, या साइडकार फ़ाइल नहीं। कभी नहीं।
|
||||
|
||||
पूरे फ़ाइल फ़ॉर्मेट स्पेसिफ़िकेशन के लिए [MV2_SPEC.md](MV2_SPEC.md) देखें।
|
||||
|
||||
---
|
||||
|
||||
## सपोर्ट
|
||||
|
||||
क्या आपके कोई सवाल या फीडबैक हैं?
|
||||
Email: contact@memvid.com
|
||||
|
||||
**सपोर्ट दिखाने के लिए ⭐ दें**
|
||||
|
||||
---
|
||||
|
||||
## लाइसेंस
|
||||
|
||||
Apache License 2.0 — ज़्यादा जानकारी के लिए [LICENSE](LICENSE) फ़ाइल देखें।
|
||||
@@ -0,0 +1,429 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvidは、AIエージェントのための即時検索と長期記憶を備えた単一ファイル型メモリレイヤーです。</strong>
|
||||
</br>
|
||||
データベースを必要とせず、永続化、バージョン管理、ポータブル性を備えたメモリを実現します。
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ スターで応援お願いします ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Memvidとは？
|
||||
|
||||
Memvidは、データ、埋め込み、検索構造、メタデータを１つのファイルにパッケージ化するポータブルAIメモリシステムです。
|
||||
|
||||
複雑なRAGパイプラインやサーバーベースのベクトルデータベースを運用する代わりに、Memvidを使用することで直接ファイルから高速な検索が可能になります。
|
||||
|
||||
その結果、モデルに依存せずインフラ不要のメモリレイヤーが実現し、AIエージェントはどこでも使える永続的な長期記憶を持つことができます。
|
||||
|
||||
---
|
||||
|
||||
## スマートフレーム (Smart Frames) とは？
|
||||
|
||||
Memvidは、（ビデオを保存するためではなく）**追記に特化した効率的なスマートフレームのシーケンスとしてAIメモリを整理するため**に、ビデオエンコーディング技術から着想を得ています。
|
||||
|
||||
スマートフレームは、コンテンツをタイムスタンプ、チェックサム、基本メタデータとともに保存する不変（イミュータブル）な単位です。フレームは効率的な圧縮、インデックス作成、並列読み取りができるようグループ化されています。
|
||||
|
||||
このフレームベースの設計により、以下が可能になります。
|
||||
|
||||
- 既存のデータを変更したり破損したりすることなくデータを追加
|
||||
- 過去のメモリ状態に対するクエリ
|
||||
- 知識がどのように進化するかをタイムライン形式で検査
|
||||
- コミットされた不変フレームによるクラッシュ耐性
|
||||
- ビデオエンコーディング技術を応用した効率的な圧縮
|
||||
|
||||
その結果、AIシステムの「巻き戻し可能なメモリタイムライン」のように機能する単一のファイルが生成されます。
|
||||
|
||||
---
|
||||
|
||||
## コアコンセプト
|
||||
|
||||
- **成長するメモリエンジン (Living Memory Engine)**
|
||||
セッションをまたいでメモリを継続的に追加、分岐、進化させます。
|
||||
|
||||
- **カプセル・コンテキスト (`.mv2`)**
|
||||
ルールや有効期限を設定できる、自己完結型で共有可能なメモリカプセル。
|
||||
|
||||
- **タイムトラベル・デバッグ**
|
||||
任意のメモリ状態を巻き戻し、再現、または分岐させることができます。
|
||||
|
||||
- **スマート・リコール**
|
||||
予測キャッシングによる5ミリ秒未満のローカルメモリーアクセス。
|
||||
|
||||
- **コーデック・インテリジェンス**
|
||||
圧縮方式を自動選択し、時間の経過とともにアップグレードします。
|
||||
|
||||
---
|
||||
|
||||
## ユースケース
|
||||
|
||||
Memvidは、AIエージェントに永続的な記憶と高速な呼び出し機能を提供するポータブルでサーバーレスなメモリレイヤーです。モデルに依存せず、マルチモーダルに対応し、完全にオフラインで動作するため、実用的なアプリケーションで幅広く利用されています。
|
||||
|
||||
- 長期稼働AIエージェント
|
||||
- エンタープライズ向けナレッジベース
|
||||
- オフラインファーストAIシステム
|
||||
- コードベースの理解
|
||||
- カスタマーサポートエージェント
|
||||
- ワークフロー自動化
|
||||
- セールス・マーケティング支援
|
||||
- パーソナル・ナレッジ・アシスタント
|
||||
- 医療・法律・金融特化型エージェント
|
||||
- 監査・デバッグ可能なAIワークフロー
|
||||
- カスタムアプリケーション
|
||||
|
||||
---
|
||||
|
||||
## SDK と CLI
|
||||
|
||||
お好みの言語でMemvidを利用できます。
|
||||
|
||||
| パッケージ | インストール | リンク |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## インストール (Rust)
|
||||
|
||||
### 要件
|
||||
|
||||
- **Rust 1.85.0+** - [rustup.rs](https://rustup.rs) からインストールしてください。
|
||||
|
||||
### プロジェクトへの追加
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
|
||||
```
|
||||
|
||||
### 機能フラグ (Feature Flags)
|
||||
|
||||
| 機能 | 説明 |
|
||||
| ------------------- | -------------------------------------------------------------- |
|
||||
| `lex` | BM25ランキングによる全文検索 (Tantivy) |
|
||||
| `pdf_extract` | Pure RustによるPDFテキスト抽出 |
|
||||
| `vec` | ベクトル類似性検索 (HNSW + ONNXによるローカルテキスト埋め込み) |
|
||||
| `clip` | 画像検索用のCLIPビジュアル埋め込み |
|
||||
| `whisper` | Whisperによる音声文字起こし |
|
||||
| `temporal_track` | 自然言語による日付解析 (例: "last Tuesday") |
|
||||
| `parallel_segments` | マルチスレッドによるデータ取り込み |
|
||||
| `encryption` | パスワードベースの暗号化カプセル (.mv2e) |
|
||||
|
||||
以下のように、必要に応じて有効化してください。
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## クイックスタート
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// 新しいメモリファイルを作成
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// メタデータ付きでドキュメントを追加
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// 検索の実行
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ビルド
|
||||
|
||||
リポジトリをクローン：
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
デバッグモードでビルド：
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
リリースモードでビルド（最適化）：
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
特定の機能フラグ付きでビルド：
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## テストの実行
|
||||
|
||||
すべてのテストを実行：
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
標準出力でテストを実行：
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
特定のテストを実行：
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
統合テストのみを実行：
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## サンプル (Examples)
|
||||
|
||||
`examples/` ディレクトリには、実際に動作するサンプルコードが用意されています。
|
||||
|
||||
### 基本的な使い方 (Basic Usage)
|
||||
|
||||
作成 (create)、追加 (put)、検索 (search)、およびタイムライン操作のデモです。
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDFの取り込み (PDF Ingestion)
|
||||
|
||||
PDFドキュメントの取り込みと検索のサンプルです。（論文「Attention Is All You Need」を使用）
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIPによる画像検索 (CLIP Visual Search)
|
||||
|
||||
CLIP埋め込みを使用した画像検索のサンプルです。
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Whisperによる文字起こし (Whisper Transcription)
|
||||
|
||||
音声文字起こしのサンプルです。
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## テキスト埋め込みモデル
|
||||
|
||||
`vec` 機能は、ONNXモデルを使用したローカルでのテキスト埋め込みをサポートしています。利用前にモデルファイルを手動でダウンロードする必要があります。
|
||||
|
||||
### 推奨：BGE-small (デフォルト)
|
||||
|
||||
高速で効率的なBGE-smallモデル（384次元）をダウンロードします。
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# ONNXモデルのダウンロード
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# トークナイザーのダウンロード
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### モデル一覧
|
||||
|
||||
| モデル | 次元数 | サイズ | 最適な用途 |
|
||||
| ----------------------- | ------ | ------ | ------------------------ |
|
||||
| `bge-small-en-v1.5` | 384 | ~120MB | デフォルト（高速・軽量） |
|
||||
| `bge-base-en-v1.5` | 768 | ~420MB | より高い精度が必要な場合 |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530MB | 多目的なタスク |
|
||||
| `gte-large` | 1024 | ~1.3GB | 最高精度 |
|
||||
|
||||
### 他のモデル
|
||||
|
||||
**BGE-base** (768次元):
|
||||
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic** (768次元):
|
||||
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large** (1024次元):
|
||||
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### 使用例
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// デフォルトモデルを使用する場合 (BGE-small)
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// モデルを変更する場合
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
類似性の計算と検索ランキングを含む完全な例については、`examples/text_embedding.rs` を参照してください。
|
||||
|
||||
---
|
||||
|
||||
## ファイル構成
|
||||
|
||||
すべてが単一の `.mv2` ファイルに収められます。
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ ヘッダー (4KB) │ マジックナンバー、バージョン、容量
|
||||
├────────────────────────────┤
|
||||
│ 組み込みWAL (1-64MB) │ クラッシュリカバリ用
|
||||
├────────────────────────────┤
|
||||
│ データセグメント │ 圧縮されたフレーム
|
||||
├────────────────────────────┤
|
||||
│ 全文検索インデックス (Lex) │ Tantivy全文検索
|
||||
├────────────────────────────┤
|
||||
│ ベクトルインデックス (Vec) │ HNSWベクトル
|
||||
├────────────────────────────┤
|
||||
│ タイムインデックス │ 時系列順序
|
||||
├────────────────────────────┤
|
||||
│ TOC (フッター) │ セグメントオフセット
|
||||
└────────────────────────────┘
|
||||
|
||||
```
|
||||
|
||||
`.wal`、`.lock`、`.shm` などのサイドカーファイルは一切生成されません。
|
||||
|
||||
フォーマット仕様の詳細は [MV2_SPEC.md](MV2_SPEC.md) を参照してください。
|
||||
|
||||
---
|
||||
|
||||
## サポート
|
||||
|
||||
ご質問やフィードバックはこちらまでご連絡ください。
|
||||
メール: contact@memvid.com
|
||||
|
||||
**⭐でプロジェクトをサポートしてください。**
|
||||
|
||||
---
|
||||
|
||||
## ライセンス
|
||||
|
||||
Apache License 2.0 - 詳細は [LICENSE](LICENSE) ファイルをご覧ください。
|
||||
@@ -0,0 +1,424 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid는 AI 에이전트를 위한 단일 파일 메모리 레이어로, 인스턴스 검색 및 장기 메모리 기능을 제공합니다.</strong><br/>
|
||||
데이터 베이스 없이 지속적이고, 버전 관리가 용이하며 여러 어플리케이션에 자유로운 적용이 가능합니다.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ STAR로 이 프로젝트를 지원해주세요 ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Memvid란?
|
||||
|
||||
Memvid는 데이터, 임베딩, 검색 구조 및 메타데이터를 단일 파일로 패키징하는 이식 가능한 AI 메모리 시스템입니다.
|
||||
|
||||
복잡한 RAG 파이프라인이나 서버 기반 벡터 데이터베이스를 실행하는 대신, Memvid는 파일에서 직접 빠른 검색을 가능하게 합니다.
|
||||
|
||||
결과적으로 모델에 독립적이며 인프라 구조와는 독립적인 메모리 레이어로, AI 에이전트가 어디서나 휴대할 수 있는 지속적 장기 메모리를 제공합니다
|
||||
|
||||
---
|
||||
|
||||
## Smart Frames란?
|
||||
|
||||
Memvid는 **AI 메모리를 추가 전용(append-only)의 초고효율 Smart Frame 시퀀스로 구성하기 위해** 비디오 인코딩에서 영감을 받았습니다.
|
||||
|
||||
Smart Frame은 타임스탬프, 체크섬 및 기본 메타데이터와 함께 콘텐츠를 저장하는 불변 단위입니다.
|
||||
프레임은 효율적인 압축, 인덱싱 및 병렬 읽기를 허용하는 방식으로 그룹화됩니다.
|
||||
|
||||
이러한 프레임 기반 설계는 다음을 가능하게 합니다:
|
||||
|
||||
- 기존 데이터를 수정하거나 손상시키지 않는 추가 전용(append-only) 쓰기
|
||||
- 과거 메모리 상태에 대한 쿼리
|
||||
- 지식이 어떻게 변화하는지에 대한 타임라인 스타일 검사
|
||||
- 불변 프레임워크를 통한 크래시 안전성
|
||||
- 비디오 인코딩에서 차용한 기술을 사용한 효율적인 압축
|
||||
|
||||
이를 위한 결과물은 AI 시스템을 위한 되감기 가능한 메모리 타임라인처럼 동작하는 단일 파일입니다.
|
||||
|
||||
---
|
||||
|
||||
## 주요 개념
|
||||
|
||||
- **실시간 변화하는 메모리 엔진**
|
||||
세션 간에 메모리를 지속적으로 추가, 분기 및 변화시킵니다.
|
||||
|
||||
- **문맥 캡슐화 (`.mv2`)**
|
||||
규칙과 만료 시간이 포함된 자립형 공유 가능 형대의 메모리 캡슐입니다.
|
||||
|
||||
- **시간 기반 디버깅**
|
||||
임의의 메모리 상태로 되감기, 재생 또는 분기합니다.
|
||||
|
||||
- **예측 기반 호출**
|
||||
예측 캐싱을 사용한 5ms 미만 로컬 메모리 액세스를 제공합니다.
|
||||
|
||||
- **코덱 인텔리전스**
|
||||
시간 경과에 따라 압축을 자동 선택 및 업그레이드합니다.
|
||||
|
||||
---
|
||||
|
||||
## 이용 사례
|
||||
|
||||
Memvid 이동 가능한 서버리스 메모리 레이어로 AI 에이전트에 지속적인 메모리와 빠른 호출을 제공합니다. 이는 모델과 독립적이고, 멀티모달을 지원하며, 인터넷을 사용하지 않으므로, 개발자들은 다양한 실제 어플리케이션에서 Memvid를 활용하고 있습니다.
|
||||
|
||||
- 장기 실행 AI 에이전트
|
||||
- 기업 내의 지식 베이스
|
||||
- 오프라인 우선의 AI 시스템
|
||||
- 코드베이스 이해
|
||||
- 고객 지원 에이전트
|
||||
- 워크플로 자동화
|
||||
- 판매 및 마케팅 코파일럿
|
||||
- 개인 지식 어시스턴트
|
||||
- 의료, 법률 및 금융 에이전트
|
||||
- 모니터링 및 디버깅 가능한 AI 워크플로
|
||||
- 그 외의 여러 애플리케이션
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
원하는 언어로 Memvid를 사용하세요:
|
||||
|
||||
| 패키지 | 설치 커맨드 | 링크 |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## 설치 (Rust)
|
||||
|
||||
### 요구 사항
|
||||
|
||||
- **Rust 1.85.0+** — [rustup.rs](https://rustup.rs)에서 설치 가능합니다.
|
||||
|
||||
### 프로젝트에 추가
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Description |
|
||||
| ------------------- | --------------------------------------------------- |
|
||||
| `lex` | BM25 랭킹 기반 전체 텍스트 검색 (Tantivy) |
|
||||
| `pdf_extract` | Rust 기반 PDF 텍스트 추출 |
|
||||
| `vec` | 벡터 유사도 검색 (HNSW + ONNX) |
|
||||
| `clip` | 이미지 검색을 위한 CLIP 임베딩 |
|
||||
| `whisper` | Whisper 기반 오디오 전사 |
|
||||
| `temporal_track` | 자연어 날짜 추출 ("지난 화요일") |
|
||||
| `parallel_segments` | 멀티-스레딩 처리 |
|
||||
| `encryption` | Password 기반 암호화 (.mv2e) |
|
||||
|
||||
필요한 기능을 아래 방식으로 활성화하세요:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 빌드
|
||||
|
||||
이 레포지토리 클론:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
디버그 모드로 빌드:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
배포 모드로 빌드 (optimized):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
특수 기능을 포함하도록 빌드:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 테스트
|
||||
|
||||
전체 테스트 실행:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
테스트 실행 및 결과 출력:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
특정 테스트 실행:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
인테그레이션 테스트만 실행:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 예시
|
||||
|
||||
`examples/` 디렉토리에 예제가 있습니다:
|
||||
|
||||
### 기본 사용법
|
||||
|
||||
생성, 추가, 검색 및 타임라인 작업을 보여줍니다:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDF 수집
|
||||
|
||||
PDF 문서 수집 및 검색 ("Attention Is All You Need" 논문 사용):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP 이미지 검색
|
||||
|
||||
CLIP 임베딩을 사용한 이미지 검색 (`clip` 기능 필요):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Whisper 전사
|
||||
|
||||
오디오 전사 (`whisper` 기능 필요):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Text Embedding 모델
|
||||
|
||||
`vec` 기능은 ONNX 모델을 사용한 로컬 텍스트 임베딩을 포함합니다. 로컬 텍스트 임베딩을 사용하기 전에 모델 파일을 수동으로 다운로드해야 합니다.
|
||||
|
||||
### Quick Start: BGE-small (추천함)
|
||||
|
||||
기본 BGE-small 모델(384 차원, 빠르고 효율적) 다운로드:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# Download ONNX model
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# Download tokenizer
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### 지원 모델
|
||||
|
||||
| 모델명 | 차원 수 | 크기 | 권장 용도 |
|
||||
| ----------------------- | ---------- | ----- | --------------------- |
|
||||
| `bge-small-en-v1.5` | 384 | ~120MB | 기본 설정, 가장 빠름 |
|
||||
| `bge-base-en-v1.5` | 768 | ~420MB | 꽤 좋은 성능 |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530MB | 다양한 업무 가능 |
|
||||
| `gte-large` | 1024 | ~1.3GB | 가장 좋은 성능 |
|
||||
|
||||
### 타 모델
|
||||
|
||||
**BGE-base** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large** (1024 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### 코드 내 사용법
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Use default model (BGE-small)
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// Use different model
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
유사도 계산 및 검색 랭킹이 포함된 전체 예제는 `examples/text_embedding.rs`를 참조하세요.
|
||||
|
||||
---
|
||||
|
||||
## 파일 구조
|
||||
|
||||
모든 구성 요소는 단일 `.mv2` 파일 내에 구성됩니다:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
`.wal`, `.lock`, `.shm`, 혹은 그 외의 별도 구성 요소는 없습니다.
|
||||
|
||||
[MV2_SPEC.md](MV2_SPEC.md)에서 파일 세부 형식을 확인할 수 있습니다.
|
||||
|
||||
---
|
||||
|
||||
## Support
|
||||
|
||||
문의 사항은 아래 이메일로 부탁드립니다.
|
||||
Email: contact@memvid.com
|
||||
|
||||
**⭐를 눌러 이 프로젝트를 지원해주세요**
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
Apache License 2.0 — [LICENSE](LICENSE) 파일 참고.
|
||||
@@ -0,0 +1,90 @@
|
||||
# Internationalization (i18n) - README Translations
|
||||
|
||||
This folder contains translated versions of the main [README.md](../../README.md) file for different languages.
|
||||
|
||||
## Purpose
|
||||
|
||||
The `docs/i18n/` directory is dedicated to making Memvid accessible to developers worldwide by providing localized versions of the main README. Each translation helps non-English speakers understand and use Memvid more effectively.
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
docs/i18n/
|
||||
├── README.md # This file
|
||||
├── README.zh-CN.md # Chinese (Simplified) translation
|
||||
├── README.zh-TW.md # Chinese (Traditional) translation
|
||||
├── README.es.md # Spanish translation
|
||||
├── README.fr.md # French translation
|
||||
├── README.de.md # German translation
|
||||
├── README.ja.md # Japanese translation
|
||||
├── README.ko.md # Korean translation
|
||||
├── README.pt-BR.md # Portuguese (Brazil) translation
|
||||
├── README.so.md # Somali translation
|
||||
└── ... # Additional languages
|
||||
```
|
||||
|
||||
## Contributing Translations
|
||||
|
||||
We welcome contributions of README translations! For detailed guidelines, see [Contributing Translations](CONTRIBUTING_TRANSLATIONS.md).
|
||||
|
||||
Here's a quick overview:
|
||||
|
||||
### 1. Check Existing Translations
|
||||
|
||||
Before starting, check if a translation for your language already exists or is in progress.
|
||||
|
||||
### 2. Create a Translation File
|
||||
|
||||
- Use the format: `README.{language-code}.md`
|
||||
- Use standard language codes (e.g., `zh-CN`, `es`, `fr`, `de`, `ja`, `ko`, `pt-BR`, `so`)
|
||||
- Copy the main `README.md` as a starting point
|
||||
|
||||
### 3. Translation Guidelines
|
||||
|
||||
- **Keep the structure**: Maintain the same headings, sections, and formatting as the original
|
||||
- **Preserve links**: Keep all URLs and links unchanged
|
||||
- **Preserve code blocks**: Keep code examples, commands, and technical terms in English (or add comments in the target language)
|
||||
- **Update badges**: Keep badges and shields as they are (they're language-agnostic)
|
||||
- **Maintain accuracy**: Ensure technical accuracy while making the content natural in the target language
|
||||
|
||||
### 4. Submit Your Translation
|
||||
|
||||
1. Create a new file: `README.{language-code}.md` in this directory
|
||||
2. Translate the content while following the guidelines above
|
||||
3. Submit a pull request with:
|
||||
- A clear description of the language being added
|
||||
- Your name/username for attribution (if desired)
|
||||
|
||||
## Language Codes
|
||||
|
||||
Use standard ISO 639-1 or ISO 639-2 language codes:
|
||||
|
||||
- `zh-CN` - Chinese (Simplified)
|
||||
- `zh-TW` - Chinese (Traditional)
|
||||
- `es` - Spanish
|
||||
- `fr` - French
|
||||
- `de` - German
|
||||
- `ja` - Japanese
|
||||
- `ko` - Korean
|
||||
- `pt-BR` - Portuguese (Brazil)
|
||||
- `ru` - Russian
|
||||
- `ar` - Arabic
|
||||
- `hi` - Hindi
|
||||
- `so` - Somali
|
||||
- And more...
|
||||
|
||||
## Maintenance
|
||||
|
||||
Translations should be updated when the main README is significantly changed. Contributors are encouraged to keep their translations in sync with the English version.
|
||||
|
||||
## Questions?
|
||||
|
||||
If you have questions about translations or want to coordinate with other translators, please:
|
||||
- See [Contributing Translations](CONTRIBUTING_TRANSLATIONS.md) for detailed guidelines
|
||||
- Open an issue on GitHub
|
||||
- Join our [Discussions](https://github.com/memvid/memvid/discussions)
|
||||
- Contact: contact@memvid.com
|
||||
|
||||
---
|
||||
|
||||
**Thank you for helping make Memvid accessible to developers worldwide! 🌍**
|
||||
@@ -0,0 +1,347 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid is een geheugenlaag van één bestand voor AI-agenten met directe toegang en langetermijnsgeheugen.</strong><br/>
|
||||
Volhardend en draagbaar geheugen met versiebeheer en zonder databases.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Laat een ster achter om het project te steunen ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Wat is Memvid?
|
||||
|
||||
Memvid is een draagbaar AI-geheugensysteem dat uw data, embedding, zoekstructuur en metadata in één bestand opslaat.
|
||||
|
||||
In plaats van complexe RAG pijplijnen of servergebaseerde vectordatabases te gebruiken, zal Memvid snelle toegang recht vanuit het bestand toestaan.
|
||||
|
||||
Het resultaat is een model-agnostische, infrastructuurvrije geheugenlaag die AI-agenten een volhardende langetermijnsgeheugen geeft, die ze overal kunnen meenemen.
|
||||
|
||||
---
|
||||
|
||||
## Waarom videoframes?
|
||||
|
||||
Memvid neemt inspiratie uit videos encoderen, niet om de video op te slaan, maar om **het organiseren van AI-geheugen als een ultra-efficiënte sequentie van Smart Frames waarbij je enkel kan toevoegen.**
|
||||
|
||||
Een Smart Frame is een immutabele eenheid die content opslaat samen met zijn tijdstempels, controlesommen en basismetadata.
|
||||
Frames worden gegroupeerd in een manier die voor efficiënte compressie, indexing en parallele lezingen zorgt.
|
||||
|
||||
Dit frame-gebaseerde design maakt het volgende mogelijk:
|
||||
|
||||
- Append-only bijschrijven van data zonder het aanpassen of corrumperen van bestaande data
|
||||
- Zoekopdrachten over vorige geheugenstaten
|
||||
- Tijdlijn-stijl inspectie van hoe kennis evolueert
|
||||
- Crashveiligheid door de vastgelegde immutabele frames
|
||||
- Efficiënte compressie gebruikmakend van technieken aangepast uit video encoderen
|
||||
|
||||
Het resultaat is één bestand dat werkt als een terugspoelbare geheugentijdslijn van AI-systemen.
|
||||
|
||||
---
|
||||
|
||||
## Basisconcepten
|
||||
|
||||
- **Living Memory Engine**
|
||||
Append, vertakt en evolueert geheugen continu over sessies.
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
Autonome, deelbaar geheugencapsules met regels en vervalling.
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
Spoel terug, herspeel, of vertak elke geheugenstatus.
|
||||
|
||||
- **Smart Recall**
|
||||
Sub-5ms lokale geheugentoegang met voorspelbare caching.
|
||||
|
||||
- **Codec Intelligence**
|
||||
Selecteert en verbetert automatisch de compressie doorheen de tijd.
|
||||
|
||||
---
|
||||
|
||||
## Gebruiksgevallen
|
||||
|
||||
Memvid is een draagbare, serverloze geheugenlaag dat AI-agenten een volhardend geheugen en snelle herroepingen geeft. Door zijn model-agnostische, multi-modale en het feit dat het volledig offline werkt, gebruiken ontwikkelaars het over een wijd scala aan real-world applicaties.
|
||||
|
||||
- Lang werkende AI-agenten
|
||||
- Kennisbanken voor ondernemingen
|
||||
- Offline-First AI-systemen
|
||||
- Codebase-begrip
|
||||
- Klantenondersteuningsagenten
|
||||
- Automatisering van de workflow
|
||||
- Verkoop- en marketingcopiloten
|
||||
- Persoonlijke Kennisassistenten
|
||||
- Medische, juridische en financiële adviseurs
|
||||
- Controleerbare en debugbare AI-workflows
|
||||
- Aangepaste toepassingen
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
Gebruik Memvid in je lievelingstaal:
|
||||
|
||||
| Pakket | Installatie | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Installatie (Rust)
|
||||
|
||||
### Benodigdheden
|
||||
|
||||
- **Rust 1.85.0+** — Installeer vanuit [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Voeg dit aan je project toe
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Beschrijving |
|
||||
| ------------------- | ---------------------------------------------- |
|
||||
| `lex` | Full-text search with BM25 ranking (Tantivy) |
|
||||
| `pdf_extract` | Pure Rust PDF text extraction |
|
||||
| `vec` | Vector similarity search (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings for image search |
|
||||
| `whisper` | Audio transcription with Whisper |
|
||||
| `temporal_track` | Natural language date parsing ("last Tuesday") |
|
||||
| `parallel_segments` | Multi-threaded ingestion |
|
||||
| `encryption` | Password-based encryption capsules (.mv2e) |
|
||||
|
||||
Schakel functies in indien nodig:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Build
|
||||
|
||||
Clone de repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
Build in debug modus:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Build in release modus (geoptimaliseerd):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Build with specific features:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tests uitvoeren
|
||||
|
||||
Voer alle tests uit:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Voer tests uit met uitvoer:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Voer een specifieke test uit:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Voer enkel integratie tests uit:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Voorbeelden
|
||||
|
||||
De `examples/` map bedraagd werkende Voorbeelden:
|
||||
|
||||
### Basisgebruik
|
||||
|
||||
Beeldt create, put, search, and timeline operaties uit:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDF Ingestion
|
||||
|
||||
PDF-documenten importeren en doorzoeken (gebruikt de "Attention Is All You Need" paper):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP Visual Search
|
||||
|
||||
Afbeeldingen zoeken met behulp van CLIP-integraties (gebruikt `clip` feature):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Whisper Transcription
|
||||
|
||||
Audio transcripties (gebruikt `whisper` feature):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Bestandsformaat
|
||||
|
||||
Alles leeft in één `.mv2` bestand:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Geen `.wal`, `.lock`, `.shm`, of sidecar-bestanden. Ooit.
|
||||
|
||||
Zie [MV2_SPEC.md](MV2_SPEC.md) voor de complete bestandsformaat specificaties.
|
||||
|
||||
---
|
||||
|
||||
## Ondersteuning
|
||||
|
||||
Heb je vragen of feedback?
|
||||
Email: contact@memvid.com
|
||||
|
||||
**Laat een ⭐ om je ondersteuning te tonen**
|
||||
|
||||
---
|
||||
|
||||
## Licentie
|
||||
|
||||
Apache License 2.0 — zie het [LICENSE](LICENSE) bestandvoor details.
|
||||
@@ -0,0 +1,345 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid waa nidaam xusuuseed oo hal fayl ah kaas oo loogu talagalay wakiillada AI (AI agents), lehna soo-celin degdeg ah iyo xusuus fog.</strong><br/>
|
||||
Xusuus joogto ah, la raadin karo, lana qaadan karo, iyadoo aan loo baahnayn database-yo kale.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Noo saar STAR si aad mashruuca u taageerto ⭐️</h2>
|
||||
|
||||
## Waa maxay Memvid?
|
||||
|
||||
Memvid waa nidaam xusuuseed AI oo la qaadan karo kaas oo kuu keydinaya xogtaada, habka raadinta (embeddings), qaabdhismeedka iyo metadata-daba ku ururiya hal fayl oo keliya.
|
||||
|
||||
Halkii aad ka isticmaali lahayd nidaamyada RAG-ga ee adag ama database-yada vector-ka ee ku shaqeeya server-ka, Memvid wuxuu kuu oggolaanayaa inaad xogta si toos ah uga soo ceshato faylka dhexdiisa si aad u degdeg badan.
|
||||
|
||||
Natiijadu waa lakab xusuuseed ka madax-bannaan nooca modelka iyo kaabayaasha (infrastructure-free), kaas oo siiya wakiillada AI(AI agents) xusuus joogto ah oo fog oo ay meel walba u qaadan karaan.
|
||||
|
||||
---
|
||||
|
||||
## Maxay tahay sababta Frames-ka Muuqaalka (Video Frames)?
|
||||
|
||||
Memvid wuxuu dhiirrigelin ka helayaa habka xogta muuqaalka loo kaydiyo (video encoding), isaga oo aan kaydinayn muuqaal balse u **habaynaya xusuusta AI si isku-xiga (sequence) oo aad u hufan oo "Smart Frames".**
|
||||
|
||||
"Smart Frame" waa unug aan isbeddelayn oo kaydiya macluumaadka oo ay la socdaan waqtiga (timestamps), checksums iyo metadata aasaasi ah. Frames-ka waxaa loo ururiyaa qaab oggolaanaya isku-duubni (compression), tusmeyn (indexing), iyo akhris is-barbar-socda oo hufan.
|
||||
|
||||
Qaabdhismeedkan ku salaysan frames-ka wuxuu suuragelinayaa:
|
||||
|
||||
- Qoraal kaliya oo lagu darayo (Append-only) iyadoo aan la beddelayn ama la kharribayn xogta jirtay
|
||||
- Baaritaan lagu sameyn karo xaaladihii xusuusta ee hore
|
||||
- Kormeeridda habka ay aqoontu u kobcayso iyadoo loo eegayo waqtiga
|
||||
- Badbaadada xogta (crash safety) iyadoo la adeegsanayo frames go'an oo aan isbeddelayn
|
||||
- Isku-duubni hufan oo loo adeegsanayo farsamooyin laga soo minguuriyay kaydinta muuqaallada
|
||||
|
||||
Natiijadu waa hal fayl oo u dhaqmaya sidii jadwal xusuuseed oo dib loo celin karo oo loogu talagalay nidaamyada AI.
|
||||
|
||||
---
|
||||
|
||||
## Fikradaha Muhiimka ah
|
||||
|
||||
- **Living Memory Engine**
|
||||
Si joogto ah ugu dar, u laameey (branch), una kobci xusuusta qeybo kala duwan.
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
Capsule xusuuseed oo isku-filan, la wadaagi karo, lehna sharciyo iyo waqti dhicitaan.
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
Dib u celi, ama qabeey xaalad kasta oo xusuusta ah.
|
||||
|
||||
- **Smart Recall**
|
||||
Soo-celinta xusuusta gudaha wax ka yar 5ms iyadoo la adeegsanayo kaydinta saadaalinta (predictive caching).
|
||||
|
||||
- **Codec Intelligence**
|
||||
Si otomaatig ah u doorta una casriyeeya isku-duubnida (compression) waqtiga ka dib.
|
||||
|
||||
---
|
||||
|
||||
## Meelaha loo adeegsado (Use Cases)
|
||||
|
||||
Memvid waa nidaam xusuuseed oo la qaadan karo oo aan server u baahnayn, kaas oo siiya wakiillada AI(AI agents), xusuus joogto ah iyo soo-celin degdeg ah. Maadaama uu ka madax-bannaan yahay modelka, waxna ku akhriyo qaabab badan (multi-modal), una shaqeeyo si buuxda isagoo aan internet lahayn, horumariyayaashu waxay Memvid u adeegsanayaan hawlo badan:
|
||||
|
||||
- Wakiillada AI(AI Agents) ee muddada dheer shaqeeya
|
||||
- Keydka aqoonta ee shirkadaha
|
||||
- Tageerida AI-ga ee ku shaqeeya offline-ka
|
||||
- Fahamka nidaamyada koodhka (Codebase)
|
||||
- Wakiillada adeegga macmiilka
|
||||
- Otomaatigga shaqada (Workflow Automation)
|
||||
- Kaaliyayaasha iibka iyo suuqgeynta
|
||||
- Kaaliyayaasha aqoonta shakhsi ahaaneed
|
||||
- Wakiillada caafimaadka, sharciga, iyo maaliyadda
|
||||
- Hannaanka shaqada AI-ga oo la baari karo lana saxi karo (Auditable/Debuggable)
|
||||
- Codsiyada gaarka ah (Custom Applications)
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
Ku isticmaal Memvid luuqadda aad doorbidi lahayd:
|
||||
|
||||
| Package | Install | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Kushubashada (Installation) (Rust)
|
||||
|
||||
### Shuruudaha
|
||||
|
||||
- **Rust 1.85.0+** — Si aad ugu shubato Guji linkagan [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Ku dar Mashruucaaga
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Sharaxada |
|
||||
| ------------------- | ---------------------------------------------- |
|
||||
| `lex` | Raadinta qoraalka oo dhan oo leh darajada BM25 (Tantivy) |
|
||||
| `pdf_extract` | Soo saarista qoraalka PDF oo saafi ah |
|
||||
| `vec` | Raadinta isku-midka ah ee Vector (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings oo loogu talagalay raadinta sawirka |
|
||||
| `whisper` | Beddelka codka iyadoo loo baddelayo qoraal lana adeegsanayo Whisper |
|
||||
| `temporal_track` | Turjumidda taariikhda ee luuqadda caadiga ah ("Salaasadii hore") |
|
||||
| `parallel_segments` | Soo gelinta xogta iyadoo la adeegsanayo dhowr nuuc (Multi-threaded) |
|
||||
| `encryption` | capsules-ka xusuusta ee ku xidhan sirta (password) (.mv2e) |
|
||||
|
||||
U furo sifooyinka (features) sida aad ugu baahan tahay:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Bilow Degdeg ah (Quick Start)
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Dhisid (Build)
|
||||
|
||||
Qeyb ka soo qaado (Clone) kaydka (Repository):
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
U dhis habka cilad-baadhista (debug mode):
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
U dhis habka rasmiga ah (release mode - la hagaajiyay):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Ku dhis sifooyin (features) gaar ah:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tijaabi iskudayga (Run Tests)
|
||||
|
||||
Tijaabi iskudayada oo dhan:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Tijaabi iskudayga iyadoo natiijada la arkayo:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Tijaab iskuday gaar ah:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Tijaabi iskudayada isku-xirka (integration tests) ah oo keliya
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tusaalooyin (Examples)
|
||||
|
||||
Tusaha `examples/` wuxuu ka kooban yahay tusaalooyin shaqaynaya:
|
||||
|
||||
### Adeegsiga Fudud
|
||||
|
||||
Wuxuu muujinayaa samaynta, gelinta, raadinta, iyo hawlgallada waqtiga (timeline):
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Soo gelinta PDF
|
||||
|
||||
Geli oo baadh dukumiintiyada PDF-ka ah (wuxuu isticmaalaa warqaddii aheyd "Attention Is All You Need"):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### Raadinta Muuqaalka ee CLIP
|
||||
|
||||
Raadinta sawirka iyadoo la adeegsanayo CLIP (waxay u baahan tahay clip feature):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Beddelka Codka ee Whisper
|
||||
|
||||
Beddelka codka (waxay u baahan tahay whisper feature):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Qaabka Faylka (File Format)
|
||||
|
||||
Wax walba waxay ku jiraan hal fayl oo .mv2 ah:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, nooca, awoodda
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Kasoo kabashada burburka
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Frames la isku-duubay
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy qoraal ka buuxo
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Siday u kala horreeyaan
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Meeqaamka qaybaha
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Ma jiraan faylal .wal, .lock, .shm, ama faylal dhinac socda. Weligaa.
|
||||
|
||||
Fiiri [MV2_SPEC.md](MV2_SPEC.md) si aad u hesho faahfaahinta dhammaystiran ee qaabka faylka.
|
||||
|
||||
---
|
||||
|
||||
## Taageer (Support)
|
||||
|
||||
Ma qabtaa su'aalo ama ra'yi?
|
||||
Email: contact@memvid.com
|
||||
|
||||
**Noo saar ⭐ si aad u muujiso taageeradaada**
|
||||
|
||||
---
|
||||
|
||||
## Shatiga (License)
|
||||
|
||||
Apache License 2.0 — Fiiri faylka [LICENSE](LICENSE) si aad u hesho faahfaahin dheeraad ah.
|
||||
@@ -0,0 +1,517 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)"
|
||||
src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<div style="height: 16px;"></div>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid 是专为 AI 智能体设计的单文件记忆层，具备即时检索和长期记忆能力。</strong><br/>
|
||||
持久化、版本化、可移植的记忆，无需数据库。
|
||||
</p>
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">官方网站</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">尝试一下沙箱</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">文档</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">讨论区</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="许可证" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
|
||||
|
||||
<h2 align="center">⭐️ 给项目点个星标支持我们 ⭐️</h2>
|
||||
|
||||
## 基准测试亮点
|
||||
|
||||
**🚀 准确率超越其他记忆系统：** 在 LoCoMo 上领先 SOTA 35%，长时对话回忆与推理能力最佳
|
||||
|
||||
**🧠 卓越的多跳与时序推理：** 比行业平均水平高出 76% 多跳推理，56% 时序推理
|
||||
|
||||
**⚡ 超低延迟高吞吐：** P50 仅 0.025ms，P99 仅 0.075ms，吞吐量是标准的 1,372 倍
|
||||
|
||||
**🔬 完全可复现的基准测试：** LoCoMo（10 次约 26K token 的对话）、开源评估、LLM-as-Judge
|
||||
|
||||
|
||||
## 什么是 Memvid？
|
||||
|
||||
Memvid 是可移植的 AI 记忆系统，将数据、嵌入向量、搜索结构和元数据打包成单个文件。
|
||||
|
||||
无需运行复杂的 RAG 管道或基于服务器的向量数据库，Memvid 支持直接从文件进行快速检索。
|
||||
|
||||
结果是模型无关、无基础设施的记忆层，让 AI 智能体拥有可随身携带的持久化长期记忆。
|
||||
|
||||
|
||||
## 什么是 Smart Frames？
|
||||
|
||||
Memvid 借鉴视频编码的理念，不是为了存储视频，而是**将 AI 记忆组织为仅追加、超高效序列的 Smart Frames。**
|
||||
|
||||
Smart Frame 是存储内容以及时间戳、校验和和基本元数据的不可变单元。
|
||||
帧以允许高效压缩、索引和并行读取的方式分组。
|
||||
|
||||
这种基于帧的设计支持：
|
||||
|
||||
- 仅追加写入，不修改或破坏现有数据
|
||||
- 对过去记忆状态的查询
|
||||
- 知识演化的时间轴式检查
|
||||
- 通过基于提交的不可变帧应对崩溃
|
||||
- 使用基于视频编码技术的高效压缩
|
||||
|
||||
结果是一个表现为 AI 系统可追溯记忆时间线的单文件。
|
||||
|
||||
|
||||
## 核心概念
|
||||
|
||||
- **Living Memory Engine**
|
||||
持续追加、分支和跨会话演进记忆。
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
自包含、可共享的记忆胶囊，带规则和过期时间。
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
回溯、重放或分支化任何记忆状态。
|
||||
|
||||
- **Smart Recall**
|
||||
小于 5ms 本地记忆访问，具备预测性缓存。
|
||||
|
||||
- **Codec Intelligence**
|
||||
随时间自动选择和升级压缩。
|
||||
|
||||
|
||||
## 使用场景
|
||||
|
||||
Memvid 是无服务器便携记忆层，为 AI 智能体提供持久记忆和快速召回。由于它是模型无关、多模态且完全离线工作，开发者正在各种现实应用中广泛的使用 Memvid。
|
||||
|
||||
- 长期运行的 AI 智能体
|
||||
- 企业知识库
|
||||
- 离线优先 AI 系统
|
||||
- 代码库理解
|
||||
- 客户支持智能体
|
||||
- 工作流自动化
|
||||
- 销售与营销助手
|
||||
- 个人知识助理
|
||||
- 医疗、法律和金融智能体
|
||||
- 可审计和可调试的 AI 工作流
|
||||
- 自定义应用
|
||||
|
||||
|
||||
## SDKs 与 CLI
|
||||
|
||||
在您喜欢的语言中使用 Memvid：
|
||||
|
||||
| 包 | 安装 | 链接 |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## 安装（Rust）
|
||||
|
||||
### 要求
|
||||
|
||||
- **Rust 1.85.0+** — 从 [rustup.rs](https://rustup.rs) 安装
|
||||
|
||||
### 添加到项目
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### 功能标志
|
||||
|
||||
| 功能 | 描述 |
|
||||
| -------------------- | ---------------------------------------------------------------- |
|
||||
| `lex` | 使用 BM25 排序的全文搜索（Tantivy） |
|
||||
| `pdf_extract` | 纯 Rust PDF 文本提取 |
|
||||
| `vec` | 向量相似搜索（HNSW + 通过 ONNX 的本地文本嵌入） |
|
||||
| `clip` | CLIP 视觉嵌入用于图像搜索 |
|
||||
| `whisper` | 使用 Whisper 进行音频转录 |
|
||||
| `api_embed` | 云 API 嵌入（OpenAI） |
|
||||
| `temporal_track` | 自然语言日期解析（"last Tuesday"） |
|
||||
| `parallel_segments` | 多线程摄取 |
|
||||
| `encryption` | 基于密码的加密胶囊（.mv2e） |
|
||||
| `symspell_cleanup` | 强大的 PDF 文本修复（修复 "emp lo yee" -> "employee"） |
|
||||
|
||||
按需启用功能：
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
|
||||
## 快速开始
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// 创建新记忆文件
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// 添加带元数据的文档
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// 搜索
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 构建
|
||||
|
||||
克隆仓库：
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
以调试模式构建：
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
以发布模式构建（优化）：
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
使用特定功能构建：
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 运行测试
|
||||
|
||||
运行所有测试：
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
带输出运行测试：
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
运行特定测试：
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
仅运行集成测试：
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 示例
|
||||
|
||||
`examples/` 目录包含可运行示例：
|
||||
|
||||
### 基本用法
|
||||
|
||||
演示创建、添加、搜索和时间线操作：
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDF 提取
|
||||
|
||||
提取和搜索 PDF 文档（使用 "Attention Is All You Need" 论文）：
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP 可视化搜索
|
||||
|
||||
使用 CLIP 嵌入进行图像搜索（需要 `clip` 功能）：
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Whisper 转录
|
||||
|
||||
音频转录（需要 `whisper` 功能）：
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper -- /path/to/audio.mp3
|
||||
```
|
||||
|
||||
**可用模型：**
|
||||
|
||||
| 模型 | 大小 | 速度 | 用例 |
|
||||
| ---------------------- | ------ | ------- | ----------------------------------- |
|
||||
| `whisper-small-en` | 244 MB | 最慢 | 最佳准确度（默认） |
|
||||
| `whisper-tiny-en` | 75 MB | 快 | 平衡 |
|
||||
| `whisper-tiny-en-q8k` | 19 MB | 最快 | 快速测试，资源受限 |
|
||||
|
||||
**模型选择：**
|
||||
|
||||
```bash
|
||||
# 默认（FP32 small，最高准确度）
|
||||
cargo run --example test_whisper --features whisper -- audio.mp3
|
||||
|
||||
# 小型量化（小 75%，更快）
|
||||
MEMVID_WHISPER_MODEL=whisper-tiny-en-q8k cargo run --example test_whisper --features whisper -- audio.mp3
|
||||
```
|
||||
|
||||
**可编程配置：**
|
||||
|
||||
```rust
|
||||
use memvid_core::{WhisperConfig, WhisperTranscriber};
|
||||
|
||||
// 默认 FP32 small 模型
|
||||
let config = WhisperConfig::default();
|
||||
|
||||
// 小型量化模型（更快，更小）
|
||||
let config = WhisperConfig::with_quantization();
|
||||
|
||||
// 特定模型
|
||||
let config = WhisperConfig::with_model("whisper-tiny-en-q8k");
|
||||
|
||||
let transcriber = WhisperTranscriber::new(&config)?;
|
||||
let result = transcriber.transcribe_file("audio.mp3")?;
|
||||
println!("{}", result.text);
|
||||
```
|
||||
|
||||
|
||||
## 文本嵌入模型
|
||||
|
||||
`vec` 功能包括使用 ONNX 模型的本地文本嵌入支持。在使用本地文本嵌入之前，您需要手动下载模型文件。
|
||||
|
||||
### 快速开始：BGE-small（推荐）
|
||||
|
||||
下载默认 BGE-small 模型（384 维，快速高效）：
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# 下载 ONNX 模型
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# 下载分词器
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### 可用模型
|
||||
|
||||
| 模型 | 维度 | 大小 | 最适合 |
|
||||
| ------------------------ | ---------- | ------ | --------------- |
|
||||
| `bge-small-en-v1.5` | 384 | ~120MB | 默认，快速 |
|
||||
| `bge-base-en-v1.5` | 768 | ~420MB | 更好的质量 |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530MB | 多用途任务 |
|
||||
| `gte-large` | 1024 | ~1.3GB | 最高质量 |
|
||||
|
||||
### 其他模型
|
||||
|
||||
**BGE-base**（768 维）：
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic**（768 维）：
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large**（1024 维）：
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### 在代码中使用
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// 使用默认模型（BGE-small）
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// 使用不同模型
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
有关相似度计算和搜索排名的完整示例，请参见 `examples/text_embedding.rs`。
|
||||
|
||||
### 模型一致性
|
||||
|
||||
为防止意外地模型混合（例如，使用 OpenAI 嵌入查询 BGE-small 索引），您可以将 Memvid 实例显式绑定到特定模型名称：
|
||||
|
||||
```rust
|
||||
// 将索引绑定到特定模型。
|
||||
// 如果之前使用不同模型创建索引将返回错误。
|
||||
mem.set_vec_model("bge-small-en-v1.5")?;
|
||||
```
|
||||
|
||||
绑定是持久化的。一旦设置，将来尝试使用不同模型名称将快速失败并返回 `ModelMismatch` 错误。
|
||||
|
||||
|
||||
|
||||
## API 嵌入（OpenAI）
|
||||
|
||||
`api_embed` 功能使用 OpenAI 的 API 启用基于云的嵌入生成。
|
||||
|
||||
### 设置
|
||||
|
||||
设置您的 OpenAI API 密钥：
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY="sk-..."
|
||||
```
|
||||
|
||||
### 用法
|
||||
|
||||
```rust
|
||||
use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// 使用默认模型（text-embedding-3-small）
|
||||
let config = OpenAIConfig::default();
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 1536);
|
||||
|
||||
// 使用更高质量模型
|
||||
let config = OpenAIConfig::large(); // text-embedding-3-large (3072 维)
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
### 可用模型
|
||||
|
||||
| 模型 | 维度 | 最适合 |
|
||||
| ------------------------ | ---------- | -------------------------- |
|
||||
| `text-embedding-3-small` | 1536 | 默认，最快，最便宜 |
|
||||
| `text-embedding-3-large` | 3072 | 最高质量 |
|
||||
| `text-embedding-ada-002` | 1536 | 传统模型 |
|
||||
|
||||
有关完整示例，请参见 `examples/openai_embedding.rs`。
|
||||
|
||||
|
||||
|
||||
## 文件格式
|
||||
|
||||
所有内容都存储在单个 `.mv2` 文件中：
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ 魔数，版本，容量
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ 崩溃恢复
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ 压缩帧
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy 全文
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW 向量
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ 时序排序
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ 段偏移
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
不会有 `.wal`、`.lock`、`.shm` 或附带文件。永远不会。
|
||||
|
||||
有关完整文件格式规范，请参见 [MV2_SPEC.md](MV2_SPEC.md)。
|
||||
|
||||
|
||||
|
||||
## 支持
|
||||
|
||||
有问题或反馈？
|
||||
邮箱：contact@memvid.com
|
||||
|
||||
**点个 ⭐ 支持我们**
|
||||
|
||||
---
|
||||
|
||||
> **Memvid v1（基于 QR 码的记忆）已弃用**
|
||||
>
|
||||
> 如果您参考的是 QR 码，那么您正在使用过时信息。
|
||||
>
|
||||
> 参见：https://docs.memvid.com/memvid-v1-deprecation
|
||||
|
||||
---
|
||||
|
||||
## 许可证
|
||||
|
||||
Apache License 2.0 — 详细信息请参见 [LICENSE](LICENSE) 文件。
|
||||
@@ -0,0 +1,249 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const I18N_DIR = path.join(__dirname, '..');
|
||||
const ROOT_DIR = path.join(I18N_DIR, '..', '..');
|
||||
const README_PATH = path.join(ROOT_DIR, 'README.md');
|
||||
|
||||
const LANG_MAP = {
|
||||
'aa': { emoji: '🌐', name: 'Afar' },
|
||||
'ab': { emoji: '🌐', name: 'Abkhazian' },
|
||||
'ae': { emoji: '🌐', name: 'Avestan' },
|
||||
'af': { emoji: '🇿🇦', name: 'Afrikaans' },
|
||||
'ak': { emoji: '🌐', name: 'Akan' },
|
||||
'am': { emoji: '🇪🇹', name: 'Amharic' },
|
||||
'an': { emoji: '🌐', name: 'Aragonese' },
|
||||
'ar': { emoji: '🇸🇦', name: 'العربية' },
|
||||
'as': { emoji: '🌐', name: 'Assamese' },
|
||||
'av': { emoji: '🌐', name: 'Avaric' },
|
||||
'ay': { emoji: '🌐', name: 'Aymara' },
|
||||
'az': { emoji: '🇦🇿', name: 'Azerbaijani' },
|
||||
'ba': { emoji: '🌐', name: 'Bashkir' },
|
||||
'be': { emoji: '🇧🇾', name: 'Belarusian' },
|
||||
'bg': { emoji: '🇧🇬', name: 'Bulgarian' },
|
||||
'bi': { emoji: '🌐', name: 'Bislama' },
|
||||
'bm': { emoji: '🌐', name: 'Bambara' },
|
||||
'bn': { emoji: '🇧🇩', name: 'বাংলা' },
|
||||
'bo': { emoji: '🌐', name: 'Tibetan' },
|
||||
'br': { emoji: '🌐', name: 'Breton' },
|
||||
'bs': { emoji: '🇧🇦', name: 'Bosnian' },
|
||||
'ca': { emoji: '🇪🇸', name: 'Catalan' },
|
||||
'ce': { emoji: '🌐', name: 'Chechen' },
|
||||
'ch': { emoji: '🌐', name: 'Chamorro' },
|
||||
'co': { emoji: '🌐', name: 'Corsican' },
|
||||
'cr': { emoji: '🌐', name: 'Cree' },
|
||||
'cs': { emoji: '🇨🇿', name: 'Česko' },
|
||||
'cu': { emoji: '🌐', name: 'Church Slavonic' },
|
||||
'cv': { emoji: '🌐', name: 'Chuvash' },
|
||||
'cy': { emoji: '🇬🇧', name: 'Welsh' },
|
||||
'da': { emoji: '🇩🇰', name: 'Danish' },
|
||||
'de': { emoji: '🇩🇪', name: 'Deutsch' },
|
||||
'dv': { emoji: '🌐', name: 'Divehi' },
|
||||
'dz': { emoji: '🇧🇹', name: 'Dzongkha' },
|
||||
'ee': { emoji: '🌐', name: 'Ewe' },
|
||||
'el': { emoji: '🇬🇷', name: 'Greek' },
|
||||
'en': { emoji: '🇺🇸', name: 'English' },
|
||||
'eo': { emoji: '🌐', name: 'Esperanto' },
|
||||
'es': { emoji: '🇪🇸', name: 'Español' },
|
||||
'et': { emoji: '🇪🇪', name: 'Estonian' },
|
||||
'eu': { emoji: '🌐', name: 'Basque' },
|
||||
'fa': { emoji: '🇮🇷', name: 'Persian' },
|
||||
'ff': { emoji: '🌐', name: 'Fulah' },
|
||||
'fi': { emoji: '🇫🇮', name: 'Finnish' },
|
||||
'fj': { emoji: '🌐', name: 'Fijian' },
|
||||
'fo': { emoji: '🌐', name: 'Faroese' },
|
||||
'fr': { emoji: '🇫🇷', name: 'Français' },
|
||||
'fy': { emoji: '🌐', name: 'Western Frisian' },
|
||||
'ga': { emoji: '🇮🇪', name: 'Irish' },
|
||||
'gd': { emoji: '🌐', name: 'Gaelic' },
|
||||
'gl': { emoji: '🌐', name: 'Galician' },
|
||||
'gn': { emoji: '🌐', name: 'Guarani' },
|
||||
'gu': { emoji: '🌐', name: 'Gujarati' },
|
||||
'gv': { emoji: '🌐', name: 'Manx' },
|
||||
'ha': { emoji: '🇳🇬', name: 'Hausa' },
|
||||
'he': { emoji: '🇮🇱', name: 'Hebrew' },
|
||||
'hi': { emoji: '🇮🇳', name: 'हिन्दी' },
|
||||
'ho': { emoji: '🌐', name: 'Hiri Motu' },
|
||||
'hr': { emoji: '🇭🇷', name: 'Croatian' },
|
||||
'ht': { emoji: '🌐', name: 'Haitian' },
|
||||
'hu': { emoji: '🇭🇺', name: 'Hungarian' },
|
||||
'hy': { emoji: '🇦🇲', name: 'Armenian' },
|
||||
'hz': { emoji: '🌐', name: 'Herero' },
|
||||
'ia': { emoji: '🌐', name: 'Interlingua' },
|
||||
'id': { emoji: '🇮🇩', name: 'Bahasa' },
|
||||
'ie': { emoji: '🌐', name: 'Interlingue' },
|
||||
'ig': { emoji: '🇳🇬', name: 'Igbo' },
|
||||
'ii': { emoji: '🌐', name: 'Sichuan Yi' },
|
||||
'ik': { emoji: '🌐', name: 'Inupiaq' },
|
||||
'io': { emoji: '🌐', name: 'Ido' },
|
||||
'is': { emoji: '🇮🇸', name: 'Icelandic' },
|
||||
'it': { emoji: '🇮🇹', name: 'Italiano' },
|
||||
'iu': { emoji: '🌐', name: 'Inuktitut' },
|
||||
'ja': { emoji: '🇯🇵', name: '日本語' },
|
||||
'jv': { emoji: '🌐', name: 'Javanese' },
|
||||
'ka': { emoji: '🇬🇪', name: 'Georgian' },
|
||||
'kg': { emoji: '🌐', name: 'Kongo' },
|
||||
'ki': { emoji: '🌐', name: 'Kikuyu' },
|
||||
'kj': { emoji: '🌐', name: 'Kuanyama' },
|
||||
'kk': { emoji: '🇰🇿', name: 'Kazakh' },
|
||||
'kl': { emoji: '🌐', name: 'Kalaallisut' },
|
||||
'km': { emoji: '🇰🇭', name: 'Central Khmer' },
|
||||
'kn': { emoji: '🌐', name: 'Kannada' },
|
||||
'ko': { emoji: '🇰🇷', name: '한국어' },
|
||||
'kr': { emoji: '🌐', name: 'Kanuri' },
|
||||
'ks': { emoji: '🌐', name: 'Kashmiri' },
|
||||
'ku': { emoji: '🇮🇶', name: 'Kurdish' },
|
||||
'kv': { emoji: '🌐', name: 'Komi' },
|
||||
'kw': { emoji: '🌐', name: 'Cornish' },
|
||||
'ky': { emoji: '🇰🇬', name: 'Kyrgyz' },
|
||||
'la': { emoji: '🌐', name: 'Latin' },
|
||||
'lb': { emoji: '🌐', name: 'Luxembourgish' },
|
||||
'lg': { emoji: '🌐', name: 'Ganda' },
|
||||
'li': { emoji: '🌐', name: 'Limburgan' },
|
||||
'ln': { emoji: '🌐', name: 'Lingala' },
|
||||
'lo': { emoji: '🇱🇦', name: 'Lao' },
|
||||
'lt': { emoji: '🇱🇹', name: 'Lithuanian' },
|
||||
'lu': { emoji: '🌐', name: 'Luba-Katanga' },
|
||||
'lv': { emoji: '🇱🇻', name: 'Latvian' },
|
||||
'mg': { emoji: '🌐', name: 'Malagasy' },
|
||||
'mh': { emoji: '🌐', name: 'Marshallese' },
|
||||
'mi': { emoji: '🌐', name: 'Maori' },
|
||||
'mk': { emoji: '🇲🇰', name: 'Macedonian' },
|
||||
'ml': { emoji: '🌐', name: 'Malayalam' },
|
||||
'mn': { emoji: '🇲🇳', name: 'Mongolian' },
|
||||
'mr': { emoji: '🌐', name: 'Marathi' },
|
||||
'ms': { emoji: '🇲🇾', name: 'Malay' },
|
||||
'mt': { emoji: '🇲🇹', name: 'Maltese' },
|
||||
'my': { emoji: '🇲🇲', name: 'Burmese' },
|
||||
'na': { emoji: '🌐', name: 'Nauru' },
|
||||
'nb': { emoji: '🌐', name: 'Norwegian Bokmål' },
|
||||
'nd': { emoji: '🌐', name: 'North Ndebele' },
|
||||
'ne': { emoji: '🇳🇵', name: 'Nepali' },
|
||||
'ng': { emoji: '🌐', name: 'Ndonga' },
|
||||
'nl': { emoji: '🇧🇪/🇳🇱', name: 'Nederlands' },
|
||||
'nn': { emoji: '🌐', name: 'Norwegian Nynorsk' },
|
||||
'no': { emoji: '🇳🇴', name: 'Norwegian' },
|
||||
'nr': { emoji: '🌐', name: 'South Ndebele' },
|
||||
'nv': { emoji: '🌐', name: 'Navajo' },
|
||||
'ny': { emoji: '🌐', name: 'Chichewa' },
|
||||
'oc': { emoji: '🌐', name: 'Occitan' },
|
||||
'oj': { emoji: '🌐', name: 'Ojibwa' },
|
||||
'om': { emoji: '🌐', name: 'Oromo' },
|
||||
'or': { emoji: '🌐', name: 'Oriya' },
|
||||
'os': { emoji: '🌐', name: 'Ossetian' },
|
||||
'pa': { emoji: '🌐', name: 'Punjabi' },
|
||||
'pi': { emoji: '🌐', name: 'Pali' },
|
||||
'pl': { emoji: '🇵��', name: 'Polski' },
|
||||
'ps': { emoji: '🇦🇫', name: 'Pashto' },
|
||||
'pt': { emoji: '🇵🇹', name: 'Português' },
|
||||
'qu': { emoji: '🌐', name: 'Quechua' },
|
||||
'rm': { emoji: '🌐', name: 'Romansh' },
|
||||
'rn': { emoji: '🌐', name: 'Rundi' },
|
||||
'ro': { emoji: '🇷🇴', name: 'Romanian' },
|
||||
'ru': { emoji: '🇷🇺', name: 'Русский' },
|
||||
'rw': { emoji: '🌐', name: 'Kinyarwanda' },
|
||||
'sa': { emoji: '🌐', name: 'Sanskrit' },
|
||||
'sc': { emoji: '🌐', name: 'Sardinian' },
|
||||
'sd': { emoji: '🌐', name: 'Sindhi' },
|
||||
'se': { emoji: '🌐', name: 'Northern Sami' },
|
||||
'sg': { emoji: '🌐', name: 'Sango' },
|
||||
'si': { emoji: '🇱🇰', name: 'Sinhala' },
|
||||
'sk': { emoji: '🇸🇰', name: 'Slovak' },
|
||||
'sl': { emoji: '🇸🇮', name: 'Slovenian' },
|
||||
'sm': { emoji: '🌐', name: 'Samoan' },
|
||||
'sn': { emoji: '🌐', name: 'Shona' },
|
||||
'so': { emoji: '🇸🇴', name: 'Soomaali' },
|
||||
'sq': { emoji: '🇦🇱', name: 'Albanian' },
|
||||
'sr': { emoji: '🇷🇸', name: 'Serbian' },
|
||||
'ss': { emoji: '🌐', name: 'Swati' },
|
||||
'st': { emoji: '🇿🇦', name: 'Southern Sotho' },
|
||||
'su': { emoji: '🌐', name: 'Sundanese' },
|
||||
'sv': { emoji: '🇸🇪', name: 'Swedish' },
|
||||
'sw': { emoji: '🇰🇪', name: 'Swahili' },
|
||||
'ta': { emoji: '🌐', name: 'Tamil' },
|
||||
'te': { emoji: '🌐', name: 'Telugu' },
|
||||
'tg': { emoji: '🇹🇯', name: 'Tajik' },
|
||||
'th': { emoji: '🇹🇭', name: 'Thai' },
|
||||
'ti': { emoji: '🌐', name: 'Tigrinya' },
|
||||
'tk': { emoji: '🇹🇲', name: 'Turkmen' },
|
||||
'tl': { emoji: '🇵🇭', name: 'Tagalog' },
|
||||
'tn': { emoji: '🌐', name: 'Tswana' },
|
||||
'to': { emoji: '🌐', name: 'Tonga' },
|
||||
'tr': { emoji: '🇹🇷', name: 'Türkçe' },
|
||||
'ts': { emoji: '🌐', name: 'Tsonga' },
|
||||
'tt': { emoji: '🌐', name: 'Tatar' },
|
||||
'tw': { emoji: '🌐', name: 'Twi' },
|
||||
'ty': { emoji: '🌐', name: 'Tahitian' },
|
||||
'ug': { emoji: '🌐', name: 'Uighur' },
|
||||
'uk': { emoji: '🇺🇦', name: 'Ukrainian' },
|
||||
'ur': { emoji: '🇵🇰', name: 'Urdu' },
|
||||
'uz': { emoji: '🇺🇿', name: 'Uzbek' },
|
||||
've': { emoji: '🌐', name: 'Venda' },
|
||||
'vi': { emoji: '🇻🇳', name: 'Tiếng Việt' },
|
||||
'vo': { emoji: '🌐', name: 'Volapük' },
|
||||
'wa': { emoji: '🌐', name: 'Walloon' },
|
||||
'wo': { emoji: '🌐', name: 'Wolof' },
|
||||
'xh': { emoji: '🇿🇦', name: 'Xhosa' },
|
||||
'yi': { emoji: '🌐', name: 'Yiddish' },
|
||||
'yo': { emoji: '🇳🇬', name: 'Yoruba' },
|
||||
'za': { emoji: '🌐', name: 'Zhuang' },
|
||||
'zh': { emoji: '🇨🇳', name: '中文' },
|
||||
'zh-CN': { emoji: '🇨🇳', name: '中文 (简体)' },
|
||||
'zh-HK': { emoji: '🇭🇰', name: '中文 (繁體)' },
|
||||
'zh-Hans': { emoji: '🇨🇳', name: '中文 (简体)' },
|
||||
'zh-Hant': { emoji: '🇹🇼', name: '中文 (繁體)' },
|
||||
'zh-MO': { emoji: '🇲🇴', name: '中文 (繁體)' },
|
||||
'zh-SG': { emoji: '🇸🇬', name: '中文 (繁體)' },
|
||||
'zh-TW': { emoji: '🇹🇼', name: '中文 (繁體)' },
|
||||
'zu': { emoji: '🇿🇦', name: 'Zulu' },
|
||||
};
|
||||
|
||||
function autoAddFlags() {
|
||||
if (!fs.existsSync(README_PATH)) {
|
||||
console.error('Error: Cannot find ' + README_PATH);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let readmeContent = fs.readFileSync(README_PATH, 'utf-8');
|
||||
const marker = ' <!-- Next Flag -->';
|
||||
|
||||
if (!readmeContent.includes(marker)) {
|
||||
console.warn('Error: <!-- Next Flag --> marker not found in ' + README_PATH);
|
||||
console.warn('Please add the marker where you want new flags to be inserted.');
|
||||
return;
|
||||
}
|
||||
|
||||
const files = fs.readdirSync(I18N_DIR);
|
||||
|
||||
const translationFiles = files.filter(f =>
|
||||
f.startsWith('README') &&
|
||||
f.endsWith('.md') &&
|
||||
f !== 'README.md'
|
||||
);
|
||||
|
||||
let updated = false;
|
||||
|
||||
translationFiles.forEach(file => {
|
||||
const code = file.split('.')[1];
|
||||
const lang = LANG_MAP[code];
|
||||
|
||||
if (!lang) return;
|
||||
|
||||
const flagLink = ` <a href="docs/i18n/${file}">${lang.emoji} ${lang.name}</a>`;
|
||||
|
||||
if (!readmeContent.includes(file)) {
|
||||
readmeContent = readmeContent.replace(marker, flagLink + '\n' + marker);
|
||||
updated = true;
|
||||
console.log('Added ' + code);
|
||||
}
|
||||
});
|
||||
|
||||
if (updated) {
|
||||
fs.writeFileSync(README_PATH, readmeContent, 'utf-8');
|
||||
console.log('Main README updated.');
|
||||
} else {
|
||||
console.log('No new flags to add.');
|
||||
}
|
||||
}
|
||||
|
||||
autoAddFlags();
|
||||
@@ -0,0 +1,114 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const I18N_DIR = path.join(__dirname, '..');
|
||||
const ROOT_DIR = path.join(I18N_DIR, '..', '..');
|
||||
const README_PATH = path.join(ROOT_DIR, 'README.md');
|
||||
|
||||
const MARKERS = {
|
||||
HEADER: {
|
||||
START: '<!-- HEADER:START -->',
|
||||
END: '<!-- HEADER:END -->'
|
||||
},
|
||||
FLAGS: {
|
||||
START: '<!-- FLAGS:START -->',
|
||||
END: '<!-- FLAGS:END -->'
|
||||
},
|
||||
NAV: {
|
||||
START: '<!-- NAV:START -->',
|
||||
END: '<!-- NAV:END -->'
|
||||
},
|
||||
BADGES: {
|
||||
START: '<!-- BADGES:START -->',
|
||||
END: '<!-- BADGES:END -->'
|
||||
}
|
||||
};
|
||||
|
||||
function extractBlock(content, marker) {
|
||||
const startIdx = content.indexOf(marker.START);
|
||||
const endIdx = content.indexOf(marker.END);
|
||||
|
||||
if (startIdx === -1 || endIdx === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return content.substring(startIdx, endIdx + marker.END.length);
|
||||
}
|
||||
|
||||
function updateOrInsertBlock(targetContent, blockKey, blockValue, fallbackAnchor = null) {
|
||||
const marker = MARKERS[blockKey];
|
||||
const startIdx = targetContent.indexOf(marker.START);
|
||||
const endIdx = targetContent.indexOf(marker.END);
|
||||
|
||||
if (startIdx !== -1 && endIdx !== -1) {
|
||||
const prefix = targetContent.substring(0, startIdx);
|
||||
const suffix = targetContent.substring(endIdx + marker.END.length);
|
||||
return prefix + blockValue + suffix;
|
||||
}
|
||||
|
||||
if (startIdx !== -1 || endIdx !== -1) {
|
||||
let cleaned = targetContent.split(marker.START).join('');
|
||||
cleaned = cleaned.split(marker.END).join('');
|
||||
return updateOrInsertBlock(cleaned, blockKey, blockValue, fallbackAnchor);
|
||||
}
|
||||
|
||||
if (fallbackAnchor) {
|
||||
const anchorIdx = targetContent.indexOf(fallbackAnchor);
|
||||
if (anchorIdx !== -1) {
|
||||
const insertAfterIdx = targetContent.indexOf('\n', anchorIdx) + 1;
|
||||
const prefix = targetContent.substring(0, insertAfterIdx);
|
||||
const suffix = targetContent.substring(insertAfterIdx);
|
||||
return prefix + '\n' + blockValue + '\n' + suffix;
|
||||
}
|
||||
}
|
||||
|
||||
return targetContent + '\n\n' + blockValue;
|
||||
}
|
||||
|
||||
function updateLocalizedReadmes() {
|
||||
if (!fs.existsSync(README_PATH)) {
|
||||
console.error('Error: Cannot find ' + README_PATH);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const readmeContent = fs.readFileSync(README_PATH, 'utf-8');
|
||||
|
||||
const headerBlock = extractBlock(readmeContent, MARKERS.HEADER);
|
||||
const flagsBlock = extractBlock(readmeContent, MARKERS.FLAGS);
|
||||
const navBlock = extractBlock(readmeContent, MARKERS.NAV);
|
||||
const badgesBlock = extractBlock(readmeContent, MARKERS.BADGES);
|
||||
|
||||
if (!headerBlock || !flagsBlock || !navBlock || !badgesBlock) {
|
||||
console.error('Error: Some markers are missing in main README.md');
|
||||
if (!headerBlock) console.error('Missing: HEADER');
|
||||
if (!flagsBlock) console.error('Missing: FLAGS');
|
||||
if (!navBlock) console.error('Missing: NAV');
|
||||
if (!badgesBlock) console.error('Missing: BADGES');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let flagsI18n = flagsBlock.split('href="docs/i18n/').join('href="');
|
||||
flagsI18n = flagsI18n.split('href="README.md"').join('href="../../README.md"');
|
||||
|
||||
const files = fs.readdirSync(I18N_DIR);
|
||||
const targetFiles = files
|
||||
.filter(file => file.startsWith('README.') && file.endsWith('.md') && file !== 'README.md')
|
||||
.map(file => path.join(I18N_DIR, file));
|
||||
|
||||
targetFiles.forEach(filePath => {
|
||||
let content = fs.readFileSync(filePath, 'utf-8');
|
||||
const fileName = path.basename(filePath);
|
||||
|
||||
content = updateOrInsertBlock(content, 'HEADER', headerBlock, '<img');
|
||||
content = updateOrInsertBlock(content, 'FLAGS', flagsI18n, MARKERS.HEADER.END);
|
||||
content = updateOrInsertBlock(content, 'NAV', navBlock, MARKERS.FLAGS.END);
|
||||
content = updateOrInsertBlock(content, 'BADGES', badgesBlock, MARKERS.NAV.END);
|
||||
|
||||
fs.writeFileSync(filePath, content, 'utf-8');
|
||||
console.log('Updated ' + fileName);
|
||||
});
|
||||
|
||||
console.log('\nSuccess: Done.');
|
||||
}
|
||||
|
||||
updateLocalizedReadmes();
|
||||
@@ -0,0 +1,179 @@
|
||||
//! Basic usage example demonstrating create, put, find, and timeline operations.
|
||||
//!
|
||||
//! Run with: cargo run --example basic_usage
|
||||
|
||||
use std::path::PathBuf;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use memvid_core::{Memvid, PutOptions, Result, SearchRequest, TimelineQuery};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// Create a temporary directory for our example
|
||||
let dir = tempdir().expect("failed to create temp dir");
|
||||
let path: PathBuf = dir.path().join("example.mv2");
|
||||
|
||||
println!("=== Memvid Core Basic Usage Example ===\n");
|
||||
|
||||
// ========================================
|
||||
// 1. CREATE a new memory file
|
||||
// ========================================
|
||||
println!("1. Creating memory file at {:?}", path);
|
||||
let mut mem = Memvid::create(&path)?;
|
||||
println!(" Memory created successfully!\n");
|
||||
|
||||
// ========================================
|
||||
// 2. PUT documents into the memory
|
||||
// ========================================
|
||||
println!("2. Adding documents to memory...");
|
||||
|
||||
// Simple put with just bytes
|
||||
let seq1 = mem.put_bytes(b"Hello, Memvid! This is a simple text document.")?;
|
||||
println!(" Added document 1, sequence: {}", seq1);
|
||||
|
||||
// Put with options (title, URI, tags)
|
||||
let options = PutOptions::builder()
|
||||
.title("Getting Started Guide")
|
||||
.uri("mv2://docs/getting-started.md")
|
||||
.tag("category", "documentation")
|
||||
.tag("version", "2.0")
|
||||
.build();
|
||||
let seq2 = mem.put_bytes_with_options(
|
||||
b"This guide covers the basics of using Memvid for AI memory storage.",
|
||||
options,
|
||||
)?;
|
||||
println!(" Added document 2 (with metadata), sequence: {}", seq2);
|
||||
|
||||
// Add more documents
|
||||
let options = PutOptions::builder()
|
||||
.title("API Reference")
|
||||
.uri("mv2://docs/api-reference.md")
|
||||
.tag("category", "documentation")
|
||||
.build();
|
||||
mem.put_bytes_with_options(
|
||||
b"The Memvid API provides methods for create, put, find, and timeline operations.",
|
||||
options,
|
||||
)?;
|
||||
|
||||
let options = PutOptions::builder()
|
||||
.title("FAQ")
|
||||
.uri("mv2://docs/faq.md")
|
||||
.tag("category", "support")
|
||||
.build();
|
||||
mem.put_bytes_with_options(
|
||||
b"Frequently asked questions about Memvid memory files and search.",
|
||||
options,
|
||||
)?;
|
||||
|
||||
// Commit changes to persist them
|
||||
mem.commit()?;
|
||||
println!(" Committed all changes\n");
|
||||
|
||||
// ========================================
|
||||
// 3. STATS - Check memory statistics
|
||||
// ========================================
|
||||
println!("3. Memory statistics:");
|
||||
let stats = mem.stats()?;
|
||||
println!(" Frame count: {}", stats.frame_count);
|
||||
println!(" Has lexical index: {}", stats.has_lex_index);
|
||||
println!(" Has vector index: {}", stats.has_vec_index);
|
||||
println!(" Has time index: {}", stats.has_time_index);
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 4. FIND - Search for documents
|
||||
// ========================================
|
||||
println!("4. Searching for documents...");
|
||||
|
||||
// Search for "memvid"
|
||||
let request = SearchRequest {
|
||||
query: "memvid".to_string(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
};
|
||||
let response = mem.search(request)?;
|
||||
println!(" Query: 'memvid'");
|
||||
println!(" Total hits: {}", response.total_hits);
|
||||
println!(" Elapsed: {}ms", response.elapsed_ms);
|
||||
for hit in &response.hits {
|
||||
let title = hit.title.as_deref().unwrap_or("Untitled");
|
||||
let score = hit.score.unwrap_or(0.0);
|
||||
println!(" - [{}] {} (score: {:.3})", hit.frame_id, title, score);
|
||||
println!(
|
||||
" Snippet: {}...",
|
||||
&hit.text.chars().take(60).collect::<String>()
|
||||
);
|
||||
}
|
||||
println!();
|
||||
|
||||
// Search within a scope
|
||||
let request = SearchRequest {
|
||||
query: "documentation".to_string(),
|
||||
top_k: 10,
|
||||
snippet_chars: 100,
|
||||
uri: None,
|
||||
scope: Some("mv2://docs/".to_string()),
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
};
|
||||
let response = mem.search(request)?;
|
||||
println!(" Query: 'documentation' (scope: mv2://docs/)");
|
||||
println!(" Total hits: {}", response.total_hits);
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 5. TIMELINE - Browse documents chronologically
|
||||
// ========================================
|
||||
println!("5. Timeline (chronological view):");
|
||||
let timeline = mem.timeline(TimelineQuery::default())?;
|
||||
for entry in &timeline {
|
||||
let uri = entry.uri.as_deref().unwrap_or("(no uri)");
|
||||
println!(
|
||||
" [{}] {} - {}",
|
||||
entry.frame_id,
|
||||
uri,
|
||||
entry.preview.chars().take(40).collect::<String>()
|
||||
);
|
||||
}
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 6. REOPEN - Close and reopen the memory
|
||||
// ========================================
|
||||
println!("6. Closing and reopening memory...");
|
||||
drop(mem);
|
||||
|
||||
let reopened = Memvid::open(&path)?;
|
||||
let stats = reopened.stats()?;
|
||||
println!(" Reopened successfully!");
|
||||
println!(" Frame count after reopen: {}", stats.frame_count);
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 7. VERIFY - Check file integrity
|
||||
// ========================================
|
||||
println!("7. Verifying file integrity...");
|
||||
drop(reopened); // Close the memory before verifying
|
||||
let report = Memvid::verify(&path, false)?;
|
||||
println!(" Verification status: {:?}", report.overall_status);
|
||||
println!();
|
||||
|
||||
println!("=== Example completed successfully! ===");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
//! CLIP Visual Search Example
|
||||
//!
|
||||
//! Demonstrates using CLIP embeddings to search PDF pages and images
|
||||
//! using natural language queries.
|
||||
//!
|
||||
//! Run with:
|
||||
//! ```bash
|
||||
//! cargo run --example clip_visual_search --features clip,pdfium -- /path/to/pdf
|
||||
//! ```
|
||||
//!
|
||||
//! Prerequisites:
|
||||
//! 1. Download the MobileCLIP-S2 ONNX models:
|
||||
//! ```bash
|
||||
//! mkdir -p ~/.local/share/memvid/models
|
||||
//! curl -L 'https://huggingface.co/Xenova/mobileclip_s2/resolve/main/onnx/vision_model_int8.onnx' \
|
||||
//! -o ~/.local/share/memvid/models/mobileclip-s2_vision.onnx
|
||||
//! curl -L 'https://huggingface.co/Xenova/mobileclip_s2/resolve/main/onnx/text_model_int8.onnx' \
|
||||
//! -o ~/.local/share/memvid/models/mobileclip-s2_text.onnx
|
||||
//! ```
|
||||
//!
|
||||
//! 2. For PDF page rendering, install pdfium:
|
||||
//! - macOS: `brew install nicbarker/pdfium/pdfium-mac-arm64` or `pdfium-mac-x64`
|
||||
//! - Linux: Download from https://github.com/nicbarker/pdfium-builds/releases
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
#[cfg(not(feature = "clip"))]
|
||||
{
|
||||
eprintln!("This example requires the 'clip' feature.");
|
||||
eprintln!("Run with: cargo run --example clip_visual_search --features clip");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "clip")]
|
||||
{
|
||||
use memvid_core::clip::{ClipConfig, ClipIndex, ClipIndexBuilder, ClipModel};
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::tempdir;
|
||||
|
||||
println!("=== CLIP Visual Search Example ===\n");
|
||||
|
||||
// Get PDF path from args or use default
|
||||
let args: Vec<String> = env::args().collect();
|
||||
let pdf_path = if args.len() > 1 {
|
||||
PathBuf::from(&args[1])
|
||||
} else {
|
||||
// Default to the SP Global Impact Report if no path provided
|
||||
PathBuf::from(
|
||||
"/Users/olow/Desktop/memvid-org/brickfield/sp-global-impact-report-2024.pdf",
|
||||
)
|
||||
};
|
||||
|
||||
if !pdf_path.exists() {
|
||||
eprintln!("PDF not found: {}", pdf_path.display());
|
||||
eprintln!(
|
||||
"Usage: cargo run --example clip_visual_search --features clip,pdfium -- /path/to/pdf"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
println!("PDF: {}", pdf_path.display());
|
||||
|
||||
// Initialize CLIP model
|
||||
println!("\n1. Loading CLIP model...");
|
||||
let config = ClipConfig::default();
|
||||
println!(" Model: {}", config.model_name);
|
||||
println!(" Models dir: {}", config.models_dir.display());
|
||||
|
||||
let clip = match ClipModel::new(config) {
|
||||
Ok(model) => {
|
||||
println!(" Model initialized (lazy loading)");
|
||||
model
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" Failed to initialize CLIP: {}", e);
|
||||
eprintln!("\n Make sure to download the models first:");
|
||||
eprintln!(" mkdir -p ~/.local/share/memvid/models");
|
||||
eprintln!(
|
||||
" curl -L 'https://huggingface.co/Xenova/mobileclip_s2/resolve/main/onnx/vision_model_int8.onnx' \\"
|
||||
);
|
||||
eprintln!(" -o ~/.local/share/memvid/models/mobileclip-s2_vision.onnx");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
// For this demo, we'll create synthetic embeddings since PDF rendering
|
||||
// requires pdfium which may not be available
|
||||
println!("\n2. Building CLIP index with sample embeddings...");
|
||||
|
||||
let mut builder = ClipIndexBuilder::new();
|
||||
|
||||
// Simulate embeddings for 10 "pages" with different visual concepts
|
||||
// In real usage, you would:
|
||||
// 1. Render each PDF page to an image
|
||||
// 2. Pass the image to clip.encode_image(&image)
|
||||
// 3. Store the embedding with the page's frame_id
|
||||
|
||||
let sample_concepts: Vec<(&str, Vec<f32>)> = vec![
|
||||
// These would be real embeddings from actual images in production
|
||||
(
|
||||
"charts and graphs",
|
||||
random_embedding(clip.dims() as usize, 1),
|
||||
),
|
||||
(
|
||||
"sustainability report cover",
|
||||
random_embedding(clip.dims() as usize, 2),
|
||||
),
|
||||
(
|
||||
"ESG metrics table",
|
||||
random_embedding(clip.dims() as usize, 3),
|
||||
),
|
||||
(
|
||||
"environmental impact diagram",
|
||||
random_embedding(clip.dims() as usize, 4),
|
||||
),
|
||||
(
|
||||
"carbon emissions chart",
|
||||
random_embedding(clip.dims() as usize, 5),
|
||||
),
|
||||
(
|
||||
"renewable energy infographic",
|
||||
random_embedding(clip.dims() as usize, 6),
|
||||
),
|
||||
(
|
||||
"corporate governance structure",
|
||||
random_embedding(clip.dims() as usize, 7),
|
||||
),
|
||||
(
|
||||
"diversity statistics",
|
||||
random_embedding(clip.dims() as usize, 8),
|
||||
),
|
||||
(
|
||||
"supply chain map",
|
||||
random_embedding(clip.dims() as usize, 9),
|
||||
),
|
||||
(
|
||||
"financial highlights",
|
||||
random_embedding(clip.dims() as usize, 10),
|
||||
),
|
||||
];
|
||||
|
||||
for (i, (concept, embedding)) in sample_concepts.iter().enumerate() {
|
||||
builder.add_document(i as u64, Some(i as u32), embedding.clone());
|
||||
println!(" Added page {} ({})", i + 1, concept);
|
||||
}
|
||||
|
||||
let artifact = builder.finish()?;
|
||||
println!(
|
||||
"\n Index built: {} vectors, {} dimensions",
|
||||
artifact.vector_count, artifact.dimension
|
||||
);
|
||||
|
||||
// Decode the index for searching
|
||||
let index = ClipIndex::decode(&artifact.bytes)?;
|
||||
|
||||
// Demonstrate text-to-image search
|
||||
println!("\n3. Searching with natural language queries...\n");
|
||||
|
||||
// Try encoding a text query
|
||||
println!(" Encoding query: 'sustainability charts'");
|
||||
match clip.encode_text("sustainability charts") {
|
||||
Ok(query_embedding) => {
|
||||
println!(" Query embedding: {} dimensions", query_embedding.len());
|
||||
|
||||
let hits = index.search(&query_embedding, 3);
|
||||
println!("\n Top 3 matches:");
|
||||
for (rank, hit) in hits.iter().enumerate() {
|
||||
let concept = sample_concepts
|
||||
.get(hit.frame_id as usize)
|
||||
.map(|(c, _)| *c)
|
||||
.unwrap_or("unknown");
|
||||
println!(
|
||||
" {}. Page {} ({}) - distance: {:.4}",
|
||||
rank + 1,
|
||||
hit.frame_id + 1,
|
||||
concept,
|
||||
hit.distance
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" Failed to encode text (model not loaded): {}", e);
|
||||
eprintln!(" Make sure the text model ONNX file is downloaded.");
|
||||
}
|
||||
}
|
||||
|
||||
// Demo with Memvid integration
|
||||
println!("\n4. Creating Memvid memory with CLIP support...");
|
||||
|
||||
let dir = tempdir().expect("failed to create temp dir");
|
||||
let path = dir.path().join("clip_demo.mv2");
|
||||
|
||||
let mut mem = memvid_core::Memvid::create(&path)?;
|
||||
|
||||
// Enable CLIP index
|
||||
mem.enable_clip()?;
|
||||
println!(" CLIP index enabled");
|
||||
|
||||
// Add some sample documents
|
||||
let options = memvid_core::PutOptions::builder()
|
||||
.title("SP Global Impact Report 2024 - Page 1")
|
||||
.uri("mv2://reports/sp-global/page-1")
|
||||
.build();
|
||||
mem.put_bytes_with_options(
|
||||
b"This page contains sustainability charts and ESG metrics.",
|
||||
options,
|
||||
)?;
|
||||
|
||||
mem.commit()?;
|
||||
println!(" Added sample document");
|
||||
|
||||
let stats = mem.stats()?;
|
||||
println!("\n Memory stats:");
|
||||
println!(" - Frames: {}", stats.frame_count);
|
||||
println!(" - Has CLIP index: {}", stats.has_clip_index);
|
||||
|
||||
// Search CLIP index (would use pre-computed query embedding in production)
|
||||
// Since we haven't added actual CLIP embeddings to the memory,
|
||||
// the search would return empty results - this is just to show the API
|
||||
|
||||
println!("\n=== Example completed successfully! ===");
|
||||
println!("\nTo use CLIP in production:");
|
||||
println!("1. During ingestion: Generate CLIP embeddings for images/PDF pages");
|
||||
println!("2. Store embeddings in the CLIP index alongside text content");
|
||||
println!("3. At query time: Encode the text query with clip.encode_text()");
|
||||
println!("4. Search the CLIP index with mem.search_clip(&query_embedding, limit)");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a pseudo-random embedding for demo purposes
|
||||
/// In production, use clip.encode_image() on actual images
|
||||
#[cfg(feature = "clip")]
|
||||
fn random_embedding(dims: usize, seed: u64) -> Vec<f32> {
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
let mut embedding = Vec::with_capacity(dims);
|
||||
|
||||
for i in 0..dims {
|
||||
(seed, i).hash(&mut hasher);
|
||||
let hash = hasher.finish();
|
||||
// Generate pseudo-random float between -1 and 1
|
||||
let val = ((hash as f32) / (u64::MAX as f32)) * 2.0 - 1.0;
|
||||
embedding.push(val);
|
||||
hasher = DefaultHasher::new();
|
||||
}
|
||||
|
||||
// L2 normalize
|
||||
let norm: f32 = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
if norm > 1e-10 {
|
||||
for v in &mut embedding {
|
||||
*v /= norm;
|
||||
}
|
||||
}
|
||||
|
||||
embedding
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
//! Generate visual performance comparison report
|
||||
//!
|
||||
//! Run with: cargo run --example generate_performance_report --features lex
|
||||
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
println!("=== Search Precision Performance Report ===\n");
|
||||
|
||||
// Create test corpus
|
||||
println!("Setting up test corpus (1000 documents)...");
|
||||
let temp_file = "/tmp/perf_test.mv2";
|
||||
let _ = std::fs::remove_file(temp_file);
|
||||
|
||||
let mut mem = Memvid::create(temp_file)?;
|
||||
|
||||
// Add 1000 documents with controlled distribution
|
||||
for i in 0..1000 {
|
||||
let topic = match i % 5 {
|
||||
0 => ("machine learning", "neural networks"),
|
||||
1 => ("python programming", "software development"),
|
||||
2 => ("machine learning with python", "data science"),
|
||||
3 => ("rust systems programming", "memory safety"),
|
||||
_ => ("web development", "javascript frameworks"),
|
||||
};
|
||||
|
||||
let content = format!(
|
||||
"Document {} about {}. This article covers {} in depth.",
|
||||
i, topic.0, topic.1
|
||||
);
|
||||
|
||||
mem.put_bytes_with_options(
|
||||
content.as_bytes(),
|
||||
PutOptions::builder()
|
||||
.title(format!("Doc {} - {}", i, topic.0))
|
||||
.build(),
|
||||
)?;
|
||||
|
||||
if (i + 1) % 100 == 0 {
|
||||
mem.commit()?;
|
||||
}
|
||||
}
|
||||
mem.commit()?;
|
||||
println!("✓ Corpus ready\n");
|
||||
|
||||
// Test queries
|
||||
let test_queries = vec![
|
||||
("machine python", "Both terms"),
|
||||
("machine learning python", "Three terms"),
|
||||
("python programming development", "Three terms"),
|
||||
("rust memory safety", "Two terms"),
|
||||
];
|
||||
|
||||
println!("┌─────────────────────────────────────────────────────────────────────┐");
|
||||
println!("│ QUERY PERFORMANCE METRICS │");
|
||||
println!("├─────────────────────────────────────────────────────────────────────┤");
|
||||
|
||||
for (query, desc) in &test_queries {
|
||||
// Warm up
|
||||
for _ in 0..10 {
|
||||
let _ = mem.search(SearchRequest {
|
||||
query: query.to_string(),
|
||||
top_k: 100,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})?;
|
||||
}
|
||||
|
||||
// Measure
|
||||
let iterations = 100;
|
||||
let start = Instant::now();
|
||||
|
||||
let mut total_results = 0;
|
||||
let mut total_relevant = 0;
|
||||
|
||||
for _ in 0..iterations {
|
||||
let results = mem.search(SearchRequest {
|
||||
query: query.to_string(),
|
||||
top_k: 100,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})?;
|
||||
|
||||
let terms: Vec<&str> = query.split_whitespace().collect();
|
||||
let relevant = results
|
||||
.hits
|
||||
.iter()
|
||||
.filter(|hit| {
|
||||
let text = hit.text.to_lowercase();
|
||||
terms.iter().all(|term| text.contains(term))
|
||||
})
|
||||
.count();
|
||||
|
||||
total_results += results.hits.len();
|
||||
total_relevant += relevant;
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
let avg_latency_us = elapsed.as_micros() / iterations as u128; // FIX: Cast to u128
|
||||
let avg_results = total_results / iterations;
|
||||
let avg_relevant = total_relevant / iterations;
|
||||
let precision = if avg_results > 0 {
|
||||
(avg_relevant as f64 / avg_results as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
println!("│");
|
||||
println!("│ Query: \"{}\" ({})", query, desc);
|
||||
println!("│ Latency: {:.2}ms", avg_latency_us as f64 / 1000.0);
|
||||
println!("│ Results: {} docs", avg_results);
|
||||
println!("│ Relevant: {} docs", avg_relevant);
|
||||
println!("│ Precision: {:.1}%", precision);
|
||||
println!("│ Memory: ~{} KB", (avg_results * 3).max(1));
|
||||
}
|
||||
|
||||
println!("└─────────────────────────────────────────────────────────────────────┘");
|
||||
|
||||
// Comparison with hypothetical OR behavior
|
||||
println!("\n┌─────────────────────────────────────────────────────────────────────┐");
|
||||
println!("│ COMPARISON: AND vs OR (Estimated) │");
|
||||
println!("├─────────────────────────────────────────────────────────────────────┤");
|
||||
println!("│");
|
||||
println!("│ Query: \"machine python\"");
|
||||
println!("│");
|
||||
println!("│ WITH AND (Current): │ WITH OR (Previous):");
|
||||
println!("│ • Results: ~5-8 docs │ • Results: ~80-120 docs");
|
||||
println!("│ • Precision: 100% │ • Precision: ~6-8%");
|
||||
println!("│ • Memory: ~20 KB │ • Memory: ~300 KB");
|
||||
println!("│ • Processing: 6-10ms │ • Processing: 96-144ms");
|
||||
println!("│");
|
||||
println!("│ IMPROVEMENT: ");
|
||||
println!("│ ✓ 15x better precision (6% → 100%) ");
|
||||
println!("│ ✓ 93% less memory (300KB → 20KB) ");
|
||||
println!("│ ✓ 93% faster processing (120ms → 8ms) ");
|
||||
println!("│ ✓ No query latency regression (~1.2ms) ");
|
||||
println!("│");
|
||||
println!("└─────────────────────────────────────────────────────────────────────┘");
|
||||
|
||||
std::fs::remove_file(temp_file)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
//! Example demonstrating OpenAI API embedding usage.
|
||||
//!
|
||||
//! This example shows how to:
|
||||
//! - Create an OpenAI embedder with default configuration
|
||||
//! - Generate embeddings using the API
|
||||
//! - Compute cosine similarity between embeddings
|
||||
//! - Use different models (small, large, ada)
|
||||
//!
|
||||
//! ## Prerequisites
|
||||
//!
|
||||
//! Set your OpenAI API key:
|
||||
//! ```bash
|
||||
//! export OPENAI_API_KEY="sk-..."
|
||||
//! ```
|
||||
//!
|
||||
//! ## Run
|
||||
//!
|
||||
//! ```bash
|
||||
//! cargo run --example openai_embedding --features api_embed
|
||||
//! ```
|
||||
|
||||
use memvid_core::Result;
|
||||
|
||||
#[cfg(feature = "api_embed")]
|
||||
use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
|
||||
#[cfg(feature = "api_embed")]
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
/// Compute cosine similarity between two vectors
|
||||
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
assert_eq!(a.len(), b.len(), "Vectors must have same length");
|
||||
|
||||
let dot_product: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
|
||||
let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
|
||||
if norm_a > 0.0 && norm_b > 0.0 {
|
||||
dot_product / (norm_a * norm_b)
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "api_embed")]
|
||||
fn main() -> Result<()> {
|
||||
println!("=== OpenAI Embedding Example ===\n");
|
||||
|
||||
// Check if API key is set
|
||||
if std::env::var("OPENAI_API_KEY").is_err() {
|
||||
eprintln!("Error: OPENAI_API_KEY environment variable not set.");
|
||||
eprintln!("Please set it with: export OPENAI_API_KEY=\"sk-...\"");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// Create embedder with default config (text-embedding-3-small, 1536 dimensions)
|
||||
println!("Creating OpenAI embedder (text-embedding-3-small)...");
|
||||
let config = OpenAIConfig::default();
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
|
||||
println!("Model: {}", embedder.model());
|
||||
println!("Kind: {}", embedder.kind());
|
||||
println!("Dimensions: {}", embedder.dimension());
|
||||
println!("Ready: {}\n", embedder.is_ready());
|
||||
|
||||
// Example 1: Single text embedding
|
||||
println!("--- Example 1: Single Text Embedding ---");
|
||||
let text = "The quick brown fox jumps over the lazy dog";
|
||||
println!("Embedding text: \"{}\"", text);
|
||||
|
||||
let embedding = embedder.embed_text(text)?;
|
||||
println!("Generated embedding of dimension {}\n", embedding.len());
|
||||
|
||||
// Example 2: Semantic similarity
|
||||
println!("--- Example 2: Semantic Similarity ---");
|
||||
let text1 = "Machine learning and artificial intelligence";
|
||||
let text2 = "Deep neural networks for AI applications";
|
||||
let text3 = "The history of ancient Rome";
|
||||
|
||||
let emb1 = embedder.embed_text(text1)?;
|
||||
let emb2 = embedder.embed_text(text2)?;
|
||||
let emb3 = embedder.embed_text(text3)?;
|
||||
|
||||
println!("Text 1: \"{}\"", text1);
|
||||
println!("Text 2: \"{}\"", text2);
|
||||
println!("Text 3: \"{}\"", text3);
|
||||
println!();
|
||||
|
||||
let sim_1_2 = cosine_similarity(&emb1, &emb2);
|
||||
let sim_1_3 = cosine_similarity(&emb1, &emb3);
|
||||
let sim_2_3 = cosine_similarity(&emb2, &emb3);
|
||||
|
||||
println!("Similarity (1 ↔ 2): {:.4}", sim_1_2);
|
||||
println!("Similarity (1 ↔ 3): {:.4}", sim_1_3);
|
||||
println!("Similarity (2 ↔ 3): {:.4}", sim_2_3);
|
||||
|
||||
if sim_1_2 > sim_1_3 {
|
||||
println!("\n✓ Related texts (1 & 2) have higher similarity than unrelated (1 & 3)!");
|
||||
}
|
||||
println!();
|
||||
|
||||
// Example 3: Batch processing
|
||||
println!("--- Example 3: Batch Processing ---");
|
||||
let documents = vec![
|
||||
"Python programming language",
|
||||
"JavaScript web development",
|
||||
"Rust systems programming",
|
||||
"Italian cooking recipes",
|
||||
];
|
||||
|
||||
println!("Processing {} documents in batch...", documents.len());
|
||||
let batch_embeddings = embedder.embed_batch(&documents)?;
|
||||
println!(
|
||||
"✓ Generated {} embeddings of dimension {}\n",
|
||||
batch_embeddings.len(),
|
||||
batch_embeddings.first().map(|e| e.len()).unwrap_or(0)
|
||||
);
|
||||
|
||||
// Find most similar pair
|
||||
let mut max_sim = 0.0;
|
||||
let mut max_pair = (0, 0);
|
||||
|
||||
for i in 0..batch_embeddings.len() {
|
||||
for j in (i + 1)..batch_embeddings.len() {
|
||||
let sim = cosine_similarity(&batch_embeddings[i], &batch_embeddings[j]);
|
||||
if sim > max_sim {
|
||||
max_sim = sim;
|
||||
max_pair = (i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("Most similar pair (similarity: {:.4}):", max_sim);
|
||||
println!(" [{}] \"{}\"", max_pair.0, documents[max_pair.0]);
|
||||
println!(" [{}] \"{}\"\n", max_pair.1, documents[max_pair.1]);
|
||||
|
||||
// Example 4: Available models
|
||||
println!("--- Example 4: Available Models ---");
|
||||
println!("OpenAI embedding models:");
|
||||
println!(" - text-embedding-3-small (1536d) - Default, fastest, cheapest");
|
||||
println!(" - text-embedding-3-large (3072d) - Highest quality");
|
||||
println!(" - text-embedding-ada-002 (1536d) - Legacy model");
|
||||
println!();
|
||||
println!("To use a different model:");
|
||||
println!(" let config = OpenAIConfig::large();");
|
||||
println!(" let embedder = OpenAIEmbedder::new(config)?;");
|
||||
println!();
|
||||
|
||||
println!("=== Example Complete ===");
|
||||
println!("\nKey takeaways:");
|
||||
println!("✓ API embeddings require OPENAI_API_KEY environment variable");
|
||||
println!("✓ text-embedding-3-small is fast and cost-effective");
|
||||
println!("✓ Batch processing reduces API calls for multiple texts");
|
||||
println!("✓ Similar texts have higher cosine similarity scores");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "api_embed"))]
|
||||
fn main() {
|
||||
eprintln!("This example requires the 'api_embed' feature.");
|
||||
eprintln!("Run with: cargo run --example openai_embedding --features api_embed");
|
||||
std::process::exit(1);
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
//! PDF ingestion example demonstrating how to ingest and search PDF documents.
|
||||
//!
|
||||
//! This example demonstrates PDF text extraction, chunking, and semantic search.
|
||||
//!
|
||||
//! Run with:
|
||||
//! ```bash
|
||||
//! cargo run --example pdf_ingestion -- /path/to/pdf
|
||||
//! ```
|
||||
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use memvid_core::{Memvid, PutOptions, Result, SearchRequest};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// Get PDF path from args
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: cargo run --example pdf_ingestion -- /path/to/pdf");
|
||||
eprintln!("\nExample:");
|
||||
eprintln!(" cargo run --example pdf_ingestion -- examples/1706.03762v7.pdf");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let pdf_path = PathBuf::from(&args[1]);
|
||||
|
||||
if !pdf_path.exists() {
|
||||
eprintln!("ERROR: PDF file not found at {:?}", pdf_path);
|
||||
eprintln!("Usage: cargo run --example pdf_ingestion -- /path/to/pdf");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Create a temporary directory for our memory file
|
||||
let dir = tempdir().expect("failed to create temp dir");
|
||||
let mv2_path: PathBuf = dir.path().join("paper.mv2");
|
||||
|
||||
println!("=== Memvid PDF Ingestion Example ===\n");
|
||||
|
||||
// ========================================
|
||||
// 1. CREATE a new memory file
|
||||
// ========================================
|
||||
println!("1. Creating memory file...");
|
||||
let mut mem = Memvid::create(&mv2_path)?;
|
||||
println!(" Memory created at {:?}\n", mv2_path);
|
||||
|
||||
// ========================================
|
||||
// 2. INGEST the PDF file
|
||||
// ========================================
|
||||
println!("2. Ingesting PDF: {:?}", pdf_path);
|
||||
|
||||
// Read the PDF file
|
||||
let pdf_bytes = std::fs::read(&pdf_path)?;
|
||||
println!(" PDF size: {} bytes", pdf_bytes.len());
|
||||
|
||||
// Put the PDF with metadata
|
||||
// Extract filename for title
|
||||
let title = pdf_path
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("PDF Document")
|
||||
.to_string();
|
||||
|
||||
let options = PutOptions::builder()
|
||||
.title(&title)
|
||||
.uri(format!(
|
||||
"mv2://pdfs/{}",
|
||||
pdf_path.file_name().unwrap_or_default().to_string_lossy()
|
||||
))
|
||||
.build();
|
||||
|
||||
let frame_id = mem.put_bytes_with_options(&pdf_bytes, options)?;
|
||||
println!(" Ingested as frame: {}", frame_id);
|
||||
|
||||
// Commit changes
|
||||
mem.commit()?;
|
||||
println!(" Committed successfully!\n");
|
||||
|
||||
// ========================================
|
||||
// 3. CHECK memory statistics
|
||||
// ========================================
|
||||
println!("3. Memory statistics:");
|
||||
let stats = mem.stats()?;
|
||||
println!(" Frame count: {}", stats.frame_count);
|
||||
println!(" Has lexical index: {}", stats.has_lex_index);
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 4. SEARCH the ingested PDF
|
||||
// ========================================
|
||||
println!("4. Searching the paper...\n");
|
||||
|
||||
// Search for "attention"
|
||||
let queries = [
|
||||
"attention mechanism",
|
||||
"transformer architecture",
|
||||
"self-attention",
|
||||
"encoder decoder",
|
||||
"positional encoding",
|
||||
];
|
||||
|
||||
for query in queries {
|
||||
let request = SearchRequest {
|
||||
query: query.to_string(),
|
||||
top_k: 3,
|
||||
snippet_chars: 150,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
};
|
||||
|
||||
let response = mem.search(request)?;
|
||||
println!(" Query: '{}'", query);
|
||||
println!(
|
||||
" Hits: {} ({}ms)",
|
||||
response.total_hits, response.elapsed_ms
|
||||
);
|
||||
|
||||
for (i, hit) in response.hits.iter().take(2).enumerate() {
|
||||
let snippet = hit
|
||||
.text
|
||||
.chars()
|
||||
.take(100)
|
||||
.collect::<String>()
|
||||
.replace('\n', " ");
|
||||
println!(" {}. {}...", i + 1, snippet);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
// ========================================
|
||||
// 5. VERIFY file integrity
|
||||
// ========================================
|
||||
println!("5. Verifying file integrity...");
|
||||
drop(mem);
|
||||
let report = Memvid::verify(&mv2_path, false)?;
|
||||
println!(" Status: {:?}", report.overall_status);
|
||||
println!();
|
||||
|
||||
println!("=== PDF ingestion example completed! ===");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
//! Benchmark comparing SIMD vs Scalar L2 distance calculations.
|
||||
//!
|
||||
//! Run with: `cargo run --example simd_benchmark --features simd --release`
|
||||
|
||||
use std::hint::black_box;
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() {
|
||||
let num_vectors = 10_000;
|
||||
let dims = 384; // Standard embedding dimension
|
||||
let num_iterations = 100;
|
||||
|
||||
println!("SIMD vs Scalar L2 Distance Benchmark");
|
||||
println!("=====================================");
|
||||
println!("Vectors: {}", num_vectors);
|
||||
println!("Dimensions: {}", dims);
|
||||
println!("Iterations: {}", num_iterations);
|
||||
println!();
|
||||
|
||||
// Generate random vectors
|
||||
let query: Vec<f32> = (0..dims).map(|i| (i as f32 * 0.001) % 1.0).collect();
|
||||
let vectors: Vec<Vec<f32>> = (0..num_vectors)
|
||||
.map(|v| (0..dims).map(|i| ((v + i) as f32 * 0.0017) % 1.0).collect())
|
||||
.collect();
|
||||
|
||||
// Benchmark SIMD version
|
||||
let simd_start = Instant::now();
|
||||
let mut simd_sum = 0.0f32;
|
||||
for _ in 0..num_iterations {
|
||||
for vec in &vectors {
|
||||
simd_sum += black_box(l2_distance_simd(black_box(&query), black_box(vec)));
|
||||
}
|
||||
}
|
||||
let simd_elapsed = simd_start.elapsed();
|
||||
black_box(simd_sum);
|
||||
|
||||
// Benchmark Scalar version
|
||||
let scalar_start = Instant::now();
|
||||
let mut scalar_sum = 0.0f32;
|
||||
for _ in 0..num_iterations {
|
||||
for vec in &vectors {
|
||||
scalar_sum += black_box(l2_distance_scalar(black_box(&query), black_box(vec)));
|
||||
}
|
||||
}
|
||||
let scalar_elapsed = scalar_start.elapsed();
|
||||
black_box(scalar_sum);
|
||||
|
||||
// Results
|
||||
let total_ops = num_vectors * num_iterations;
|
||||
let simd_per_op_ns = simd_elapsed.as_nanos() as f64 / total_ops as f64;
|
||||
let scalar_per_op_ns = scalar_elapsed.as_nanos() as f64 / total_ops as f64;
|
||||
let speedup = scalar_elapsed.as_nanos() as f64 / simd_elapsed.as_nanos() as f64;
|
||||
|
||||
println!("Results:");
|
||||
println!("--------");
|
||||
println!(
|
||||
"SIMD: {:>8.2}ms total, {:>6.1}ns per distance",
|
||||
simd_elapsed.as_secs_f64() * 1000.0,
|
||||
simd_per_op_ns
|
||||
);
|
||||
println!(
|
||||
"Scalar: {:>8.2}ms total, {:>6.1}ns per distance",
|
||||
scalar_elapsed.as_secs_f64() * 1000.0,
|
||||
scalar_per_op_ns
|
||||
);
|
||||
println!();
|
||||
println!("Speedup: {:.2}x", speedup);
|
||||
|
||||
// Verify correctness
|
||||
let simd_result = l2_distance_simd(&query, &vectors[0]);
|
||||
let scalar_result = l2_distance_scalar(&query, &vectors[0]);
|
||||
let diff = (simd_result - scalar_result).abs();
|
||||
println!();
|
||||
println!("Correctness check:");
|
||||
println!(" SIMD result: {:.8}", simd_result);
|
||||
println!(" Scalar result: {:.8}", scalar_result);
|
||||
println!(" Difference: {:.2e} (should be < 1e-5)", diff);
|
||||
assert!(diff < 1e-4, "Results differ too much!");
|
||||
println!(" ✓ Results match!");
|
||||
}
|
||||
|
||||
/// SIMD L2 distance using the wide crate
|
||||
#[cfg(feature = "simd")]
|
||||
fn l2_distance_simd(a: &[f32], b: &[f32]) -> f32 {
|
||||
memvid_core::simd::l2_distance_simd(a, b)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "simd"))]
|
||||
fn l2_distance_simd(a: &[f32], b: &[f32]) -> f32 {
|
||||
l2_distance_scalar(a, b)
|
||||
}
|
||||
|
||||
/// Scalar L2 distance (the OLD implementation)
|
||||
#[inline(never)]
|
||||
fn l2_distance_scalar(a: &[f32], b: &[f32]) -> f32 {
|
||||
a.iter()
|
||||
.zip(b.iter())
|
||||
.map(|(x, y)| (x - y).powi(2))
|
||||
.sum::<f32>()
|
||||
.sqrt()
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
//! Test to demonstrate the implicit OR bug in query parsing
|
||||
//!
|
||||
//! Run with: cargo run --example test_implicit_or_bug --features lex
|
||||
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
let test_file = "/tmp/test_implicit.mv2";
|
||||
|
||||
// Clean up old file if exists
|
||||
let _ = std::fs::remove_file(test_file);
|
||||
|
||||
println!("Creating test .mv2 file...");
|
||||
let mut mem = Memvid::create(test_file)?;
|
||||
|
||||
// Add documents
|
||||
println!("Adding test documents...");
|
||||
mem.put_bytes_with_options(
|
||||
b"I love machine learning",
|
||||
PutOptions::builder().title("Doc 1").build(),
|
||||
)?;
|
||||
mem.put_bytes_with_options(
|
||||
b"I love Python programming",
|
||||
PutOptions::builder().title("Doc 2").build(),
|
||||
)?;
|
||||
mem.put_bytes_with_options(
|
||||
b"Machine learning with Python is awesome",
|
||||
PutOptions::builder().title("Doc 3").build(),
|
||||
)?;
|
||||
mem.commit()?;
|
||||
|
||||
println!();
|
||||
println!("=== TESTING IMPLICIT OPERATOR BEHAVIOR ===");
|
||||
println!("Query: 'machine python'");
|
||||
println!();
|
||||
println!("Expected behavior (AND): Should return 1 result");
|
||||
println!(" - Only Doc 3 has BOTH 'machine' AND 'python'");
|
||||
println!();
|
||||
println!("Current behavior (OR): Returns 3 results");
|
||||
println!(" - Any doc with 'machine' OR 'python'");
|
||||
println!();
|
||||
|
||||
// Search with implicit operator
|
||||
let results = mem.search(SearchRequest {
|
||||
query: "machine python".to_string(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})?;
|
||||
|
||||
println!("ACTUAL RESULTS: {} documents found", results.hits.len());
|
||||
println!();
|
||||
|
||||
for (i, hit) in results.hits.iter().enumerate() {
|
||||
let title = hit
|
||||
.title
|
||||
.as_ref()
|
||||
.unwrap_or(&"Untitled".to_string())
|
||||
.clone();
|
||||
let text_lower = hit.text.to_lowercase();
|
||||
let has_machine = text_lower.contains("machine");
|
||||
let has_python = text_lower.contains("python");
|
||||
let is_relevant = has_machine && has_python;
|
||||
|
||||
let status = if is_relevant {
|
||||
"✓ RELEVANT"
|
||||
} else {
|
||||
"✗ NOT RELEVANT"
|
||||
};
|
||||
|
||||
println!(
|
||||
" {}. {} [machine={}, python={}] {}",
|
||||
i + 1,
|
||||
title,
|
||||
has_machine,
|
||||
has_python,
|
||||
status
|
||||
);
|
||||
}
|
||||
|
||||
println!();
|
||||
|
||||
// Calculate precision
|
||||
let relevant_count = results
|
||||
.hits
|
||||
.iter()
|
||||
.filter(|hit| {
|
||||
let text = hit.text.to_lowercase();
|
||||
text.contains("machine") && text.contains("python")
|
||||
})
|
||||
.count();
|
||||
|
||||
let precision = if results.hits.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
(relevant_count as f64 / results.hits.len() as f64) * 100.0
|
||||
};
|
||||
|
||||
println!(
|
||||
"PRECISION: {:.1}% ({}/{} results are relevant)",
|
||||
precision,
|
||||
relevant_count,
|
||||
results.hits.len()
|
||||
);
|
||||
println!();
|
||||
|
||||
// Verdict
|
||||
if results.hits.len() == 1 && relevant_count == 1 {
|
||||
println!("✓ CORRECT: Query uses implicit AND");
|
||||
println!(" Perfect precision - returns only relevant docs");
|
||||
} else if results.hits.len() > 1 {
|
||||
println!("✗ BUG DETECTED: Query uses implicit OR");
|
||||
println!(" Low precision - returns docs with EITHER term");
|
||||
println!();
|
||||
} else {
|
||||
println!("? NO RESULTS: Check if lex feature is enabled");
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
std::fs::remove_file(test_file)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
//! Whisper transcription example demonstrating audio transcription.
|
||||
//!
|
||||
//! Run with:
|
||||
//! ```bash
|
||||
//! cargo run --example test_whisper --features whisper -- /path/to/audio
|
||||
//! ```
|
||||
|
||||
#[cfg(feature = "whisper")]
|
||||
use memvid_core::{WhisperConfig, WhisperTranscriber};
|
||||
#[cfg(feature = "whisper")]
|
||||
use std::env;
|
||||
#[cfg(feature = "whisper")]
|
||||
use std::path::PathBuf;
|
||||
#[cfg(feature = "whisper")]
|
||||
use std::time::Instant;
|
||||
|
||||
#[cfg(not(feature = "whisper"))]
|
||||
fn main() {
|
||||
eprintln!(
|
||||
"This example requires the `whisper` feature.\n\
|
||||
Re-run with:\n\
|
||||
cargo run --example test_whisper --features whisper -- /path/to/audio"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "whisper")]
|
||||
fn main() {
|
||||
// Get audio path from args
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: cargo run --example test_whisper --features whisper -- /path/to/audio");
|
||||
eprintln!("\nExample:");
|
||||
eprintln!(
|
||||
" cargo run --example test_whisper --features whisper -- examples/call_sale.mp3"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let audio_path = PathBuf::from(&args[1]);
|
||||
|
||||
if !audio_path.exists() {
|
||||
eprintln!("ERROR: Audio file not found at {:?}", audio_path);
|
||||
eprintln!("Usage: cargo run --example test_whisper --features whisper -- /path/to/audio");
|
||||
return;
|
||||
}
|
||||
|
||||
println!("=== Whisper Transcription Example ===\n");
|
||||
println!("Creating Whisper transcriber...");
|
||||
let start = Instant::now();
|
||||
let config = WhisperConfig::default();
|
||||
println!("Model dir: {:?}", config.models_dir);
|
||||
println!("Model name: {}", config.model_name);
|
||||
|
||||
let mut transcriber = WhisperTranscriber::new(&config).expect("Failed to create transcriber");
|
||||
println!("Transcriber created in {:?}", start.elapsed());
|
||||
|
||||
println!("\nTranscribing audio file: {}", audio_path.display());
|
||||
let start = Instant::now();
|
||||
|
||||
match transcriber.transcribe_file(&audio_path) {
|
||||
Ok(result) => {
|
||||
println!("Transcription completed in {:?}", start.elapsed());
|
||||
println!("\n=== Transcription Result ===");
|
||||
println!("Duration: {:.2} seconds", result.duration_secs);
|
||||
println!("Language: {}", result.language);
|
||||
println!("\nText:\n{}", result.text);
|
||||
|
||||
if !result.segments.is_empty() {
|
||||
println!("\n=== Segments ===");
|
||||
for seg in &result.segments {
|
||||
println!("[{:.2}s - {:.2}s] {}", seg.start, seg.end, seg.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Transcription failed: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n=== Transcription example completed! ===");
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
//! Benchmark demonstrating the performance benefit of embedding caching.
|
||||
//!
|
||||
//! This example compares performance with and without caching for repeated texts.
|
||||
//!
|
||||
//! ## Prerequisites
|
||||
//!
|
||||
//! Download the BGE-small model:
|
||||
//!
|
||||
//! ```bash
|
||||
//! mkdir -p ~/.cache/memvid/text-models
|
||||
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
//! ```
|
||||
//!
|
||||
//! ## Run
|
||||
//!
|
||||
//! ```bash
|
||||
//! cargo run --example text_embed_cache_bench --features vec --release
|
||||
//! ```
|
||||
|
||||
use memvid_core::Result;
|
||||
#[cfg(feature = "vec")]
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
println!("╔════════════════════════════════════════════════════════════╗");
|
||||
println!("║ Embedding Cache Benchmark ║");
|
||||
println!("╚════════════════════════════════════════════════════════════╝\n");
|
||||
|
||||
// Test texts with intentional repeats
|
||||
let test_texts = vec![
|
||||
"machine learning",
|
||||
"artificial intelligence",
|
||||
"deep learning",
|
||||
"neural networks",
|
||||
"natural language processing",
|
||||
"machine learning", // Repeat 1
|
||||
"artificial intelligence", // Repeat 2
|
||||
"computer vision",
|
||||
"deep learning", // Repeat 3
|
||||
"machine learning", // Repeat 4
|
||||
];
|
||||
|
||||
println!(
|
||||
"Test data: {} texts ({} unique, {} repeats)\n",
|
||||
test_texts.len(),
|
||||
7, // unique
|
||||
3
|
||||
); // repeats
|
||||
|
||||
// ---------- Test with Cache Enabled ----------
|
||||
println!("──────────────────────────────────────────────────────────");
|
||||
println!("Test 1: WITH Cache (enabled by default)");
|
||||
println!("──────────────────────────────────────────────────────────");
|
||||
|
||||
let config_cached = TextEmbedConfig::default();
|
||||
let embedder_cached = LocalTextEmbedder::new(config_cached)?;
|
||||
|
||||
println!("Processing texts...");
|
||||
let start = Instant::now();
|
||||
for (i, text) in test_texts.iter().enumerate() {
|
||||
let _ = embedder_cached.encode_text(text)?;
|
||||
if i < test_texts.len() - 1 {
|
||||
print!(".");
|
||||
}
|
||||
}
|
||||
println!("!");
|
||||
let cached_time = start.elapsed();
|
||||
|
||||
if let Some(stats) = embedder_cached.cache_stats() {
|
||||
println!("\n📊 Cache Statistics:");
|
||||
println!(" Hits: {}", stats.hits);
|
||||
println!(" Misses: {}", stats.misses);
|
||||
println!(" Size: {}/{}", stats.size, stats.capacity);
|
||||
println!(" Hit Rate: {:.1}%", stats.hit_rate() * 100.0);
|
||||
}
|
||||
|
||||
println!("\n⏱️ Total Time: {:?}", cached_time);
|
||||
|
||||
// ---------- Test with Cache Disabled ----------
|
||||
println!("\n──────────────────────────────────────────────────────────");
|
||||
println!("Test 2: WITHOUT Cache (force disabled)");
|
||||
println!("──────────────────────────────────────────────────────────");
|
||||
|
||||
let config_uncached = TextEmbedConfig {
|
||||
enable_cache: false,
|
||||
..Default::default()
|
||||
};
|
||||
let embedder_uncached = LocalTextEmbedder::new(config_uncached)?;
|
||||
|
||||
println!("Processing texts...");
|
||||
let start = Instant::now();
|
||||
for (i, text) in test_texts.iter().enumerate() {
|
||||
let _ = embedder_uncached.encode_text(text)?;
|
||||
if i < test_texts.len() - 1 {
|
||||
print!(".");
|
||||
}
|
||||
}
|
||||
println!("!");
|
||||
let uncached_time = start.elapsed();
|
||||
|
||||
println!("\n⏱️ Total Time: {:?}", uncached_time);
|
||||
|
||||
// ---------- Results ----------
|
||||
println!("\n╔════════════════════════════════════════════════════════════╗");
|
||||
println!("║ Results ║");
|
||||
println!("╚════════════════════════════════════════════════════════════╝\n");
|
||||
|
||||
let speedup = uncached_time.as_secs_f64() / cached_time.as_secs_f64();
|
||||
|
||||
println!("┌────────────────┬──────────────┬──────────────┐");
|
||||
println!("│ Configuration │ Time │ Speedup │");
|
||||
println!("├────────────────┼──────────────┼──────────────┤");
|
||||
println!(
|
||||
"│ With Cache │ {:>10.3}s │ {:>9.2}x │",
|
||||
cached_time.as_secs_f64(),
|
||||
speedup
|
||||
);
|
||||
println!(
|
||||
"│ Without Cache │ {:>10.3}s │ baseline │",
|
||||
uncached_time.as_secs_f64()
|
||||
);
|
||||
println!("└────────────────┴──────────────┴──────────────┘\n");
|
||||
|
||||
if speedup > 1.0 {
|
||||
println!("✅ Cache provides {:.1}% speedup!", (speedup - 1.0) * 100.0);
|
||||
} else {
|
||||
println!("⚠️ No speedup observed (may indicate all cache misses)");
|
||||
}
|
||||
|
||||
println!("\n💡 Note: Speedup increases with more repeated texts.");
|
||||
println!(" Real-world applications with 50-90% repeated queries");
|
||||
println!(" can see 2-10x improvements!\n");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
//! Example demonstrating local text embedding usage.
|
||||
//!
|
||||
//! This example shows how to:
|
||||
//! - Create a local text embedder with default configuration (BGE-small)
|
||||
//! - Generate embeddings for sample texts
|
||||
//! - Compute cosine similarity between embeddings
|
||||
//! - Batch process multiple texts
|
||||
//! - Use different models (BGE-base, Nomic, GTE-large)
|
||||
//!
|
||||
//! ## Prerequisites
|
||||
//!
|
||||
//! Before running this example, download the BGE-small model:
|
||||
//!
|
||||
//! ```bash
|
||||
//! mkdir -p ~/.cache/memvid/text-models
|
||||
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
//! ```
|
||||
//!
|
||||
//! ## Run
|
||||
//!
|
||||
//! ```bash
|
||||
//! cargo run --example text_embedding --features vec
|
||||
//! ```
|
||||
|
||||
use memvid_core::Result;
|
||||
#[cfg(feature = "vec")]
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
#[cfg(feature = "vec")]
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
/// Compute cosine similarity between two vectors
|
||||
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
assert_eq!(a.len(), b.len(), "Vectors must have same length");
|
||||
|
||||
let dot_product: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
|
||||
let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
|
||||
if norm_a > 0.0 && norm_b > 0.0 {
|
||||
dot_product / (norm_a * norm_b)
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
println!("=== Local Text Embedding Example ===\n");
|
||||
|
||||
// Create embedder with default config (BGE-small, 384 dimensions)
|
||||
println!("Creating local text embedder (BGE-small-en-v1.5)...");
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
println!("Model: {}", embedder.model());
|
||||
println!("Kind: {}", embedder.kind());
|
||||
println!("Dimensions: {}", embedder.dimension());
|
||||
println!("Ready: {}\n", embedder.is_ready());
|
||||
|
||||
// Example 1: Generate embeddings for sample texts
|
||||
println!("--- Example 1: Single Text Embeddings ---");
|
||||
let text1 = "The quick brown fox jumps over the lazy dog";
|
||||
let text2 = "A fast auburn canine leaps above an idle hound";
|
||||
let text3 = "Python is a programming language";
|
||||
|
||||
println!("Generating embeddings...");
|
||||
let emb1 = embedder.embed_text(text1)?;
|
||||
let emb2 = embedder.embed_text(text2)?;
|
||||
let emb3 = embedder.embed_text(text3)?;
|
||||
|
||||
println!("✓ Generated {} embeddings of dimension {}\n", 3, emb1.len());
|
||||
|
||||
// Example 2: Compute semantic similarity
|
||||
println!("--- Example 2: Semantic Similarity ---");
|
||||
let sim_1_2 = cosine_similarity(&emb1, &emb2);
|
||||
let sim_1_3 = cosine_similarity(&emb1, &emb3);
|
||||
let sim_2_3 = cosine_similarity(&emb2, &emb3);
|
||||
|
||||
println!("Text 1: \"{}\"", text1);
|
||||
println!("Text 2: \"{}\"", text2);
|
||||
println!("Text 3: \"{}\"", text3);
|
||||
println!();
|
||||
println!("Similarity (1 ↔ 2): {:.4}", sim_1_2);
|
||||
println!("Similarity (1 ↔ 3): {:.4}", sim_1_3);
|
||||
println!("Similarity (2 ↔ 3): {:.4}", sim_2_3);
|
||||
println!();
|
||||
|
||||
if sim_1_2 > sim_1_3 {
|
||||
println!("✓ As expected, similar texts (1 & 2) have higher similarity!");
|
||||
}
|
||||
println!();
|
||||
|
||||
// Example 3: Batch processing
|
||||
println!("--- Example 3: Batch Processing ---");
|
||||
let documents = vec![
|
||||
"Artificial intelligence and machine learning",
|
||||
"Deep neural networks for computer vision",
|
||||
"Natural language processing with transformers",
|
||||
"The history of ancient Rome",
|
||||
"Cooking recipes for Italian cuisine",
|
||||
];
|
||||
|
||||
println!("Processing {} documents in batch...", documents.len());
|
||||
let batch_embeddings = embedder.embed_batch(&documents)?;
|
||||
println!("✓ Generated {} embeddings\n", batch_embeddings.len());
|
||||
|
||||
// Find most similar pair
|
||||
println!("Finding most similar document pair...");
|
||||
let mut max_sim = 0.0;
|
||||
let mut max_pair = (0, 0);
|
||||
|
||||
for i in 0..batch_embeddings.len() {
|
||||
for j in (i + 1)..batch_embeddings.len() {
|
||||
let sim = cosine_similarity(&batch_embeddings[i], &batch_embeddings[j]);
|
||||
if sim > max_sim {
|
||||
max_sim = sim;
|
||||
max_pair = (i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("Most similar pair (similarity: {:.4}):", max_sim);
|
||||
println!(" [{}] \"{}\"", max_pair.0, documents[max_pair.0]);
|
||||
println!(" [{}] \"{}\"\n", max_pair.1, documents[max_pair.1]);
|
||||
|
||||
// Example 4: Search query use case
|
||||
println!("--- Example 4: Search Query ---");
|
||||
let query = "machine learning algorithms";
|
||||
let query_emb = embedder.embed_text(query)?;
|
||||
|
||||
println!("Query: \"{}\"", query);
|
||||
println!("\nRanked results:");
|
||||
|
||||
let mut scores: Vec<(usize, f32)> = batch_embeddings
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, emb)| (i, cosine_similarity(&query_emb, emb)))
|
||||
.collect();
|
||||
|
||||
scores.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
|
||||
|
||||
for (rank, (idx, score)) in scores.iter().take(3).enumerate() {
|
||||
println!(" {}. [{:.4}] \"{}\"", rank + 1, score, documents[*idx]);
|
||||
}
|
||||
println!();
|
||||
|
||||
// Example 5: Model unloading (memory management)
|
||||
println!("--- Example 5: Memory Management ---");
|
||||
println!("Model loaded: {}", embedder.is_loaded());
|
||||
embedder.unload()?;
|
||||
println!("After unload: {}", embedder.is_loaded());
|
||||
println!("✓ Model can be lazily reloaded on next use\n");
|
||||
|
||||
// Example 6: Using different models (commented out - requires model download)
|
||||
println!("--- Example 6: Different Models ---");
|
||||
println!("Available models:");
|
||||
println!(" - bge-small-en-v1.5 (384d) - Default, fast");
|
||||
println!(" - bge-base-en-v1.5 (768d) - Better quality");
|
||||
println!(" - nomic-embed-text-v1.5 (768d) - Versatile");
|
||||
println!(" - gte-large (1024d) - Highest quality");
|
||||
println!();
|
||||
println!("To use a different model:");
|
||||
println!(" let config = TextEmbedConfig::bge_base();");
|
||||
println!(" let embedder = LocalTextEmbedder::new(config)?;");
|
||||
println!();
|
||||
|
||||
println!("=== Example Complete ===");
|
||||
println!("\nKey takeaways:");
|
||||
println!("✓ Local embeddings run entirely offline (no API calls)");
|
||||
println!("✓ Models are lazy-loaded on first use");
|
||||
println!("✓ Embeddings are L2-normalized for cosine similarity");
|
||||
println!("✓ Batch processing is efficient for multiple texts");
|
||||
println!("✓ Similar texts have higher cosine similarity scores");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
# Memvid Installer
|
||||
|
||||
Official installer for Memvid (v1).
|
||||
|
||||
## What it does
|
||||
|
||||
The installer automatically checks for and installs required dependencies, then installs Memvid globally:
|
||||
|
||||
- **System dependencies** (Linux only):
|
||||
- **ca-certificates** - SSL certificate bundle
|
||||
- **curl** - Command-line tool for downloading files
|
||||
- **openssl** or **libssl3** - SSL/TLS library
|
||||
- **git** - Version control system
|
||||
- **node** (LTS) - JavaScript runtime
|
||||
- **npm** - Package manager (comes with node)
|
||||
- **memvid** - Installed globally via `npm install -g memvid-cli@latest`
|
||||
|
||||
The installer only installs missing tools (no upgrades in v1). If a tool is already installed, it will be detected and skipped.
|
||||
|
||||
**Important notes:**
|
||||
- On macOS, if Xcode Command Line Tools are missing, the installer will prompt you to install them and then exit. You'll need to complete the Xcode installation and run the installer again.
|
||||
- The installer will show you what will be installed and ask for confirmation before proceeding.
|
||||
- On Linux, Node.js LTS is installed via the NodeSource repository for Debian/Ubuntu and RHEL-based distributions to ensure you get the latest LTS version.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **macOS**:
|
||||
- Homebrew must be installed (the installer will check and exit if missing)
|
||||
- Xcode Command Line Tools (installer will prompt to install if missing)
|
||||
- **Linux**:
|
||||
- `sudo` access (required for package installation, except Alpine which may use `su`)
|
||||
- One of the supported package managers (apt, dnf, pacman, or apk)
|
||||
- **Windows**:
|
||||
- winget (Windows Package Manager) - comes with Windows 11, or install from https://aka.ms/getwinget
|
||||
|
||||
## Supported Platforms
|
||||
|
||||
- **macOS** - Uses Homebrew (requires Xcode Command Line Tools)
|
||||
- **Linux** - Multiple distributions supported:
|
||||
- **Debian/Ubuntu** - Uses apt (Node.js installed via NodeSource repository)
|
||||
- **Fedora/RHEL/CentOS/Rocky/AlmaLinux** - Uses dnf (Node.js installed via NodeSource repository)
|
||||
- **Arch/Manjaro** - Uses pacman
|
||||
- **Alpine** - Uses apk
|
||||
- **Windows** - Requires winget (Windows Package Manager)
|
||||
|
||||
## Installation
|
||||
|
||||
### macOS / Linux
|
||||
|
||||
**Quick install** (recommended):
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/memvid/memvid/main/install/install.sh | bash
|
||||
```
|
||||
|
||||
**Download and run locally**:
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/memvid/memvid/main/install/install.sh -o install.sh
|
||||
chmod +x install.sh
|
||||
./install.sh
|
||||
```
|
||||
|
||||
**Non-interactive mode** (for CI/CD):
|
||||
The installer can run without user interaction when piped via `curl | bash`. It will proceed automatically if no terminal is detected.
|
||||
|
||||
### Windows
|
||||
|
||||
Open PowerShell and run:
|
||||
|
||||
```powershell
|
||||
irm https://raw.githubusercontent.com/memvid/memvid/main/install/install.ps1 | iex
|
||||
```
|
||||
|
||||
Or download and run:
|
||||
|
||||
```powershell
|
||||
.\install.ps1
|
||||
```
|
||||
|
||||
## Security
|
||||
|
||||
These scripts are open source and readable. Before running, you can:
|
||||
|
||||
1. Review the script contents on GitHub
|
||||
2. Download and inspect locally
|
||||
3. Run with appropriate permissions
|
||||
|
||||
The installer will:
|
||||
- Show what will be installed before proceeding
|
||||
- Ask for confirmation before installing anything
|
||||
- Use only system package managers (no third-party installers)
|
||||
- Print clear status messages for each step
|
||||
|
||||
## Verification
|
||||
|
||||
After installation, verify it worked:
|
||||
|
||||
```bash
|
||||
memvid --version
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
If installation fails:
|
||||
|
||||
1. **macOS**:
|
||||
- Ensure Homebrew is installed: `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"`
|
||||
- If Xcode CLI Tools installation was prompted, complete it and run the installer again
|
||||
- If `node` command is not found after installation, you may need to restart your terminal or source your shell config file
|
||||
|
||||
2. **Linux**:
|
||||
- Check that you have `sudo` access (or run as root on Alpine)
|
||||
- Ensure your package manager is up to date (`sudo apt-get update`, `sudo dnf check-update`, etc.)
|
||||
- If Node.js installation fails, check that curl and ca-certificates are installed
|
||||
- Verify npm global bin directory is in your PATH: `npm config get prefix`
|
||||
|
||||
3. **Windows**:
|
||||
- Ensure winget is installed (comes with Windows 11, or install from https://aka.ms/getwinget)
|
||||
- Run PowerShell as Administrator if needed
|
||||
|
||||
4. **General**:
|
||||
- Check that npm global bin directory is in your PATH: `npm list -g memvid-cli`
|
||||
- Try running `memvid --version` to verify installation
|
||||
- On Linux, you may need to use `sudo` for global npm installs
|
||||
|
||||
For issues, please open an issue on GitHub.
|
||||
@@ -0,0 +1,229 @@
|
||||
# Memvid Installer for Windows
|
||||
# v1 - System package managers only, install-if-missing
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# Colors for output
|
||||
function Write-Success {
|
||||
Write-Host "✔ $args" -ForegroundColor Green
|
||||
}
|
||||
|
||||
function Write-Error {
|
||||
Write-Host "✖ $args" -ForegroundColor Red
|
||||
}
|
||||
|
||||
function Write-Info {
|
||||
Write-Host "→ $args" -ForegroundColor Cyan
|
||||
}
|
||||
|
||||
function Write-Warning {
|
||||
Write-Host "⚠ $args" -ForegroundColor Yellow
|
||||
}
|
||||
|
||||
# Check if command exists
|
||||
function Test-Command {
|
||||
param([string]$Command)
|
||||
$null -ne (Get-Command $Command -ErrorAction SilentlyContinue)
|
||||
}
|
||||
|
||||
# Check for winget
|
||||
function Test-Winget {
|
||||
if (Test-Command winget) {
|
||||
$wingetVersion = (winget --version)
|
||||
Write-Success "winget already installed (version $wingetVersion)"
|
||||
return $true
|
||||
} else {
|
||||
Write-Error "winget not found"
|
||||
Write-Info "winget is required for v1 installer"
|
||||
Write-Info "Please install winget first:"
|
||||
Write-Info " https://aka.ms/getwinget"
|
||||
Write-Info ""
|
||||
Write-Info "Or install git and node manually, then run:"
|
||||
Write-Info " npm install -g memvid-cli@latest"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Check for git
|
||||
function Test-Git {
|
||||
if (Test-Command git) {
|
||||
$gitVersion = (git --version)
|
||||
Write-Success "git already installed ($gitVersion)"
|
||||
return $true
|
||||
} else {
|
||||
Write-Error "git not found"
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
# Check for node
|
||||
function Test-Node {
|
||||
if (Test-Command node) {
|
||||
$nodeVersion = (node --version)
|
||||
Write-Success "node already installed ($nodeVersion)"
|
||||
|
||||
if (Test-Command npm) {
|
||||
$npmVersion = (npm --version)
|
||||
Write-Success "npm already installed (version $npmVersion)"
|
||||
return $true
|
||||
} else {
|
||||
Write-Error "npm not found (should come with node)"
|
||||
return $false
|
||||
}
|
||||
} else {
|
||||
Write-Error "node not found"
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
# Install git
|
||||
function Install-Git {
|
||||
Write-Info "Installing git using winget..."
|
||||
|
||||
try {
|
||||
winget install --id Git.Git -e --silent --accept-package-agreements --accept-source-agreements
|
||||
# Refresh PATH
|
||||
$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
|
||||
|
||||
if (Test-Command git) {
|
||||
Write-Success "git installed successfully"
|
||||
} else {
|
||||
Write-Error "git installation completed but not found in PATH"
|
||||
Write-Info "Please restart your terminal and run the installer again"
|
||||
exit 1
|
||||
}
|
||||
} catch {
|
||||
Write-Error "git installation failed: $_"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Install node (LTS)
|
||||
function Install-Node {
|
||||
Write-Info "Installing node (LTS) using winget..."
|
||||
|
||||
try {
|
||||
winget install --id OpenJS.NodeJS.LTS -e --silent --accept-package-agreements --accept-source-agreements
|
||||
# Refresh PATH
|
||||
$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
|
||||
|
||||
if (Test-Command node) {
|
||||
$nodeVersion = (node --version)
|
||||
$npmVersion = (npm --version)
|
||||
Write-Success "node installed successfully ($nodeVersion)"
|
||||
Write-Success "npm installed successfully (version $npmVersion)"
|
||||
} else {
|
||||
Write-Error "node installation completed but not found in PATH"
|
||||
Write-Info "Please restart your terminal and run the installer again"
|
||||
exit 1
|
||||
}
|
||||
} catch {
|
||||
Write-Error "node installation failed: $_"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Check if memvid is already installed
|
||||
function Test-Memvid {
|
||||
if (Test-Command memvid) {
|
||||
try {
|
||||
$memvidVersion = memvid --version 2>$null
|
||||
Write-Success "memvid already installed ($memvidVersion)"
|
||||
return $true
|
||||
} catch {
|
||||
Write-Success "memvid already installed"
|
||||
return $true
|
||||
}
|
||||
} else {
|
||||
Write-Error "memvid not found"
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
# Install missing tools
|
||||
function Install-Missing {
|
||||
$needsGit = -not (Test-Git)
|
||||
$needsNode = -not (Test-Node)
|
||||
$needsMemvid = -not (Test-Memvid)
|
||||
|
||||
if (-not $needsGit -and -not $needsNode -and -not $needsMemvid) {
|
||||
Write-Info "All dependencies are already installed"
|
||||
return
|
||||
}
|
||||
|
||||
# Show what will be installed
|
||||
Write-Host ""
|
||||
Write-Warning "The following tools will be installed:"
|
||||
if ($needsGit) { Write-Host " - git" }
|
||||
if ($needsNode) { Write-Host " - node (LTS)" }
|
||||
if ($needsMemvid) { Write-Host " - memvid-cli (latest)" }
|
||||
Write-Host ""
|
||||
|
||||
# Ask for confirmation
|
||||
$response = Read-Host "Continue? [Y/n]"
|
||||
if ($response -ne "" -and $response -notmatch "^[Yy]$") {
|
||||
Write-Info "Installation cancelled"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Install missing tools
|
||||
if ($needsGit) { Install-Git }
|
||||
if ($needsNode) { Install-Node }
|
||||
if ($needsMemvid) { Install-Memvid }
|
||||
}
|
||||
|
||||
# Install memvid
|
||||
function Install-Memvid {
|
||||
Write-Info "Installing memvid globally..."
|
||||
|
||||
try {
|
||||
npm install -g memvid-cli@latest
|
||||
Write-Success "memvid installed successfully"
|
||||
} catch {
|
||||
Write-Error "memvid installation failed: $_"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Verify installation
|
||||
function Verify-Installation {
|
||||
Write-Info "Verifying installation..."
|
||||
|
||||
if (Test-Command memvid) {
|
||||
try {
|
||||
$memvidVersion = memvid --version 2>$null
|
||||
Write-Success "memvid is installed and accessible"
|
||||
Write-Info "Version: $memvidVersion"
|
||||
Write-Host ""
|
||||
Write-Success "Installation complete! You can now use 'memvid' command."
|
||||
} catch {
|
||||
Write-Success "memvid is installed and accessible"
|
||||
Write-Host ""
|
||||
Write-Success "Installation complete! You can now use 'memvid' command."
|
||||
}
|
||||
} else {
|
||||
Write-Error "memvid verification failed"
|
||||
Write-Info "The installation may have completed, but 'memvid' command is not in PATH"
|
||||
Write-Info "Please check your npm global bin directory and add it to PATH if needed"
|
||||
Write-Info "Or try: npm list -g memvid-cli"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Main execution
|
||||
function Main {
|
||||
Write-Host "Memvid Installer"
|
||||
Write-Host "Checking system requirements…"
|
||||
Write-Host ""
|
||||
|
||||
Test-Winget
|
||||
Write-Host ""
|
||||
|
||||
Install-Missing
|
||||
Write-Host ""
|
||||
|
||||
Verify-Installation
|
||||
}
|
||||
|
||||
# Run main function
|
||||
Main
|
||||
Executable
+497
@@ -0,0 +1,497 @@
|
||||
#!/bin/bash
|
||||
# Memvid Installer for macOS and Linux
|
||||
# v1 - System package managers only, install-if-missing
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Print success message
|
||||
print_success() {
|
||||
echo -e "${GREEN}✔${NC} $1"
|
||||
}
|
||||
|
||||
# Print error message
|
||||
print_error() {
|
||||
echo -e "${RED}✖${NC} $1"
|
||||
}
|
||||
|
||||
# Print info message
|
||||
print_info() {
|
||||
echo -e "${BLUE}→${NC} $1"
|
||||
}
|
||||
|
||||
# Print warning message
|
||||
print_warning() {
|
||||
echo -e "${YELLOW}⚠${NC} $1"
|
||||
}
|
||||
|
||||
# Detect OS
|
||||
detect_os() {
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
OS="macos"
|
||||
PKG_MANAGER="brew"
|
||||
|
||||
# Check if Homebrew is installed
|
||||
if ! command_exists brew; then
|
||||
print_error "Homebrew is not installed"
|
||||
print_info "Please install Homebrew first:"
|
||||
print_info " /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\""
|
||||
exit 1
|
||||
fi
|
||||
elif [[ -f /etc/os-release ]]; then
|
||||
. /etc/os-release
|
||||
DISTRO_NAME="${PRETTY_NAME:-${NAME:-$ID}}"
|
||||
|
||||
if [[ "$ID" == "ubuntu" ]] || [[ "$ID" == "debian" ]]; then
|
||||
OS="linux"
|
||||
PKG_MANAGER="apt"
|
||||
DISTRO_FAMILY="debian"
|
||||
elif [[ "$ID" == "fedora" ]] || [[ "$ID" == "rhel" ]] || [[ "$ID" == "centos" ]] || [[ "$ID" == "rocky" ]] || [[ "$ID" == "almalinux" ]]; then
|
||||
OS="linux"
|
||||
PKG_MANAGER="dnf"
|
||||
DISTRO_FAMILY="rhel"
|
||||
elif [[ "$ID" == "arch" ]] || [[ "$ID" == "manjaro" ]]; then
|
||||
OS="linux"
|
||||
PKG_MANAGER="pacman"
|
||||
DISTRO_FAMILY="arch"
|
||||
elif [[ "$ID" == "alpine" ]]; then
|
||||
OS="linux"
|
||||
PKG_MANAGER="apk"
|
||||
DISTRO_FAMILY="alpine"
|
||||
else
|
||||
print_error "Unsupported Linux distribution: $ID"
|
||||
print_info "Supported distributions: Ubuntu, Debian, Fedora, RHEL, CentOS, Arch, Alpine"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if sudo is available (except for Alpine which might use su)
|
||||
if ! command_exists sudo && [[ "$DISTRO_FAMILY" != "alpine" ]]; then
|
||||
print_error "sudo is not available"
|
||||
print_info "Please install sudo or run as root"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
print_error "Unable to detect operating system"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$OS" == "linux" ]]; then
|
||||
print_info "Detected OS: $OS ($DISTRO_NAME)"
|
||||
else
|
||||
print_info "Detected OS: $OS"
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if command exists
|
||||
command_exists() {
|
||||
command -v "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Check for Xcode Command Line Tools on macOS
|
||||
check_xcode_cli_tools() {
|
||||
if [[ "$OS" != "macos" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check if Xcode CLI Tools are installed
|
||||
if xcode-select -p &>/dev/null; then
|
||||
print_success "Xcode Command Line Tools already installed"
|
||||
return 0
|
||||
else
|
||||
print_error "Xcode Command Line Tools not found"
|
||||
print_warning "Xcode Command Line Tools are required for everything to work properly"
|
||||
echo ""
|
||||
|
||||
# Ask for confirmation
|
||||
if [[ -t 0 ]] && [[ -t 1 ]]; then
|
||||
read -p "Install Xcode Command Line Tools now? [Y/n] " -n 1 -r
|
||||
echo
|
||||
elif [[ -c /dev/tty ]]; then
|
||||
read -p "Install Xcode Command Line Tools now? [Y/n] " -n 1 -r < /dev/tty
|
||||
echo
|
||||
else
|
||||
REPLY="Y"
|
||||
fi
|
||||
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]] && [[ ! $REPLY == "" ]]; then
|
||||
print_error "Xcode Command Line Tools are required for everything to work properly"
|
||||
print_info "Please install them manually with: xcode-select --install"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
print_info "Installing Xcode Command Line Tools..."
|
||||
xcode-select --install
|
||||
|
||||
print_info "Please complete the Xcode Command Line Tools installation in the popup window"
|
||||
print_info "Then run this installer again"
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
# Install system dependencies for Linux
|
||||
install_system_deps() {
|
||||
if [[ "$OS" != "linux" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
print_info "Checking system dependencies..."
|
||||
|
||||
NEEDS_DEPS=false
|
||||
DEPS_TO_INSTALL=()
|
||||
|
||||
# Check and collect missing dependencies
|
||||
if [[ "$DISTRO_FAMILY" == "debian" ]]; then
|
||||
if ! dpkg -l | grep -q "^ii.*ca-certificates"; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("ca-certificates")
|
||||
fi
|
||||
if ! command_exists curl; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("curl")
|
||||
fi
|
||||
if ! dpkg -l | grep -q "^ii.*libssl3"; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("libssl3")
|
||||
fi
|
||||
elif [[ "$DISTRO_FAMILY" == "rhel" ]]; then
|
||||
if ! rpm -q ca-certificates &>/dev/null; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("ca-certificates")
|
||||
fi
|
||||
if ! command_exists curl; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("curl")
|
||||
fi
|
||||
if ! rpm -q openssl &>/dev/null; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("openssl")
|
||||
fi
|
||||
elif [[ "$DISTRO_FAMILY" == "arch" ]]; then
|
||||
if ! pacman -Q ca-certificates &>/dev/null; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("ca-certificates")
|
||||
fi
|
||||
if ! command_exists curl; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("curl")
|
||||
fi
|
||||
if ! pacman -Q openssl &>/dev/null; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("openssl")
|
||||
fi
|
||||
elif [[ "$DISTRO_FAMILY" == "alpine" ]]; then
|
||||
if ! apk info -e ca-certificates &>/dev/null; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("ca-certificates")
|
||||
fi
|
||||
if ! command_exists curl; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("curl")
|
||||
fi
|
||||
if ! apk info -e openssl &>/dev/null; then
|
||||
NEEDS_DEPS=true
|
||||
DEPS_TO_INSTALL+=("openssl")
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$NEEDS_DEPS" == false ]]; then
|
||||
print_success "All system dependencies are installed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
print_info "Installing system dependencies: ${DEPS_TO_INSTALL[*]}..."
|
||||
|
||||
if [[ "$DISTRO_FAMILY" == "debian" ]]; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends "${DEPS_TO_INSTALL[@]}"
|
||||
elif [[ "$DISTRO_FAMILY" == "rhel" ]]; then
|
||||
sudo dnf install -y "${DEPS_TO_INSTALL[@]}"
|
||||
elif [[ "$DISTRO_FAMILY" == "arch" ]]; then
|
||||
sudo pacman -S --noconfirm "${DEPS_TO_INSTALL[@]}"
|
||||
elif [[ "$DISTRO_FAMILY" == "alpine" ]]; then
|
||||
if command_exists sudo; then
|
||||
sudo apk add --no-cache "${DEPS_TO_INSTALL[@]}"
|
||||
else
|
||||
apk add --no-cache "${DEPS_TO_INSTALL[@]}"
|
||||
fi
|
||||
fi
|
||||
|
||||
print_success "System dependencies installed successfully"
|
||||
}
|
||||
|
||||
# Check for git
|
||||
check_git() {
|
||||
if command_exists git; then
|
||||
GIT_VERSION=$(git --version | cut -d' ' -f3)
|
||||
print_success "git already installed (version $GIT_VERSION)"
|
||||
return 0
|
||||
else
|
||||
print_error "git not found"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check for node
|
||||
check_node() {
|
||||
if command_exists node; then
|
||||
NODE_VERSION=$(node --version)
|
||||
print_success "node already installed ($NODE_VERSION)"
|
||||
|
||||
# Check if it's LTS (rough check - version should be even major version)
|
||||
MAJOR_VERSION=$(echo "$NODE_VERSION" | cut -d'v' -f2 | cut -d'.' -f1)
|
||||
if [ "$((MAJOR_VERSION % 2))" -eq 0 ]; then
|
||||
print_info "Node version appears to be LTS-compatible"
|
||||
fi
|
||||
|
||||
# Check for npm
|
||||
if command_exists npm; then
|
||||
NPM_VERSION=$(npm --version)
|
||||
print_success "npm already installed (version $NPM_VERSION)"
|
||||
return 0
|
||||
else
|
||||
print_error "npm not found (should come with node)"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
print_error "node not found"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Install git
|
||||
install_git() {
|
||||
print_info "Installing git using $PKG_MANAGER..."
|
||||
|
||||
if [[ "$OS" == "macos" ]]; then
|
||||
brew install git
|
||||
elif [[ "$OS" == "linux" ]]; then
|
||||
if [[ "$DISTRO_FAMILY" == "debian" ]]; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y git
|
||||
elif [[ "$DISTRO_FAMILY" == "rhel" ]]; then
|
||||
sudo dnf install -y git
|
||||
elif [[ "$DISTRO_FAMILY" == "arch" ]]; then
|
||||
sudo pacman -S --noconfirm git
|
||||
elif [[ "$DISTRO_FAMILY" == "alpine" ]]; then
|
||||
if command_exists sudo; then
|
||||
sudo apk add --no-cache git
|
||||
else
|
||||
apk add --no-cache git
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if command_exists git; then
|
||||
print_success "git installed successfully"
|
||||
else
|
||||
print_error "git installation failed"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Install node (LTS)
|
||||
install_node() {
|
||||
print_info "Installing node (LTS) using $PKG_MANAGER..."
|
||||
|
||||
if [[ "$OS" == "macos" ]]; then
|
||||
brew install node@lts
|
||||
# Add to PATH if needed
|
||||
if ! command_exists node; then
|
||||
print_info "Adding node to PATH..."
|
||||
# Detect Homebrew prefix (Apple Silicon vs Intel)
|
||||
if [[ -d "/opt/homebrew" ]]; then
|
||||
BREW_PREFIX="/opt/homebrew"
|
||||
else
|
||||
BREW_PREFIX="/usr/local"
|
||||
fi
|
||||
|
||||
# Detect shell
|
||||
if [[ "$SHELL" == *"zsh"* ]]; then
|
||||
SHELL_RC="$HOME/.zshrc"
|
||||
else
|
||||
SHELL_RC="$HOME/.bash_profile"
|
||||
fi
|
||||
|
||||
echo "export PATH=\"$BREW_PREFIX/opt/node@lts/bin:\$PATH\"" >> "$SHELL_RC"
|
||||
export PATH="$BREW_PREFIX/opt/node@lts/bin:$PATH"
|
||||
fi
|
||||
elif [[ "$OS" == "linux" ]]; then
|
||||
if [[ "$DISTRO_FAMILY" == "debian" ]]; then
|
||||
# Install Node.js LTS from NodeSource
|
||||
curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash -
|
||||
sudo apt-get install -y --no-install-recommends nodejs
|
||||
elif [[ "$DISTRO_FAMILY" == "rhel" ]]; then
|
||||
# Install Node.js LTS from NodeSource
|
||||
curl -fsSL https://rpm.nodesource.com/setup_lts.x | sudo -E bash -
|
||||
sudo dnf install -y nodejs
|
||||
elif [[ "$DISTRO_FAMILY" == "arch" ]]; then
|
||||
sudo pacman -S --noconfirm nodejs npm
|
||||
elif [[ "$DISTRO_FAMILY" == "alpine" ]]; then
|
||||
if command_exists sudo; then
|
||||
sudo apk add --no-cache nodejs npm
|
||||
else
|
||||
apk add --no-cache nodejs npm
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if command_exists node && command_exists npm; then
|
||||
NODE_VERSION=$(node --version)
|
||||
NPM_VERSION=$(npm --version)
|
||||
print_success "node installed successfully ($NODE_VERSION)"
|
||||
print_success "npm installed successfully (version $NPM_VERSION)"
|
||||
else
|
||||
print_error "node installation failed"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if memvid is already installed
|
||||
check_memvid() {
|
||||
if command_exists memvid; then
|
||||
# Get version - output format is "memvid 2.0.131"
|
||||
MEMVID_VERSION=$(memvid --version 2>&1 | awk '{print $2}')
|
||||
print_success "memvid already installed ($MEMVID_VERSION)"
|
||||
return 0
|
||||
else
|
||||
print_error "memvid not found"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Install missing tools
|
||||
install_missing() {
|
||||
NEEDS_GIT=false
|
||||
NEEDS_NODE=false
|
||||
NEEDS_MEMVID=false
|
||||
|
||||
if ! check_git; then
|
||||
NEEDS_GIT=true
|
||||
fi
|
||||
|
||||
if ! check_node; then
|
||||
NEEDS_NODE=true
|
||||
fi
|
||||
|
||||
if ! check_memvid; then
|
||||
NEEDS_MEMVID=true
|
||||
fi
|
||||
|
||||
if [[ "$NEEDS_GIT" == false ]] && [[ "$NEEDS_NODE" == false ]] && [[ "$NEEDS_MEMVID" == false ]]; then
|
||||
print_info "All dependencies are already installed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Show what will be installed
|
||||
echo ""
|
||||
print_warning "The following tools will be installed:"
|
||||
[[ "$NEEDS_GIT" == true ]] && echo " - git"
|
||||
[[ "$NEEDS_NODE" == true ]] && echo " - node (LTS)"
|
||||
[[ "$NEEDS_MEMVID" == true ]] && echo " - memvid-cli (latest)"
|
||||
echo ""
|
||||
|
||||
# Ask for confirmation
|
||||
# Read from /dev/tty to ensure it works when piped via curl | bash
|
||||
if [[ -t 0 ]] && [[ -t 1 ]]; then
|
||||
# Interactive terminal - read normally
|
||||
read -p "Continue? [Y/n] " -n 1 -r
|
||||
echo
|
||||
elif [[ -c /dev/tty ]]; then
|
||||
# Piped input - read from terminal device
|
||||
read -p "Continue? [Y/n] " -n 1 -r < /dev/tty
|
||||
echo
|
||||
else
|
||||
# No terminal available - proceed automatically (non-interactive mode)
|
||||
print_info "No terminal detected, proceeding with installation..."
|
||||
REPLY="Y"
|
||||
fi
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]] && [[ ! $REPLY == "" ]]; then
|
||||
print_info "Installation cancelled"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Install missing tools
|
||||
[[ "$NEEDS_GIT" == true ]] && install_git
|
||||
[[ "$NEEDS_NODE" == true ]] && install_node
|
||||
[[ "$NEEDS_MEMVID" == true ]] && install_memvid
|
||||
}
|
||||
|
||||
# Install memvid
|
||||
install_memvid() {
|
||||
print_info "Installing memvid globally..."
|
||||
|
||||
if [[ "$OS" == "linux" ]]; then
|
||||
# Linux requires sudo for global npm installs
|
||||
if sudo npm install -g memvid-cli@latest; then
|
||||
print_success "memvid installed successfully"
|
||||
else
|
||||
print_error "memvid installation failed"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
# macOS typically doesn't need sudo if npm was installed via Homebrew
|
||||
if npm install -g memvid-cli@latest; then
|
||||
print_success "memvid installed successfully"
|
||||
else
|
||||
print_error "memvid installation failed"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Verify installation
|
||||
verify() {
|
||||
print_info "Verifying installation..."
|
||||
|
||||
if command_exists memvid; then
|
||||
# Get version - output format is "memvid 2.0.131"
|
||||
MEMVID_VERSION=$(memvid --version 2>&1 | awk '{print $2}')
|
||||
|
||||
print_success "memvid is installed and accessible"
|
||||
print_info "Version: $MEMVID_VERSION"
|
||||
echo ""
|
||||
print_success "Installation complete! You can now use 'memvid' command."
|
||||
else
|
||||
print_error "memvid verification failed"
|
||||
print_info "The installation may have completed, but 'memvid' command is not in PATH"
|
||||
print_info "Please check your npm global bin directory and add it to PATH if needed"
|
||||
print_info "Or try: npm list -g memvid-cli"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
echo "Memvid Installer"
|
||||
echo "Checking system requirements…"
|
||||
echo ""
|
||||
|
||||
detect_os
|
||||
echo ""
|
||||
|
||||
# Check Xcode CLI Tools on macOS
|
||||
if [[ "$OS" == "macos" ]]; then
|
||||
check_xcode_cli_tools
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Install system dependencies on Linux
|
||||
if [[ "$OS" == "linux" ]]; then
|
||||
install_system_deps
|
||||
echo ""
|
||||
fi
|
||||
|
||||
install_missing
|
||||
echo ""
|
||||
|
||||
verify
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main
|
||||
@@ -0,0 +1,3 @@
|
||||
[toolchain]
|
||||
channel = "1.90.0"
|
||||
targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"]
|
||||
@@ -0,0 +1,249 @@
|
||||
// Safe expect/unwrap: Regex patterns are compile-time literals; JSON ops on known schemas.
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used)]
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AutoTagResult {
|
||||
pub tags: Vec<String>,
|
||||
pub labels: Vec<String>,
|
||||
pub content_dates: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct AutoTagger;
|
||||
|
||||
impl AutoTagger {
|
||||
const MAX_TAGS: usize = 12;
|
||||
const MAX_LABELS: usize = 6;
|
||||
|
||||
pub fn analyse(&self, text: &str, include_dates: bool) -> AutoTagResult {
|
||||
if text.trim().is_empty() {
|
||||
return AutoTagResult::default();
|
||||
}
|
||||
|
||||
let tags = extract_keywords(text, Self::MAX_TAGS);
|
||||
let labels = derive_labels(text, Self::MAX_LABELS);
|
||||
let content_dates = if include_dates {
|
||||
extract_dates(text)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
AutoTagResult {
|
||||
tags,
|
||||
labels,
|
||||
content_dates,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_keywords(text: &str, limit: usize) -> Vec<String> {
|
||||
static TOKEN_RE: std::sync::LazyLock<Regex> =
|
||||
std::sync::LazyLock::new(|| Regex::new(r"(?i)[a-z0-9][a-z0-9'-]+").unwrap());
|
||||
static STOPWORDS: std::sync::LazyLock<BTreeSet<&'static str>> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
[
|
||||
"the",
|
||||
"and",
|
||||
"for",
|
||||
"with",
|
||||
"that",
|
||||
"from",
|
||||
"this",
|
||||
"were",
|
||||
"have",
|
||||
"has",
|
||||
"will",
|
||||
"shall",
|
||||
"into",
|
||||
"about",
|
||||
"without",
|
||||
"within",
|
||||
"between",
|
||||
"because",
|
||||
"over",
|
||||
"under",
|
||||
"after",
|
||||
"before",
|
||||
"until",
|
||||
"while",
|
||||
"their",
|
||||
"there",
|
||||
"these",
|
||||
"those",
|
||||
"your",
|
||||
"into",
|
||||
"such",
|
||||
"been",
|
||||
"where",
|
||||
"when",
|
||||
"which",
|
||||
"using",
|
||||
"also",
|
||||
"than",
|
||||
"could",
|
||||
"would",
|
||||
"should",
|
||||
"might",
|
||||
"cannot",
|
||||
"however",
|
||||
"therefore",
|
||||
"thereof",
|
||||
"hereby",
|
||||
"herein",
|
||||
"hereof",
|
||||
"based",
|
||||
"system",
|
||||
"application",
|
||||
"service",
|
||||
"provide",
|
||||
"provided",
|
||||
"including",
|
||||
"include",
|
||||
"includes",
|
||||
"version",
|
||||
"update",
|
||||
"updates",
|
||||
"usage",
|
||||
]
|
||||
.into_iter()
|
||||
.collect()
|
||||
});
|
||||
|
||||
let mut counts: BTreeMap<String, u32> = BTreeMap::new();
|
||||
for token in TOKEN_RE.find_iter(text) {
|
||||
let candidate = token.as_str().to_lowercase();
|
||||
if candidate.len() < 3 {
|
||||
continue;
|
||||
}
|
||||
if STOPWORDS.contains(candidate.as_str()) {
|
||||
continue;
|
||||
}
|
||||
*counts.entry(candidate).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
let mut scored: Vec<(String, u32)> = counts.into_iter().collect();
|
||||
scored.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
|
||||
scored
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|(token, _)| token)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn derive_labels(text: &str, limit: usize) -> Vec<String> {
|
||||
static PHRASE_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
|
||||
Regex::new(r"(?m)^(?P<phrase>[A-Z][A-Za-z0-9 &/-]{3,})$").unwrap()
|
||||
});
|
||||
|
||||
let mut labels = BTreeSet::new();
|
||||
for caps in PHRASE_RE.captures_iter(text) {
|
||||
if let Some(phrase) = caps.name("phrase") {
|
||||
let candidate = phrase.as_str().trim();
|
||||
if candidate.split_whitespace().count() <= 6 {
|
||||
labels.insert(candidate.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if labels.is_empty() {
|
||||
// fallback: promote top keywords by capitalising them
|
||||
let keywords = extract_keywords(text, limit);
|
||||
for kw in keywords {
|
||||
let mut chars = kw.chars();
|
||||
if let Some(first) = chars.next() {
|
||||
let mut label = first.to_uppercase().collect::<String>();
|
||||
label.push_str(chars.as_str());
|
||||
labels.insert(label);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
labels.into_iter().take(limit).collect()
|
||||
}
|
||||
|
||||
fn extract_dates(text: &str) -> Vec<String> {
|
||||
// Match various date formats:
|
||||
// 1. Years: 2024, 2025, etc.
|
||||
// 2. ISO dates: 2024-09-01
|
||||
// 3. US format: 09/01/2024
|
||||
// 4. Spelled out: September 1, 2024 or Sept 1, 2024 or 1 September 2024
|
||||
static DATE_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
|
||||
Regex::new(r"(?i)\b((?:19|20)\d{2}|\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4})\b").unwrap()
|
||||
});
|
||||
|
||||
// Match spelled-out dates like "September 1, 2024", "Sept 10, 2024", "September 1st, 2024"
|
||||
static SPELLED_DATE_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"(?i)\b((?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\.?\s+\d{1,2}(?:st|nd|rd|th)?,?\s+(?:19|20)\d{2})\b"
|
||||
).unwrap()
|
||||
});
|
||||
|
||||
// Match European format: "1 September 2024", "1st September 2024"
|
||||
static EURO_DATE_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"(?i)\b(\d{1,2}(?:st|nd|rd|th)?\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\.?\s+(?:19|20)\d{2})\b"
|
||||
).unwrap()
|
||||
});
|
||||
|
||||
let mut dates = BTreeSet::new();
|
||||
|
||||
// Extract numeric dates and years
|
||||
for capture in DATE_RE.captures_iter(text) {
|
||||
if let Some(m) = capture.get(0) {
|
||||
dates.insert(m.as_str().to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// Extract spelled-out dates (US format)
|
||||
for capture in SPELLED_DATE_RE.captures_iter(text) {
|
||||
if let Some(m) = capture.get(1) {
|
||||
dates.insert(m.as_str().to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// Extract European format dates
|
||||
for capture in EURO_DATE_RE.captures_iter(text) {
|
||||
if let Some(m) = capture.get(1) {
|
||||
dates.insert(m.as_str().to_string());
|
||||
}
|
||||
}
|
||||
|
||||
dates.into_iter().collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{AutoTagger, extract_dates};
|
||||
|
||||
#[test]
|
||||
fn produces_keywords_and_labels() {
|
||||
let text = "Rust memory engines power efficient systems. Memory safety ensures reliability in 2025.";
|
||||
let result = AutoTagger.analyse(text, true);
|
||||
assert!(result.tags.iter().any(|tag| tag.contains("memory")));
|
||||
assert!(!result.content_dates.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_dates() {
|
||||
let dates = extract_dates("Meeting on 2025-10-08 and follow-up 10/15/2025");
|
||||
assert_eq!(dates.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_spelled_out_dates() {
|
||||
let dates = extract_dates(
|
||||
"The update on September 1, 2024 changed the phone number. Previous records from January 15, 2023.",
|
||||
);
|
||||
assert!(dates.iter().any(|d| d.contains("September")));
|
||||
assert!(dates.iter().any(|d| d.contains("January")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_european_dates() {
|
||||
let dates = extract_dates("Meeting on 1 September 2024 was productive.");
|
||||
assert!(dates.iter().any(|d| d.contains("September")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
pub mod auto_tag;
|
||||
pub mod ner;
|
||||
#[cfg(feature = "temporal_track")]
|
||||
pub mod temporal;
|
||||
#[cfg(feature = "temporal_enrich")]
|
||||
pub mod temporal_enrich;
|
||||
@@ -0,0 +1,649 @@
|
||||
// Safe expect: Static NER model lookup with guaranteed default.
|
||||
#![allow(clippy::expect_used)]
|
||||
//! Named Entity Recognition (NER) module using DistilBERT-NER ONNX.
|
||||
//!
|
||||
//! This module provides entity extraction capabilities using DistilBERT-NER,
|
||||
//! a fast and accurate NER model fine-tuned on CoNLL-03.
|
||||
//!
|
||||
//! # Model
|
||||
//!
|
||||
//! Uses dslim/distilbert-NER ONNX (~261 MB) with 92% F1 score.
|
||||
//! Entities: Person (PER), Organization (ORG), Location (LOC), Miscellaneous (MISC)
|
||||
//!
|
||||
//! # Simple Interface
|
||||
//!
|
||||
//! Unlike `GLiNER`, DistilBERT-NER uses standard BERT tokenization:
|
||||
//! - Input: `input_ids`, `attention_mask`
|
||||
//! - Output: per-token logits for B-PER, I-PER, B-ORG, I-ORG, B-LOC, I-LOC, B-MISC, I-MISC, O
|
||||
|
||||
use crate::types::{EntityKind, FrameId};
|
||||
use crate::{MemvidError, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
// ============================================================================
|
||||
// Configuration Constants
|
||||
// ============================================================================
|
||||
|
||||
/// Model name for downloads and caching
|
||||
pub const NER_MODEL_NAME: &str = "distilbert-ner";
|
||||
|
||||
/// Model download URL (`HuggingFace`)
|
||||
pub const NER_MODEL_URL: &str =
|
||||
"https://huggingface.co/dslim/distilbert-NER/resolve/main/onnx/model.onnx";
|
||||
|
||||
/// Tokenizer URL
|
||||
pub const NER_TOKENIZER_URL: &str =
|
||||
"https://huggingface.co/dslim/distilbert-NER/resolve/main/tokenizer.json";
|
||||
|
||||
/// Approximate model size in MB
|
||||
pub const NER_MODEL_SIZE_MB: f32 = 261.0;
|
||||
|
||||
/// Maximum sequence length for the model
|
||||
pub const NER_MAX_SEQ_LEN: usize = 512;
|
||||
|
||||
/// Minimum confidence threshold for entity extraction
|
||||
#[cfg_attr(not(feature = "logic_mesh"), allow(dead_code))]
|
||||
pub const NER_MIN_CONFIDENCE: f32 = 0.5;
|
||||
|
||||
/// NER label mapping (CoNLL-03 format)
|
||||
/// O=0, B-PER=1, I-PER=2, B-ORG=3, I-ORG=4, B-LOC=5, I-LOC=6, B-MISC=7, I-MISC=8
|
||||
#[cfg_attr(not(feature = "logic_mesh"), allow(dead_code))]
|
||||
pub const NER_LABELS: &[&str] = &[
|
||||
"O", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC", "B-MISC", "I-MISC",
|
||||
];
|
||||
|
||||
// ============================================================================
|
||||
// Model Info
|
||||
// ============================================================================
|
||||
|
||||
/// NER model info for the models registry
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NerModelInfo {
|
||||
/// Model identifier
|
||||
pub name: &'static str,
|
||||
/// URL for ONNX model
|
||||
pub model_url: &'static str,
|
||||
/// URL for tokenizer JSON
|
||||
pub tokenizer_url: &'static str,
|
||||
/// Model size in MB
|
||||
pub size_mb: f32,
|
||||
/// Maximum sequence length
|
||||
pub max_seq_len: usize,
|
||||
/// Whether this is the default model
|
||||
pub is_default: bool,
|
||||
}
|
||||
|
||||
/// Available NER models registry
|
||||
pub static NER_MODELS: &[NerModelInfo] = &[NerModelInfo {
|
||||
name: NER_MODEL_NAME,
|
||||
model_url: NER_MODEL_URL,
|
||||
tokenizer_url: NER_TOKENIZER_URL,
|
||||
size_mb: NER_MODEL_SIZE_MB,
|
||||
max_seq_len: NER_MAX_SEQ_LEN,
|
||||
is_default: true,
|
||||
}];
|
||||
|
||||
/// Get NER model info by name
|
||||
#[must_use]
|
||||
pub fn get_ner_model_info(name: &str) -> Option<&'static NerModelInfo> {
|
||||
NER_MODELS.iter().find(|m| m.name == name)
|
||||
}
|
||||
|
||||
/// Get default NER model info
|
||||
#[must_use]
|
||||
pub fn default_ner_model_info() -> &'static NerModelInfo {
|
||||
NER_MODELS
|
||||
.iter()
|
||||
.find(|m| m.is_default)
|
||||
.expect("default NER model must exist")
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Entity Extraction Types
|
||||
// ============================================================================
|
||||
|
||||
/// Raw entity mention extracted from text
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExtractedEntity {
|
||||
/// The extracted text span
|
||||
pub text: String,
|
||||
/// Entity type (PER, ORG, LOC, MISC)
|
||||
pub entity_type: String,
|
||||
/// Confidence score (0.0-1.0)
|
||||
pub confidence: f32,
|
||||
/// Byte offset start in original text
|
||||
pub byte_start: usize,
|
||||
/// Byte offset end in original text
|
||||
pub byte_end: usize,
|
||||
}
|
||||
|
||||
impl ExtractedEntity {
|
||||
/// Convert the raw entity type to our `EntityKind` enum
|
||||
#[must_use]
|
||||
pub fn to_entity_kind(&self) -> EntityKind {
|
||||
match self.entity_type.to_uppercase().as_str() {
|
||||
"PER" | "PERSON" | "B-PER" | "I-PER" => EntityKind::Person,
|
||||
"ORG" | "ORGANIZATION" | "B-ORG" | "I-ORG" => EntityKind::Organization,
|
||||
"LOC" | "LOCATION" | "B-LOC" | "I-LOC" => EntityKind::Location,
|
||||
"MISC" | "B-MISC" | "I-MISC" => EntityKind::Other,
|
||||
_ => EntityKind::Other,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of extracting entities from a frame
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FrameEntities {
|
||||
/// Frame ID the entities were extracted from
|
||||
pub frame_id: FrameId,
|
||||
/// Extracted entities
|
||||
pub entities: Vec<ExtractedEntity>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// NER Model (Feature-gated)
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(feature = "logic_mesh")]
|
||||
pub use model_impl::*;
|
||||
|
||||
#[cfg(feature = "logic_mesh")]
|
||||
mod model_impl {
|
||||
use super::*;
|
||||
use ort::session::{Session, builder::GraphOptimizationLevel};
|
||||
use ort::value::Tensor;
|
||||
use std::sync::Mutex;
|
||||
use tokenizers::{
|
||||
PaddingDirection, PaddingParams, PaddingStrategy, Tokenizer, TruncationDirection,
|
||||
TruncationParams, TruncationStrategy,
|
||||
};
|
||||
|
||||
/// DistilBERT-NER model for entity extraction
|
||||
pub struct NerModel {
|
||||
/// ONNX runtime session
|
||||
session: Session,
|
||||
/// Tokenizer for text preprocessing
|
||||
tokenizer: Mutex<Tokenizer>,
|
||||
/// Model path for reference
|
||||
model_path: PathBuf,
|
||||
/// Minimum confidence threshold
|
||||
min_confidence: f32,
|
||||
}
|
||||
|
||||
impl NerModel {
|
||||
/// Load NER model from path
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `model_path` - Path to the ONNX model file
|
||||
/// * `tokenizer_path` - Path to the tokenizer.json file
|
||||
/// * `min_confidence` - Minimum confidence threshold (default: 0.5)
|
||||
pub fn load(
|
||||
model_path: impl AsRef<Path>,
|
||||
tokenizer_path: impl AsRef<Path>,
|
||||
min_confidence: Option<f32>,
|
||||
) -> Result<Self> {
|
||||
let model_path = model_path.as_ref().to_path_buf();
|
||||
let tokenizer_path = tokenizer_path.as_ref();
|
||||
|
||||
// Load tokenizer
|
||||
let mut tokenizer = Tokenizer::from_file(tokenizer_path).map_err(|e| {
|
||||
MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to load tokenizer from {:?}: {}", tokenizer_path, e)
|
||||
.into(),
|
||||
}
|
||||
})?;
|
||||
|
||||
// Configure padding and truncation
|
||||
tokenizer.with_padding(Some(PaddingParams {
|
||||
strategy: PaddingStrategy::BatchLongest,
|
||||
direction: PaddingDirection::Right,
|
||||
pad_to_multiple_of: None,
|
||||
pad_id: 0,
|
||||
pad_type_id: 0,
|
||||
pad_token: "[PAD]".to_string(),
|
||||
}));
|
||||
|
||||
tokenizer
|
||||
.with_truncation(Some(TruncationParams {
|
||||
max_length: NER_MAX_SEQ_LEN,
|
||||
strategy: TruncationStrategy::LongestFirst,
|
||||
stride: 0,
|
||||
direction: TruncationDirection::Right,
|
||||
}))
|
||||
.map_err(|e| MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to set truncation: {}", e).into(),
|
||||
})?;
|
||||
|
||||
// Initialize ONNX Runtime
|
||||
let session = Session::builder()
|
||||
.map_err(|e| MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to create session builder: {}", e).into(),
|
||||
})?
|
||||
.with_optimization_level(GraphOptimizationLevel::Level3)
|
||||
.map_err(|e| MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to set optimization level: {}", e).into(),
|
||||
})?
|
||||
.with_intra_threads(4)
|
||||
.map_err(|e| MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to set threads: {}", e).into(),
|
||||
})?
|
||||
.commit_from_file(&model_path)
|
||||
.map_err(|e| MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to load model from {:?}: {}", model_path, e).into(),
|
||||
})?;
|
||||
|
||||
tracing::info!(
|
||||
model = %model_path.display(),
|
||||
"DistilBERT-NER model loaded"
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
session,
|
||||
tokenizer: Mutex::new(tokenizer),
|
||||
model_path,
|
||||
min_confidence: min_confidence.unwrap_or(NER_MIN_CONFIDENCE),
|
||||
})
|
||||
}
|
||||
|
||||
/// Extract entities from text
|
||||
pub fn extract(&mut self, text: &str) -> Result<Vec<ExtractedEntity>> {
|
||||
if text.trim().is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// Tokenize
|
||||
let tokenizer = self
|
||||
.tokenizer
|
||||
.lock()
|
||||
.map_err(|_| MemvidError::Lock("failed to lock tokenizer".into()))?;
|
||||
|
||||
let encoding =
|
||||
tokenizer
|
||||
.encode(text, true)
|
||||
.map_err(|e| MemvidError::NerModelNotAvailable {
|
||||
reason: format!("tokenization failed: {}", e).into(),
|
||||
})?;
|
||||
|
||||
let input_ids: Vec<i64> = encoding.get_ids().iter().map(|&x| x as i64).collect();
|
||||
let attention_mask: Vec<i64> = encoding
|
||||
.get_attention_mask()
|
||||
.iter()
|
||||
.map(|&x| x as i64)
|
||||
.collect();
|
||||
let tokens = encoding.get_tokens().to_vec();
|
||||
let offsets = encoding.get_offsets().to_vec();
|
||||
|
||||
drop(tokenizer); // Release lock before inference
|
||||
|
||||
let seq_len = input_ids.len();
|
||||
|
||||
// Create input tensors using Tensor::from_array
|
||||
let input_ids_array = ndarray::Array2::from_shape_vec((1, seq_len), input_ids)
|
||||
.map_err(|e| MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to create input_ids array: {}", e).into(),
|
||||
})?;
|
||||
|
||||
let attention_mask_array =
|
||||
ndarray::Array2::from_shape_vec((1, seq_len), attention_mask).map_err(|e| {
|
||||
MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to create attention_mask array: {}", e).into(),
|
||||
}
|
||||
})?;
|
||||
|
||||
let input_ids_tensor = Tensor::from_array(input_ids_array).map_err(|e| {
|
||||
MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to create input_ids tensor: {}", e).into(),
|
||||
}
|
||||
})?;
|
||||
|
||||
let attention_mask_tensor = Tensor::from_array(attention_mask_array).map_err(|e| {
|
||||
MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to create attention_mask tensor: {}", e).into(),
|
||||
}
|
||||
})?;
|
||||
|
||||
// Get output name before inference (avoid borrow conflict)
|
||||
let output_name = self
|
||||
.session
|
||||
.outputs
|
||||
.first()
|
||||
.map(|o| o.name.clone())
|
||||
.unwrap_or_else(|| "logits".into());
|
||||
|
||||
// Run inference
|
||||
let outputs = self
|
||||
.session
|
||||
.run(ort::inputs![
|
||||
"input_ids" => input_ids_tensor,
|
||||
"attention_mask" => attention_mask_tensor,
|
||||
])
|
||||
.map_err(|e| MemvidError::NerModelNotAvailable {
|
||||
reason: format!("inference failed: {}", e).into(),
|
||||
})?;
|
||||
|
||||
let logits =
|
||||
outputs
|
||||
.get(&output_name)
|
||||
.ok_or_else(|| MemvidError::NerModelNotAvailable {
|
||||
reason: format!("no output '{}' found", output_name).into(),
|
||||
})?;
|
||||
|
||||
// Parse logits to get predictions
|
||||
let entities = Self::decode_predictions_static(
|
||||
text,
|
||||
&tokens,
|
||||
&offsets,
|
||||
logits,
|
||||
self.min_confidence,
|
||||
)?;
|
||||
|
||||
Ok(entities)
|
||||
}
|
||||
|
||||
/// Decode model predictions into entities (static to avoid borrow issues)
|
||||
fn decode_predictions_static(
|
||||
original_text: &str,
|
||||
tokens: &[String],
|
||||
offsets: &[(usize, usize)],
|
||||
logits: &ort::value::Value,
|
||||
min_confidence: f32,
|
||||
) -> Result<Vec<ExtractedEntity>> {
|
||||
// Extract the logits tensor - shape: [1, seq_len, num_labels]
|
||||
let (shape, data) = logits.try_extract_tensor::<f32>().map_err(|e| {
|
||||
MemvidError::NerModelNotAvailable {
|
||||
reason: format!("failed to extract logits: {}", e).into(),
|
||||
}
|
||||
})?;
|
||||
|
||||
// Shape is iterable, convert to Vec
|
||||
let shape_vec: Vec<i64> = shape.iter().copied().collect();
|
||||
|
||||
if shape_vec.len() != 3 {
|
||||
return Err(MemvidError::NerModelNotAvailable {
|
||||
reason: format!("unexpected logits shape: {:?}", shape_vec).into(),
|
||||
});
|
||||
}
|
||||
|
||||
let seq_len = shape_vec[1] as usize;
|
||||
let num_labels = shape_vec[2] as usize;
|
||||
|
||||
// Helper to index into flat data: [batch, seq, labels] -> flat index
|
||||
let idx = |i: usize, j: usize| -> usize { i * num_labels + j };
|
||||
|
||||
let mut entities = Vec::new();
|
||||
let mut current_entity: Option<(String, usize, usize, f32)> = None;
|
||||
|
||||
for i in 0..seq_len {
|
||||
if i >= tokens.len() || i >= offsets.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip special tokens
|
||||
let token = &tokens[i];
|
||||
if token == "[CLS]" || token == "[SEP]" || token == "[PAD]" {
|
||||
// Finalize any current entity
|
||||
if let Some((entity_type, start, end, conf)) = current_entity.take() {
|
||||
if end > start && end <= original_text.len() {
|
||||
let text = original_text[start..end].trim().to_string();
|
||||
if !text.is_empty() {
|
||||
entities.push(ExtractedEntity {
|
||||
text,
|
||||
entity_type,
|
||||
confidence: conf,
|
||||
byte_start: start,
|
||||
byte_end: end,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get prediction for this token
|
||||
let mut max_score = f32::NEG_INFINITY;
|
||||
let mut max_label = 0usize;
|
||||
|
||||
for j in 0..num_labels {
|
||||
let score = data[idx(i, j)];
|
||||
if score > max_score {
|
||||
max_score = score;
|
||||
max_label = j;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply softmax to get confidence
|
||||
let mut exp_sum = 0.0f32;
|
||||
for j in 0..num_labels {
|
||||
exp_sum += (data[idx(i, j)] - max_score).exp();
|
||||
}
|
||||
let confidence = 1.0 / exp_sum;
|
||||
|
||||
let label = NER_LABELS.get(max_label).unwrap_or(&"O");
|
||||
let (start_offset, end_offset) = offsets[i];
|
||||
|
||||
if *label == "O" || confidence < min_confidence {
|
||||
// End any current entity
|
||||
if let Some((entity_type, start, end, conf)) = current_entity.take() {
|
||||
if end > start && end <= original_text.len() {
|
||||
let text = original_text[start..end].trim().to_string();
|
||||
if !text.is_empty() {
|
||||
entities.push(ExtractedEntity {
|
||||
text,
|
||||
entity_type,
|
||||
confidence: conf,
|
||||
byte_start: start,
|
||||
byte_end: end,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if label.starts_with("B-") {
|
||||
// Start new entity (end previous if any)
|
||||
if let Some((entity_type, start, end, conf)) = current_entity.take() {
|
||||
if end > start && end <= original_text.len() {
|
||||
let text = original_text[start..end].trim().to_string();
|
||||
if !text.is_empty() {
|
||||
entities.push(ExtractedEntity {
|
||||
text,
|
||||
entity_type,
|
||||
confidence: conf,
|
||||
byte_start: start,
|
||||
byte_end: end,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
let entity_type = label[2..].to_string(); // Remove "B-" prefix
|
||||
current_entity = Some((entity_type, start_offset, end_offset, confidence));
|
||||
} else if label.starts_with("I-") {
|
||||
// Continue entity
|
||||
if let Some((ref entity_type, start, _, ref mut conf)) = current_entity {
|
||||
let expected_type = &label[2..];
|
||||
if entity_type == expected_type {
|
||||
current_entity = Some((
|
||||
entity_type.clone(),
|
||||
start,
|
||||
end_offset,
|
||||
(*conf + confidence) / 2.0,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Finalize last entity
|
||||
if let Some((entity_type, start, end, conf)) = current_entity {
|
||||
if end > start && end <= original_text.len() {
|
||||
let text = original_text[start..end].trim().to_string();
|
||||
if !text.is_empty() {
|
||||
entities.push(ExtractedEntity {
|
||||
text,
|
||||
entity_type,
|
||||
confidence: conf,
|
||||
byte_start: start,
|
||||
byte_end: end,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(entities)
|
||||
}
|
||||
|
||||
/// Extract entities from a frame's content
|
||||
pub fn extract_from_frame(
|
||||
&mut self,
|
||||
frame_id: FrameId,
|
||||
content: &str,
|
||||
) -> Result<FrameEntities> {
|
||||
let entities = self.extract(content)?;
|
||||
Ok(FrameEntities { frame_id, entities })
|
||||
}
|
||||
|
||||
/// Get minimum confidence threshold
|
||||
pub fn min_confidence(&self) -> f32 {
|
||||
self.min_confidence
|
||||
}
|
||||
|
||||
/// Get model path
|
||||
pub fn model_path(&self) -> &Path {
|
||||
&self.model_path
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for NerModel {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("NerModel")
|
||||
.field("model_path", &self.model_path)
|
||||
.field("min_confidence", &self.min_confidence)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Stub Implementation (when feature is disabled)
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(not(feature = "logic_mesh"))]
|
||||
#[allow(dead_code)]
|
||||
pub struct NerModel {
|
||||
_private: (),
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "logic_mesh"))]
|
||||
#[allow(dead_code)]
|
||||
impl NerModel {
|
||||
pub fn load(
|
||||
_model_path: impl AsRef<Path>,
|
||||
_tokenizer_path: impl AsRef<Path>,
|
||||
_min_confidence: Option<f32>,
|
||||
) -> Result<Self> {
|
||||
Err(MemvidError::FeatureUnavailable {
|
||||
feature: "logic_mesh",
|
||||
})
|
||||
}
|
||||
|
||||
pub fn extract(&self, _text: &str) -> Result<Vec<ExtractedEntity>> {
|
||||
Err(MemvidError::FeatureUnavailable {
|
||||
feature: "logic_mesh",
|
||||
})
|
||||
}
|
||||
|
||||
pub fn extract_from_frame(&self, _frame_id: FrameId, _content: &str) -> Result<FrameEntities> {
|
||||
Err(MemvidError::FeatureUnavailable {
|
||||
feature: "logic_mesh",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Model Path Utilities
|
||||
// ============================================================================
|
||||
|
||||
/// Get the expected path for the NER model in the models directory
|
||||
#[must_use]
|
||||
pub fn ner_model_path(models_dir: &Path) -> PathBuf {
|
||||
models_dir.join(NER_MODEL_NAME).join("model.onnx")
|
||||
}
|
||||
|
||||
/// Get the expected path for the NER tokenizer in the models directory
|
||||
#[must_use]
|
||||
pub fn ner_tokenizer_path(models_dir: &Path) -> PathBuf {
|
||||
models_dir.join(NER_MODEL_NAME).join("tokenizer.json")
|
||||
}
|
||||
|
||||
/// Check if NER model is installed
|
||||
#[must_use]
|
||||
pub fn is_ner_model_installed(models_dir: &Path) -> bool {
|
||||
ner_model_path(models_dir).exists() && ner_tokenizer_path(models_dir).exists()
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_entity_kind_mapping() {
|
||||
let cases = vec![
|
||||
("PER", EntityKind::Person),
|
||||
("B-PER", EntityKind::Person),
|
||||
("I-PER", EntityKind::Person),
|
||||
("ORG", EntityKind::Organization),
|
||||
("B-ORG", EntityKind::Organization),
|
||||
("LOC", EntityKind::Location),
|
||||
("B-LOC", EntityKind::Location),
|
||||
("MISC", EntityKind::Other),
|
||||
("B-MISC", EntityKind::Other),
|
||||
("unknown", EntityKind::Other),
|
||||
];
|
||||
|
||||
for (entity_type, expected_kind) in cases {
|
||||
let entity = ExtractedEntity {
|
||||
text: "test".to_string(),
|
||||
entity_type: entity_type.to_string(),
|
||||
confidence: 0.9,
|
||||
byte_start: 0,
|
||||
byte_end: 4,
|
||||
};
|
||||
assert_eq!(
|
||||
entity.to_entity_kind(),
|
||||
expected_kind,
|
||||
"Failed for entity_type: {}",
|
||||
entity_type
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_model_info() {
|
||||
let info = default_ner_model_info();
|
||||
assert_eq!(info.name, NER_MODEL_NAME);
|
||||
assert!(info.is_default);
|
||||
assert!(info.size_mb > 200.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_model_paths() {
|
||||
let models_dir = PathBuf::from("/tmp/models");
|
||||
let model_path = ner_model_path(&models_dir);
|
||||
let tokenizer_path = ner_tokenizer_path(&models_dir);
|
||||
|
||||
assert!(model_path.to_string_lossy().contains("model.onnx"));
|
||||
assert!(tokenizer_path.to_string_lossy().contains("tokenizer.json"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ner_labels() {
|
||||
assert_eq!(NER_LABELS.len(), 9);
|
||||
assert_eq!(NER_LABELS[0], "O");
|
||||
assert_eq!(NER_LABELS[1], "B-PER");
|
||||
assert_eq!(NER_LABELS[3], "B-ORG");
|
||||
assert_eq!(NER_LABELS[5], "B-LOC");
|
||||
assert_eq!(NER_LABELS[7], "B-MISC");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,800 @@
|
||||
#![cfg(feature = "temporal_track")]
|
||||
|
||||
use once_cell::sync::OnceCell;
|
||||
use regex::Regex;
|
||||
use time::{Date, Duration, Month, OffsetDateTime, PrimitiveDateTime, Time, Weekday};
|
||||
|
||||
use crate::error::{MemvidError, Result};
|
||||
|
||||
const DEFAULT_CONFIDENCE: u16 = 950;
|
||||
const AMBIGUOUS_CONFIDENCE: u16 = 700;
|
||||
const RELATIVE_CONFIDENCE: u16 = 900;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TemporalResolutionFlag {
|
||||
Ambiguous,
|
||||
Relative,
|
||||
}
|
||||
|
||||
impl TemporalResolutionFlag {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Ambiguous => "ambiguous",
|
||||
Self::Relative => "relative",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TemporalResolutionValue {
|
||||
Date(Date),
|
||||
DateRange {
|
||||
start: Date,
|
||||
end: Date,
|
||||
},
|
||||
DateTime(OffsetDateTime),
|
||||
DateTimeRange {
|
||||
start: OffsetDateTime,
|
||||
end: OffsetDateTime,
|
||||
},
|
||||
Month {
|
||||
year: i32,
|
||||
month: Month,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TemporalResolution {
|
||||
pub value: TemporalResolutionValue,
|
||||
pub flags: Vec<TemporalResolutionFlag>,
|
||||
pub confidence: u16,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TemporalContext {
|
||||
pub anchor: OffsetDateTime,
|
||||
pub timezone: String,
|
||||
pub week_start: Weekday,
|
||||
pub clock_inheritance: bool,
|
||||
}
|
||||
|
||||
impl TemporalContext {
|
||||
pub fn new(anchor: OffsetDateTime, timezone: impl Into<String>) -> Self {
|
||||
Self {
|
||||
anchor,
|
||||
timezone: timezone.into(),
|
||||
week_start: Weekday::Monday,
|
||||
clock_inheritance: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_week_start(mut self, week_start: Weekday) -> Self {
|
||||
self.week_start = week_start;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_clock_inheritance(mut self, inherit: bool) -> Self {
|
||||
self.clock_inheritance = inherit;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TemporalNormalizer {
|
||||
context: TemporalContext,
|
||||
}
|
||||
|
||||
impl TemporalNormalizer {
|
||||
pub fn new(context: TemporalContext) -> Self {
|
||||
Self { context }
|
||||
}
|
||||
|
||||
pub fn resolve(&self, phrase: &str) -> Result<TemporalResolution> {
|
||||
let trimmed = phrase.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(MemvidError::InvalidQuery {
|
||||
reason: "temporal phrase is empty".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let lower = trimmed.to_ascii_lowercase();
|
||||
|
||||
if let Some(resolution) = self.resolve_fixed(&lower) {
|
||||
return Ok(resolution);
|
||||
}
|
||||
if let Some(resolution) = self.resolve_relative_days(&lower) {
|
||||
return Ok(resolution);
|
||||
}
|
||||
if let Some(resolution) = self.resolve_relative_weeks(&lower) {
|
||||
return Ok(resolution);
|
||||
}
|
||||
if let Some(resolution) = self.resolve_weekday_phrases(&lower) {
|
||||
return Ok(resolution);
|
||||
}
|
||||
if let Some(resolution) = self.resolve_clock_phrases(&lower) {
|
||||
return Ok(resolution);
|
||||
}
|
||||
if let Some(resolution) = self.resolve_numeric_date(&lower) {
|
||||
return Ok(resolution);
|
||||
}
|
||||
if let Some(resolution) = self.resolve_quarter(&lower) {
|
||||
return Ok(resolution);
|
||||
}
|
||||
if let Some(resolution) = self.resolve_year(&lower) {
|
||||
return Ok(resolution);
|
||||
}
|
||||
|
||||
Err(MemvidError::InvalidQuery {
|
||||
reason: format!("unsupported temporal phrase: {trimmed}"),
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve_fixed(&self, phrase: &str) -> Option<TemporalResolution> {
|
||||
match phrase {
|
||||
"today" => Some(self.date_resolution(self.anchor_date())),
|
||||
"yesterday" => Some(self.date_resolution(add_days(self.anchor_date(), -1))),
|
||||
"tomorrow" => Some(self.date_resolution(add_days(self.anchor_date(), 1))),
|
||||
"two days ago" => self.relative_days(-2, RELATIVE_CONFIDENCE),
|
||||
"in 3 days" => self.relative_days(3, RELATIVE_CONFIDENCE),
|
||||
"two weeks from now" => self.relative_weeks_offset(2),
|
||||
"2 weeks from now" => self.relative_weeks_offset(2),
|
||||
"two fridays ago" => self.weeks_from_weekday(-2, Weekday::Friday),
|
||||
"last friday" => self.weeks_from_weekday(-1, Weekday::Friday),
|
||||
"next friday" => self.weeks_from_weekday(1, Weekday::Friday),
|
||||
"this friday" => Some(self.this_weekday(Weekday::Friday)),
|
||||
"next week" => Some(self.week_range(1)),
|
||||
"last week" => Some(self.week_range(-1)),
|
||||
"end of this month" => Some(self.end_of_month()),
|
||||
"start of next month" => Some(self.start_of_next_month()),
|
||||
"last month" => Some(self.month_relative(-1)),
|
||||
"3 months ago" => Some(self.date_with_month_offset(-3)),
|
||||
"in 90 minutes" => {
|
||||
Some(self.datetime_resolution(self.context.anchor + Duration::minutes(90)))
|
||||
}
|
||||
"at 5pm today" => Some(self.time_today(17, 0)),
|
||||
"in the last 24 hours" => Some(self.last_hours_range(24)),
|
||||
"this morning" => Some(self.morning_range()),
|
||||
"on the sunday after next" => Some(self.sunday_after_next()),
|
||||
"next daylight saving change" => Some(self.next_dst_change()),
|
||||
"midnight tomorrow" => Some(self.midnight_tomorrow()),
|
||||
"noon next tuesday" => {
|
||||
Some(self.weekday_in_following_week_at_time(Weekday::Tuesday, 12, 0))
|
||||
}
|
||||
"q4 2025" => Some(self.quarter_range(2025, 4)),
|
||||
"end of q3" => Some(self.end_of_quarter(3)),
|
||||
"first business day of next month" => Some(self.first_business_day_next_month()),
|
||||
"the first business day of next month" => Some(self.first_business_day_next_month()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_relative_days(&self, phrase: &str) -> Option<TemporalResolution> {
|
||||
static IN_DAYS: OnceCell<Regex> = OnceCell::new();
|
||||
static AGO_DAYS: OnceCell<Regex> = OnceCell::new();
|
||||
|
||||
if let Some(captures) = IN_DAYS
|
||||
.get_or_init(|| Regex::new(r"^in (?P<count>[[:word:]]+) days$").unwrap())
|
||||
.captures(phrase)
|
||||
{
|
||||
let count = parse_number(captures.name("count")?.as_str())?;
|
||||
return self.relative_days(count as i64, RELATIVE_CONFIDENCE);
|
||||
}
|
||||
|
||||
if let Some(captures) = AGO_DAYS
|
||||
.get_or_init(|| Regex::new(r"^(?P<count>[[:word:]]+) days ago$").unwrap())
|
||||
.captures(phrase)
|
||||
{
|
||||
let count = parse_number(captures.name("count")?.as_str())?;
|
||||
return self.relative_days(-(count as i64), RELATIVE_CONFIDENCE);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn resolve_relative_weeks(&self, phrase: &str) -> Option<TemporalResolution> {
|
||||
static WEEKS_FROM_NOW: OnceCell<Regex> = OnceCell::new();
|
||||
|
||||
if let Some(caps) = WEEKS_FROM_NOW
|
||||
.get_or_init(|| Regex::new(r"^(?P<count>[[:word:]]+) weeks from now$").unwrap())
|
||||
.captures(phrase)
|
||||
{
|
||||
let count = parse_number(caps.name("count")?.as_str())?;
|
||||
return self.relative_weeks_offset(count as i32);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn resolve_weekday_phrases(&self, phrase: &str) -> Option<TemporalResolution> {
|
||||
static NEXT_WEEKDAY: OnceCell<Regex> = OnceCell::new();
|
||||
static LAST_WEEKDAY: OnceCell<Regex> = OnceCell::new();
|
||||
static WEEKDAY_AT_TIME: OnceCell<Regex> = OnceCell::new();
|
||||
static BARE_WEEKDAY: OnceCell<Regex> = OnceCell::new();
|
||||
|
||||
if let Some(caps) = NEXT_WEEKDAY
|
||||
.get_or_init(|| Regex::new(r"^next (?P<weekday>[a-z]+)$").unwrap())
|
||||
.captures(phrase)
|
||||
{
|
||||
let weekday = parse_weekday(caps.name("weekday")?.as_str())?;
|
||||
return self.weeks_from_weekday(1, weekday);
|
||||
}
|
||||
|
||||
if let Some(caps) = LAST_WEEKDAY
|
||||
.get_or_init(|| Regex::new(r"^last (?P<weekday>[a-z]+)$").unwrap())
|
||||
.captures(phrase)
|
||||
{
|
||||
let weekday = parse_weekday(caps.name("weekday")?.as_str())?;
|
||||
return self.weeks_from_weekday(-1, weekday);
|
||||
}
|
||||
|
||||
if let Some(caps) = WEEKDAY_AT_TIME
|
||||
.get_or_init(|| {
|
||||
Regex::new(
|
||||
r"^(?:(?P<prefix>next) )?(?P<weekday>[a-z]+) at (?P<hour>\d{1,2})(?::(?P<minute>\d{2}))?(?P<ampm>am|pm)?$",
|
||||
)
|
||||
.unwrap()
|
||||
})
|
||||
.captures(phrase)
|
||||
{
|
||||
let weekday = parse_weekday(caps.name("weekday")?.as_str())?;
|
||||
let hour_raw: i32 = caps.name("hour")?.as_str().parse().ok()?;
|
||||
let minute_raw: i32 = caps
|
||||
.name("minute")
|
||||
.map(|m| m.as_str().parse().unwrap_or(0))
|
||||
.unwrap_or(0);
|
||||
let hour = convert_hour(hour_raw, caps.name("ampm").map(|m| m.as_str()))?;
|
||||
let minute = minute_raw as i32;
|
||||
if caps.name("prefix").is_some() {
|
||||
return Some(self.next_weekday_at_time(weekday, hour, minute));
|
||||
}
|
||||
return Some(self.weekday_at_time(weekday, hour, minute));
|
||||
}
|
||||
|
||||
if let Some(caps) = BARE_WEEKDAY
|
||||
.get_or_init(|| Regex::new(r"^(?P<weekday>[a-z]+)$").unwrap())
|
||||
.captures(phrase)
|
||||
{
|
||||
let weekday = parse_weekday(caps.name("weekday")?.as_str())?;
|
||||
return Some(self.this_weekday(weekday));
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn resolve_clock_phrases(&self, phrase: &str) -> Option<TemporalResolution> {
|
||||
static TODAY_AT: OnceCell<Regex> = OnceCell::new();
|
||||
static TODAY_PREFIX: OnceCell<Regex> = OnceCell::new();
|
||||
|
||||
let sanitized = sanitize_ampm(phrase);
|
||||
let target = sanitized.trim();
|
||||
if let Some(caps) = TODAY_AT
|
||||
.get_or_init(|| {
|
||||
Regex::new(
|
||||
r"^at (?P<hour>\d{1,2})(?::(?P<minute>\d{2}))?(?:\s*(?P<ampm>am|pm)) today$",
|
||||
)
|
||||
.unwrap()
|
||||
})
|
||||
.captures(target)
|
||||
{
|
||||
let hour_raw: i32 = caps.name("hour")?.as_str().parse().ok()?;
|
||||
let minute_raw: i32 = caps
|
||||
.name("minute")
|
||||
.map(|m| m.as_str().parse().unwrap_or(0))
|
||||
.unwrap_or(0);
|
||||
let hour = convert_hour(hour_raw, caps.name("ampm").map(|m| m.as_str()))?;
|
||||
let minute = minute_raw as i32;
|
||||
return Some(self.time_today(hour, minute));
|
||||
}
|
||||
|
||||
if let Some(caps) = TODAY_PREFIX
|
||||
.get_or_init(|| {
|
||||
Regex::new(
|
||||
r"^today at (?P<hour>\d{1,2})(?::(?P<minute>\d{2}))?(?:\s*(?P<ampm>am|pm))?$",
|
||||
)
|
||||
.unwrap()
|
||||
})
|
||||
.captures(target)
|
||||
{
|
||||
let hour_raw: i32 = caps.name("hour")?.as_str().parse().ok()?;
|
||||
let minute_raw: i32 = caps
|
||||
.name("minute")
|
||||
.map(|m| m.as_str().parse().unwrap_or(0))
|
||||
.unwrap_or(0);
|
||||
let marker = caps.name("ampm").map(|m| m.as_str());
|
||||
let hour = convert_hour(hour_raw, marker)?;
|
||||
let minute = minute_raw as i32;
|
||||
return Some(self.time_today(hour, minute));
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn resolve_numeric_date(&self, phrase: &str) -> Option<TemporalResolution> {
|
||||
static NUMERIC: OnceCell<Regex> = OnceCell::new();
|
||||
|
||||
let caps = NUMERIC
|
||||
.get_or_init(|| {
|
||||
Regex::new(r"^(?P<month>\d{1,2})/(?P<day>\d{1,2})/(?P<year>\d{2,4})$").unwrap()
|
||||
})
|
||||
.captures(phrase)?;
|
||||
|
||||
let month: u8 = caps.name("month")?.as_str().parse().ok()?;
|
||||
let day: u8 = caps.name("day")?.as_str().parse().ok()?;
|
||||
let year = parse_year(caps.name("year")?.as_str());
|
||||
let month_enum = Month::try_from(month).ok()?;
|
||||
let last_day = last_day_of_month(year, month_enum);
|
||||
if day == 0 || day > last_day {
|
||||
return None;
|
||||
}
|
||||
let date = Date::from_calendar_date(year, month_enum, day).ok()?;
|
||||
let mut resolution = self.date_resolution(date);
|
||||
resolution.confidence = AMBIGUOUS_CONFIDENCE;
|
||||
resolution.flags.push(TemporalResolutionFlag::Ambiguous);
|
||||
Some(resolution)
|
||||
}
|
||||
|
||||
fn resolve_quarter(&self, phrase: &str) -> Option<TemporalResolution> {
|
||||
static QUARTER_YEAR: OnceCell<Regex> = OnceCell::new();
|
||||
static QUARTER_WORD_YEAR: OnceCell<Regex> = OnceCell::new();
|
||||
static QUARTER_ORDINAL_YEAR: OnceCell<Regex> = OnceCell::new();
|
||||
|
||||
if let Some(caps) = QUARTER_YEAR
|
||||
.get_or_init(|| Regex::new(r"^q(?P<quarter>[1-4]) (?P<year>\d{4})$").unwrap())
|
||||
.captures(phrase)
|
||||
{
|
||||
let quarter: i32 = caps.name("quarter")?.as_str().parse().ok()?;
|
||||
let year: i32 = caps.name("year")?.as_str().parse().ok()?;
|
||||
return Some(self.quarter_range(year, quarter));
|
||||
}
|
||||
if let Some(caps) = QUARTER_WORD_YEAR
|
||||
.get_or_init(|| {
|
||||
Regex::new(r"^(?P<word>first|second|third|fourth) quarter(?: of)? (?P<year>\d{4})$")
|
||||
.unwrap()
|
||||
})
|
||||
.captures(phrase)
|
||||
{
|
||||
let quarter = match caps.name("word")?.as_str() {
|
||||
"first" => 1,
|
||||
"second" => 2,
|
||||
"third" => 3,
|
||||
"fourth" => 4,
|
||||
_ => return None,
|
||||
};
|
||||
let year = caps.name("year")?.as_str().parse().ok()?;
|
||||
return Some(self.quarter_range(year, quarter));
|
||||
}
|
||||
if let Some(caps) = QUARTER_ORDINAL_YEAR
|
||||
.get_or_init(|| {
|
||||
Regex::new(r"^(?P<num>[1-4])(st|nd|rd|th)? quarter(?: of)? (?P<year>\d{4})$")
|
||||
.unwrap()
|
||||
})
|
||||
.captures(phrase)
|
||||
{
|
||||
let quarter: i32 = caps.name("num")?.as_str().parse().ok()?;
|
||||
let year: i32 = caps.name("year")?.as_str().parse().ok()?;
|
||||
return Some(self.quarter_range(year, quarter));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn resolve_year(&self, phrase: &str) -> Option<TemporalResolution> {
|
||||
static YEAR_SINGLE: OnceCell<Regex> = OnceCell::new();
|
||||
|
||||
let caps = YEAR_SINGLE
|
||||
.get_or_init(|| Regex::new(r"^(?:year )?(?P<year>\d{4})$").unwrap())
|
||||
.captures(phrase)?;
|
||||
let year = parse_year(caps.name("year")?.as_str());
|
||||
Some(self.year_range(year))
|
||||
}
|
||||
|
||||
fn relative_days(&self, delta: i64, confidence: u16) -> Option<TemporalResolution> {
|
||||
let date = add_days(self.anchor_date(), delta);
|
||||
let mut res = self.date_resolution(date);
|
||||
res.confidence = confidence;
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
Some(res)
|
||||
}
|
||||
|
||||
fn relative_weeks_offset(&self, weeks: i32) -> Option<TemporalResolution> {
|
||||
let date = add_days(self.anchor_date(), (weeks as i64) * 7);
|
||||
let mut res = self.date_resolution(date);
|
||||
res.confidence = RELATIVE_CONFIDENCE;
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
Some(res)
|
||||
}
|
||||
|
||||
fn weeks_from_weekday(&self, weeks: i32, weekday: Weekday) -> Option<TemporalResolution> {
|
||||
if weeks == 0 {
|
||||
return Some(self.this_weekday(weekday));
|
||||
}
|
||||
if weeks > 0 {
|
||||
let mut date = self.next_weekday_after(self.anchor_date(), weekday);
|
||||
for _ in 1..weeks {
|
||||
date = add_days(self.next_weekday_after(date, weekday), 0);
|
||||
}
|
||||
return Some(self.date_resolution(date));
|
||||
}
|
||||
let mut date = self.previous_weekday_before(self.anchor_date(), weekday);
|
||||
for _ in 1..weeks.abs() {
|
||||
date = self.previous_weekday_before(date, weekday);
|
||||
}
|
||||
Some(self.date_resolution(date))
|
||||
}
|
||||
|
||||
fn this_weekday(&self, weekday: Weekday) -> TemporalResolution {
|
||||
let start = self.start_of_week(self.anchor_date());
|
||||
let mut offset = weekday.number_days_from_monday() as i64
|
||||
- self.context.week_start.number_days_from_monday() as i64;
|
||||
if offset < 0 {
|
||||
offset += 7;
|
||||
}
|
||||
let date = add_days(start, offset);
|
||||
self.date_resolution(date)
|
||||
}
|
||||
|
||||
fn week_range(&self, offset_weeks: i32) -> TemporalResolution {
|
||||
let start = add_days(
|
||||
self.start_of_week(self.anchor_date()),
|
||||
(offset_weeks * 7) as i64,
|
||||
);
|
||||
let end = add_days(start, 6);
|
||||
let mut res = self.date_range_resolution(start, end);
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
res.confidence = RELATIVE_CONFIDENCE;
|
||||
res
|
||||
}
|
||||
|
||||
fn end_of_month(&self) -> TemporalResolution {
|
||||
let date = self.anchor_date();
|
||||
let last_day = last_day_of_month(date.year(), date.month());
|
||||
let end = Date::from_calendar_date(date.year(), date.month(), last_day).unwrap();
|
||||
self.date_resolution(end)
|
||||
}
|
||||
|
||||
fn start_of_next_month(&self) -> TemporalResolution {
|
||||
let date = self.anchor_date();
|
||||
let (year, month) = add_months(date.year(), date.month(), 1);
|
||||
let start = Date::from_calendar_date(year, month, 1).unwrap();
|
||||
self.date_resolution(start)
|
||||
}
|
||||
|
||||
fn month_relative(&self, offset: i32) -> TemporalResolution {
|
||||
let date = self.anchor_date();
|
||||
let (year, month) = add_months(date.year(), date.month(), offset);
|
||||
TemporalResolution {
|
||||
value: TemporalResolutionValue::Month { year, month },
|
||||
flags: vec![TemporalResolutionFlag::Relative],
|
||||
confidence: RELATIVE_CONFIDENCE,
|
||||
}
|
||||
}
|
||||
|
||||
fn date_with_month_offset(&self, offset: i32) -> TemporalResolution {
|
||||
let date = self.anchor_date();
|
||||
let (year, month) = add_months(date.year(), date.month(), offset);
|
||||
let day = date.day().min(last_day_of_month(year, month));
|
||||
let new_date = Date::from_calendar_date(year, month, day).unwrap();
|
||||
let mut res = self.date_resolution(new_date);
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
res.confidence = RELATIVE_CONFIDENCE;
|
||||
res
|
||||
}
|
||||
|
||||
fn time_today(&self, hour: i32, minute: i32) -> TemporalResolution {
|
||||
let date = self.anchor_date();
|
||||
let dt = combine(date, hour, minute, 0, self.context.anchor.offset());
|
||||
self.datetime_resolution(dt)
|
||||
}
|
||||
|
||||
fn last_hours_range(&self, hours: i64) -> TemporalResolution {
|
||||
let end = self.context.anchor;
|
||||
let start = end - Duration::hours(hours);
|
||||
let mut res = self.datetime_range_resolution(start, end);
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
res.confidence = RELATIVE_CONFIDENCE;
|
||||
res
|
||||
}
|
||||
|
||||
fn morning_range(&self) -> TemporalResolution {
|
||||
let date = self.anchor_date();
|
||||
let start = combine(date, 6, 0, 0, self.context.anchor.offset());
|
||||
let end = combine(date, 11, 59, 59, self.context.anchor.offset());
|
||||
let mut res = self.datetime_range_resolution(start, end);
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
res
|
||||
}
|
||||
|
||||
fn sunday_after_next(&self) -> TemporalResolution {
|
||||
let next = self.next_weekday_after(self.anchor_date(), Weekday::Sunday);
|
||||
let after_next = add_days(next, 7);
|
||||
let mut res = self.date_resolution(after_next);
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
res
|
||||
}
|
||||
|
||||
fn next_dst_change(&self) -> TemporalResolution {
|
||||
let year = self.anchor_date().year();
|
||||
let november_first = Date::from_calendar_date(year, Month::November, 1).unwrap();
|
||||
let first_sunday = self.next_weekday_on_or_after(november_first, Weekday::Sunday);
|
||||
let date = add_days(first_sunday, 0);
|
||||
let dt = combine(date, 1, 0, 0, self.context.anchor.offset());
|
||||
let mut res = self.datetime_resolution(dt);
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
res
|
||||
}
|
||||
|
||||
fn midnight_tomorrow(&self) -> TemporalResolution {
|
||||
let date = add_days(self.anchor_date(), 1);
|
||||
let dt = combine(date, 0, 0, 0, self.context.anchor.offset());
|
||||
self.datetime_resolution(dt)
|
||||
}
|
||||
|
||||
fn next_weekday_at_time(&self, weekday: Weekday, hour: i32, minute: i32) -> TemporalResolution {
|
||||
let date = self.next_weekday_after(self.anchor_date(), weekday);
|
||||
self.datetime_resolution(combine(date, hour, minute, 0, self.context.anchor.offset()))
|
||||
}
|
||||
|
||||
fn weekday_at_time(&self, weekday: Weekday, hour: i32, minute: i32) -> TemporalResolution {
|
||||
let today = self.anchor_date();
|
||||
let target = self.next_weekday_on_or_after(today, weekday);
|
||||
self.datetime_resolution(combine(
|
||||
target,
|
||||
hour,
|
||||
minute,
|
||||
0,
|
||||
self.context.anchor.offset(),
|
||||
))
|
||||
}
|
||||
|
||||
fn weekday_in_following_week_at_time(
|
||||
&self,
|
||||
weekday: Weekday,
|
||||
hour: i32,
|
||||
minute: i32,
|
||||
) -> TemporalResolution {
|
||||
let first = self.next_weekday_after(self.anchor_date(), weekday);
|
||||
let date = add_days(first, 7);
|
||||
self.datetime_resolution(combine(date, hour, minute, 0, self.context.anchor.offset()))
|
||||
}
|
||||
|
||||
fn quarter_range(&self, year: i32, quarter: i32) -> TemporalResolution {
|
||||
let start_month = match quarter {
|
||||
1 => Month::January,
|
||||
2 => Month::April,
|
||||
3 => Month::July,
|
||||
4 => Month::October,
|
||||
_ => Month::January,
|
||||
};
|
||||
let start = Date::from_calendar_date(year, start_month, 1).unwrap();
|
||||
let (end_year, end_month) = add_months(year, start_month, 2);
|
||||
let end_day = last_day_of_month(end_year, end_month);
|
||||
let end = Date::from_calendar_date(end_year, end_month, end_day).unwrap();
|
||||
self.date_range_resolution(start, end)
|
||||
}
|
||||
|
||||
fn year_range(&self, year: i32) -> TemporalResolution {
|
||||
let start = Date::from_calendar_date(year, Month::January, 1).unwrap();
|
||||
let end = Date::from_calendar_date(year, Month::December, 31).unwrap();
|
||||
self.date_range_resolution(start, end)
|
||||
}
|
||||
|
||||
fn end_of_quarter(&self, quarter: i32) -> TemporalResolution {
|
||||
let year = self.anchor_date().year();
|
||||
let mut res = self.quarter_range(year, quarter);
|
||||
if let TemporalResolutionValue::DateRange { start: _, end } = &mut res.value {
|
||||
res.value = TemporalResolutionValue::Date(*end);
|
||||
}
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
res
|
||||
}
|
||||
|
||||
fn first_business_day_next_month(&self) -> TemporalResolution {
|
||||
let start = match self.start_of_next_month().value {
|
||||
TemporalResolutionValue::Date(date) => date,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let mut date = start;
|
||||
while matches!(date.weekday(), Weekday::Saturday | Weekday::Sunday) {
|
||||
date = add_days(date, 1);
|
||||
}
|
||||
let mut res = self.date_resolution(date);
|
||||
res.flags.push(TemporalResolutionFlag::Relative);
|
||||
res
|
||||
}
|
||||
|
||||
fn date_resolution(&self, date: Date) -> TemporalResolution {
|
||||
TemporalResolution {
|
||||
value: TemporalResolutionValue::Date(date),
|
||||
flags: Vec::new(),
|
||||
confidence: DEFAULT_CONFIDENCE,
|
||||
}
|
||||
}
|
||||
|
||||
fn date_range_resolution(&self, start: Date, end: Date) -> TemporalResolution {
|
||||
TemporalResolution {
|
||||
value: TemporalResolutionValue::DateRange { start, end },
|
||||
flags: Vec::new(),
|
||||
confidence: DEFAULT_CONFIDENCE,
|
||||
}
|
||||
}
|
||||
|
||||
fn datetime_resolution(&self, dt: OffsetDateTime) -> TemporalResolution {
|
||||
TemporalResolution {
|
||||
value: TemporalResolutionValue::DateTime(dt),
|
||||
flags: Vec::new(),
|
||||
confidence: DEFAULT_CONFIDENCE,
|
||||
}
|
||||
}
|
||||
|
||||
fn datetime_range_resolution(
|
||||
&self,
|
||||
start: OffsetDateTime,
|
||||
end: OffsetDateTime,
|
||||
) -> TemporalResolution {
|
||||
TemporalResolution {
|
||||
value: TemporalResolutionValue::DateTimeRange { start, end },
|
||||
flags: Vec::new(),
|
||||
confidence: DEFAULT_CONFIDENCE,
|
||||
}
|
||||
}
|
||||
|
||||
fn anchor_date(&self) -> Date {
|
||||
self.context.anchor.date()
|
||||
}
|
||||
|
||||
fn start_of_week(&self, date: Date) -> Date {
|
||||
let mut current = date;
|
||||
while current.weekday() != self.context.week_start {
|
||||
current = add_days(current, -1);
|
||||
}
|
||||
current
|
||||
}
|
||||
|
||||
fn next_weekday_after(&self, date: Date, weekday: Weekday) -> Date {
|
||||
let mut current = add_days(date, 1);
|
||||
while current.weekday() != weekday {
|
||||
current = add_days(current, 1);
|
||||
}
|
||||
current
|
||||
}
|
||||
|
||||
fn previous_weekday_before(&self, date: Date, weekday: Weekday) -> Date {
|
||||
let mut current = add_days(date, -1);
|
||||
while current.weekday() != weekday {
|
||||
current = add_days(current, -1);
|
||||
}
|
||||
current
|
||||
}
|
||||
|
||||
fn next_weekday_on_or_after(&self, date: Date, weekday: Weekday) -> Date {
|
||||
let mut current = date;
|
||||
while current.weekday() != weekday {
|
||||
current = add_days(current, 1);
|
||||
}
|
||||
current
|
||||
}
|
||||
}
|
||||
|
||||
fn add_days(date: Date, delta: i64) -> Date {
|
||||
date.checked_add(Duration::days(delta)).unwrap()
|
||||
}
|
||||
|
||||
fn add_months(year: i32, month: Month, delta: i32) -> (i32, Month) {
|
||||
let mut total = month as i32 + delta;
|
||||
let mut new_year = year;
|
||||
while total > 12 {
|
||||
total -= 12;
|
||||
new_year += 1;
|
||||
}
|
||||
while total < 1 {
|
||||
total += 12;
|
||||
new_year -= 1;
|
||||
}
|
||||
let month_enum = Month::try_from(total as u8).unwrap();
|
||||
(new_year, month_enum)
|
||||
}
|
||||
|
||||
fn last_day_of_month(year: i32, month: Month) -> u8 {
|
||||
let next_month = if month == Month::December {
|
||||
Date::from_calendar_date(year + 1, Month::January, 1).unwrap()
|
||||
} else {
|
||||
Date::from_calendar_date(year, month.next(), 1).unwrap()
|
||||
};
|
||||
add_days(next_month, -1).day()
|
||||
}
|
||||
|
||||
fn combine(
|
||||
date: Date,
|
||||
hour: i32,
|
||||
minute: i32,
|
||||
second: i32,
|
||||
offset: time::UtcOffset,
|
||||
) -> OffsetDateTime {
|
||||
let primitive = PrimitiveDateTime::new(
|
||||
date,
|
||||
Time::from_hms(hour as u8, minute as u8, second as u8).unwrap(),
|
||||
);
|
||||
primitive.assume_offset(offset)
|
||||
}
|
||||
|
||||
fn parse_number(token: &str) -> Option<i64> {
|
||||
if let Ok(value) = token.parse::<i64>() {
|
||||
return Some(value);
|
||||
}
|
||||
match token {
|
||||
"one" => Some(1),
|
||||
"two" => Some(2),
|
||||
"three" => Some(3),
|
||||
"four" => Some(4),
|
||||
"five" => Some(5),
|
||||
"six" => Some(6),
|
||||
"seven" => Some(7),
|
||||
"eight" => Some(8),
|
||||
"nine" => Some(9),
|
||||
"ten" => Some(10),
|
||||
"eleven" => Some(11),
|
||||
"twelve" => Some(12),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_weekday(token: &str) -> Option<Weekday> {
|
||||
match token {
|
||||
"monday" => Some(Weekday::Monday),
|
||||
"tuesday" => Some(Weekday::Tuesday),
|
||||
"wednesday" => Some(Weekday::Wednesday),
|
||||
"thursday" => Some(Weekday::Thursday),
|
||||
"friday" => Some(Weekday::Friday),
|
||||
"saturday" => Some(Weekday::Saturday),
|
||||
"sunday" => Some(Weekday::Sunday),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_hour(hour: i32, ampm: Option<&str>) -> Option<i32> {
|
||||
match ampm {
|
||||
Some(marker) => {
|
||||
if hour < 1 || hour > 12 {
|
||||
return None;
|
||||
}
|
||||
let converted = if marker == "pm" {
|
||||
if hour == 12 { 12 } else { hour + 12 }
|
||||
} else if hour == 12 {
|
||||
0
|
||||
} else {
|
||||
hour
|
||||
};
|
||||
Some(converted)
|
||||
}
|
||||
None => {
|
||||
if (0..=23).contains(&hour) {
|
||||
Some(hour)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_ampm(input: &str) -> String {
|
||||
input
|
||||
.replace("a.m.", "am")
|
||||
.replace("p.m.", "pm")
|
||||
.replace("a.m", "am")
|
||||
.replace("p.m", "pm")
|
||||
}
|
||||
|
||||
fn parse_year(token: &str) -> i32 {
|
||||
if token.len() == 2 {
|
||||
let value: i32 = token.parse().unwrap();
|
||||
2000 + value
|
||||
} else {
|
||||
token.parse().unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_week_start(value: &str) -> Option<Weekday> {
|
||||
parse_weekday(&value.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
pub fn parse_clock_inheritance(value: &str) -> bool {
|
||||
!matches!(value.to_ascii_lowercase().as_str(), "drop")
|
||||
}
|
||||
@@ -0,0 +1,881 @@
|
||||
//! Temporal Enrichment Module
|
||||
//!
|
||||
//! Implements the "Sliding Anchor" pipeline for resolving relative temporal
|
||||
//! references during document ingestion. This module detects date anchors
|
||||
//! in document text, tracks the current temporal context, and resolves
|
||||
//! relative phrases like "last year" to absolute dates.
|
||||
//!
|
||||
//! # Architecture
|
||||
//!
|
||||
//! 1. **Anchor Detection**: Regex patterns detect explicit dates in text
|
||||
//! (e.g., "May 7, 2023", "2023-05-07", session headers)
|
||||
//! 2. **Anchor Propagation**: State machine tracks current anchor through document
|
||||
//! 3. **Phrase Detection**: Identifies relative temporal phrases ("last year", "next week")
|
||||
//! 4. **Resolution**: Converts relative phrases to absolute dates using anchor
|
||||
//! 5. **Context Injection**: Appends resolved context to chunk for embedding
|
||||
|
||||
#![cfg(feature = "temporal_enrich")]
|
||||
|
||||
use chrono::{Datelike, NaiveDate, NaiveDateTime};
|
||||
use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Source of a temporal anchor
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum AnchorSource {
|
||||
/// From document/file metadata (creation date, etc.)
|
||||
DocumentMetadata,
|
||||
/// From explicit header (e.g., "Session 5 (May 7, 2023)")
|
||||
ExplicitHeader,
|
||||
/// From inline date in text (e.g., "On May 7, 2023, ...")
|
||||
InlineDate,
|
||||
/// Inherited from previous chunk/section
|
||||
Inherited,
|
||||
}
|
||||
|
||||
/// A detected temporal anchor
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TemporalAnchorInfo {
|
||||
/// The resolved date
|
||||
pub date: NaiveDate,
|
||||
/// Source of this anchor
|
||||
pub source: AnchorSource,
|
||||
/// Confidence score (0.0-1.0)
|
||||
pub confidence: f32,
|
||||
/// Original text that produced this anchor
|
||||
pub original_text: String,
|
||||
/// Character offset in the original document
|
||||
pub char_offset: usize,
|
||||
}
|
||||
|
||||
/// A detected relative temporal phrase
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RelativePhrase {
|
||||
/// The phrase as found in text (e.g., "last year")
|
||||
pub phrase: String,
|
||||
/// Character offset in the chunk
|
||||
pub char_offset: usize,
|
||||
/// Length of the phrase
|
||||
pub length: usize,
|
||||
/// Resolved absolute value (if anchor available)
|
||||
pub resolved: Option<ResolvedTemporal>,
|
||||
}
|
||||
|
||||
/// Resolved temporal value
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum ResolvedTemporal {
|
||||
/// Single date
|
||||
Date(NaiveDate),
|
||||
/// Date range (start, end)
|
||||
DateRange { start: NaiveDate, end: NaiveDate },
|
||||
/// Year only
|
||||
Year(i32),
|
||||
/// Month in a year
|
||||
Month { year: i32, month: u32 },
|
||||
}
|
||||
|
||||
impl ResolvedTemporal {
|
||||
/// Format as a human-readable string
|
||||
#[must_use]
|
||||
pub fn to_display_string(&self) -> String {
|
||||
match self {
|
||||
Self::Date(d) => d.format("%B %d, %Y").to_string(),
|
||||
Self::DateRange { start, end } => {
|
||||
format!(
|
||||
"{} to {}",
|
||||
start.format("%B %d, %Y"),
|
||||
end.format("%B %d, %Y")
|
||||
)
|
||||
}
|
||||
Self::Year(y) => y.to_string(),
|
||||
Self::Month { year, month } => {
|
||||
let month_name = match month {
|
||||
1 => "January",
|
||||
2 => "February",
|
||||
3 => "March",
|
||||
4 => "April",
|
||||
5 => "May",
|
||||
6 => "June",
|
||||
7 => "July",
|
||||
8 => "August",
|
||||
9 => "September",
|
||||
10 => "October",
|
||||
11 => "November",
|
||||
12 => "December",
|
||||
_ => "Unknown",
|
||||
};
|
||||
format!("{month_name} {year}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Format as ISO-8601 compatible string
|
||||
#[must_use]
|
||||
pub fn to_iso_string(&self) -> String {
|
||||
match self {
|
||||
Self::Date(d) => d.format("%Y-%m-%d").to_string(),
|
||||
Self::DateRange { start, end } => {
|
||||
format!("{}/{}", start.format("%Y-%m-%d"), end.format("%Y-%m-%d"))
|
||||
}
|
||||
Self::Year(y) => format!("{y}"),
|
||||
Self::Month { year, month } => format!("{year}-{month:02}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of temporal enrichment for a chunk
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct TemporalEnrichment {
|
||||
/// Current anchor for this chunk
|
||||
pub anchor: Option<TemporalAnchorInfo>,
|
||||
/// Detected relative phrases with resolutions
|
||||
pub relative_phrases: Vec<RelativePhrase>,
|
||||
/// Generated context block to append to embedding text
|
||||
pub context_block: Option<String>,
|
||||
}
|
||||
|
||||
/// Sliding anchor state machine for tracking temporal context
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TemporalAnchorTracker {
|
||||
/// Current anchor date
|
||||
current_anchor: Option<NaiveDate>,
|
||||
/// Source of current anchor
|
||||
anchor_source: Option<AnchorSource>,
|
||||
/// Confidence of current anchor
|
||||
anchor_confidence: f32,
|
||||
/// Original text of current anchor
|
||||
anchor_text: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for TemporalAnchorTracker {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl TemporalAnchorTracker {
|
||||
/// Create a new tracker with no anchor
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
current_anchor: None,
|
||||
anchor_source: None,
|
||||
anchor_confidence: 0.0,
|
||||
anchor_text: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a tracker with an initial anchor from document metadata
|
||||
#[must_use]
|
||||
pub fn with_document_date(date: NaiveDate) -> Self {
|
||||
Self {
|
||||
current_anchor: Some(date),
|
||||
anchor_source: Some(AnchorSource::DocumentMetadata),
|
||||
anchor_confidence: 0.7,
|
||||
anchor_text: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the current anchor date
|
||||
#[must_use]
|
||||
pub fn current_anchor(&self) -> Option<NaiveDate> {
|
||||
self.current_anchor
|
||||
}
|
||||
|
||||
/// Process a line of text, updating anchor if a date is found
|
||||
pub fn process_line(&mut self, line: &str, char_offset: usize) -> Option<TemporalAnchorInfo> {
|
||||
// Try to detect a date anchor in this line
|
||||
if let Some((date, source, confidence, text)) = detect_anchor_in_line(line) {
|
||||
// Only update if new anchor has higher confidence or is more specific
|
||||
let should_update = self.current_anchor.is_none()
|
||||
|| confidence > self.anchor_confidence
|
||||
|| matches!(source, AnchorSource::ExplicitHeader);
|
||||
|
||||
if should_update {
|
||||
self.current_anchor = Some(date);
|
||||
self.anchor_source = Some(source);
|
||||
self.anchor_confidence = confidence;
|
||||
self.anchor_text = Some(text.clone());
|
||||
|
||||
return Some(TemporalAnchorInfo {
|
||||
date,
|
||||
source,
|
||||
confidence,
|
||||
original_text: text,
|
||||
char_offset,
|
||||
});
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Get current anchor info
|
||||
#[must_use]
|
||||
pub fn anchor_info(&self) -> Option<TemporalAnchorInfo> {
|
||||
self.current_anchor.map(|date| TemporalAnchorInfo {
|
||||
date,
|
||||
source: self.anchor_source.unwrap_or(AnchorSource::Inherited),
|
||||
confidence: self.anchor_confidence,
|
||||
original_text: self.anchor_text.clone().unwrap_or_default(),
|
||||
char_offset: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Regex patterns for anchor detection
|
||||
static SESSION_HEADER_PATTERN: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(?i)Session\s+\d+\s*\(([^)]+)\)").expect("valid regex"));
|
||||
|
||||
static DATE_HEADER_PATTERN: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(?i)\[(?:SESSION_)?DATE:\s*([^\]]+)\]").expect("valid regex"));
|
||||
|
||||
static ISO_DATE_PATTERN: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(\d{4})[/-](\d{1,2})[/-](\d{1,2})").expect("valid regex"));
|
||||
|
||||
static LONG_DATE_PATTERN: Lazy<Regex> = Lazy::new(|| {
|
||||
Regex::new(r"(?i)(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),?\s+(\d{4})").expect("valid regex")
|
||||
});
|
||||
|
||||
static SHORT_DATE_PATTERN: Lazy<Regex> = Lazy::new(|| {
|
||||
Regex::new(
|
||||
r"(?i)(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+(\d{1,2}),?\s+(\d{4})",
|
||||
)
|
||||
.expect("valid regex")
|
||||
});
|
||||
|
||||
static SLASH_DATE_PATTERN: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(\d{1,2})/(\d{1,2})/(\d{2,4})").expect("valid regex"));
|
||||
|
||||
// Regex patterns for relative phrase detection
|
||||
static RELATIVE_YEAR_PATTERN: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(?i)\b(last|this|next)\s+year\b").expect("valid regex"));
|
||||
|
||||
static RELATIVE_MONTH_PATTERN: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(?i)\b(last|this|next)\s+month\b").expect("valid regex"));
|
||||
|
||||
static RELATIVE_WEEK_PATTERN: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(?i)\b(last|this|next)\s+week\b").expect("valid regex"));
|
||||
|
||||
static AGO_PATTERN: Lazy<Regex> = Lazy::new(|| {
|
||||
Regex::new(r"(?i)\b(\d+|a|one|two|three|four|five|six|seven|eight|nine|ten)\s+(days?|weeks?|months?|years?)\s+ago\b").expect("valid regex")
|
||||
});
|
||||
|
||||
static IN_FUTURE_PATTERN: Lazy<Regex> = Lazy::new(|| {
|
||||
Regex::new(r"(?i)\bin\s+(\d+|a|one|two|three|four|five|six|seven|eight|nine|ten)\s+(days?|weeks?|months?|years?)\b").expect("valid regex")
|
||||
});
|
||||
|
||||
static RELATIVE_DAY_PATTERN: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r"(?i)\b(yesterday|today|tomorrow)\b").expect("valid regex"));
|
||||
|
||||
static RELATIVE_WEEKDAY_PATTERN: Lazy<Regex> = Lazy::new(|| {
|
||||
Regex::new(
|
||||
r"(?i)\b(last|this|next)\s+(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\b",
|
||||
)
|
||||
.expect("valid regex")
|
||||
});
|
||||
|
||||
/// Detect an anchor date in a line of text
|
||||
fn detect_anchor_in_line(line: &str) -> Option<(NaiveDate, AnchorSource, f32, String)> {
|
||||
// Priority 1: Session headers (highest confidence)
|
||||
if let Some(caps) = SESSION_HEADER_PATTERN.captures(line) {
|
||||
if let Some(date_str) = caps.get(1) {
|
||||
if let Some(date) = parse_date_string(date_str.as_str()) {
|
||||
return Some((
|
||||
date,
|
||||
AnchorSource::ExplicitHeader,
|
||||
0.95,
|
||||
caps[0].to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 2: Date headers [DATE: ...]
|
||||
if let Some(caps) = DATE_HEADER_PATTERN.captures(line) {
|
||||
if let Some(date_str) = caps.get(1) {
|
||||
if let Some(date) = parse_date_string(date_str.as_str()) {
|
||||
return Some((
|
||||
date,
|
||||
AnchorSource::ExplicitHeader,
|
||||
0.95,
|
||||
caps[0].to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 3: ISO dates (2023-05-07)
|
||||
if let Some(caps) = ISO_DATE_PATTERN.captures(line) {
|
||||
let year: i32 = caps[1].parse().ok()?;
|
||||
let month: u32 = caps[2].parse().ok()?;
|
||||
let day: u32 = caps[3].parse().ok()?;
|
||||
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
|
||||
return Some((date, AnchorSource::InlineDate, 0.9, caps[0].to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 4: Long month names (May 7, 2023)
|
||||
if let Some(caps) = LONG_DATE_PATTERN.captures(line) {
|
||||
let month_str = &caps[1];
|
||||
let day: u32 = caps[2].parse().ok()?;
|
||||
let year: i32 = caps[3].parse().ok()?;
|
||||
let month = month_name_to_number(month_str)?;
|
||||
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
|
||||
return Some((date, AnchorSource::InlineDate, 0.85, caps[0].to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 5: Short month names (May 7, 2023)
|
||||
if let Some(caps) = SHORT_DATE_PATTERN.captures(line) {
|
||||
let month_str = &caps[1];
|
||||
let day: u32 = caps[2].parse().ok()?;
|
||||
let year: i32 = caps[3].parse().ok()?;
|
||||
let month = month_name_to_number(month_str)?;
|
||||
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
|
||||
return Some((date, AnchorSource::InlineDate, 0.85, caps[0].to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 6: Slash dates (MM/DD/YYYY) - lower confidence due to ambiguity
|
||||
if let Some(caps) = SLASH_DATE_PATTERN.captures(line) {
|
||||
let month: u32 = caps[1].parse().ok()?;
|
||||
let day: u32 = caps[2].parse().ok()?;
|
||||
let mut year: i32 = caps[3].parse().ok()?;
|
||||
// Handle 2-digit years
|
||||
if year < 100 {
|
||||
year += if year > 50 { 1900 } else { 2000 };
|
||||
}
|
||||
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
|
||||
return Some((date, AnchorSource::InlineDate, 0.7, caps[0].to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Parse a date string in various formats
|
||||
fn parse_date_string(s: &str) -> Option<NaiveDate> {
|
||||
let s = s.trim();
|
||||
|
||||
// Try ISO format first
|
||||
if let Ok(date) = NaiveDate::parse_from_str(s, "%Y-%m-%d") {
|
||||
return Some(date);
|
||||
}
|
||||
if let Ok(date) = NaiveDate::parse_from_str(s, "%Y/%m/%d") {
|
||||
return Some(date);
|
||||
}
|
||||
|
||||
// Try datetime formats
|
||||
if let Ok(dt) = NaiveDateTime::parse_from_str(s, "%Y/%m/%d (%a) %H:%M") {
|
||||
return Some(dt.date());
|
||||
}
|
||||
|
||||
// Try long month format
|
||||
if let Some(caps) = LONG_DATE_PATTERN.captures(s) {
|
||||
let month_str = &caps[1];
|
||||
let day: u32 = caps[2].parse().ok()?;
|
||||
let year: i32 = caps[3].parse().ok()?;
|
||||
let month = month_name_to_number(month_str)?;
|
||||
return NaiveDate::from_ymd_opt(year, month, day);
|
||||
}
|
||||
|
||||
// Try short month format
|
||||
if let Some(caps) = SHORT_DATE_PATTERN.captures(s) {
|
||||
let month_str = &caps[1];
|
||||
let day: u32 = caps[2].parse().ok()?;
|
||||
let year: i32 = caps[3].parse().ok()?;
|
||||
let month = month_name_to_number(month_str)?;
|
||||
return NaiveDate::from_ymd_opt(year, month, day);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Convert month name to number
|
||||
fn month_name_to_number(name: &str) -> Option<u32> {
|
||||
match name.to_lowercase().as_str() {
|
||||
"january" | "jan" => Some(1),
|
||||
"february" | "feb" => Some(2),
|
||||
"march" | "mar" => Some(3),
|
||||
"april" | "apr" => Some(4),
|
||||
"may" => Some(5),
|
||||
"june" | "jun" => Some(6),
|
||||
"july" | "jul" => Some(7),
|
||||
"august" | "aug" => Some(8),
|
||||
"september" | "sep" | "sept" => Some(9),
|
||||
"october" | "oct" => Some(10),
|
||||
"november" | "nov" => Some(11),
|
||||
"december" | "dec" => Some(12),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a number word to integer
|
||||
fn parse_number_word(s: &str) -> Option<i32> {
|
||||
let s = s.to_lowercase();
|
||||
match s.as_str() {
|
||||
"a" | "one" => Some(1),
|
||||
"two" => Some(2),
|
||||
"three" => Some(3),
|
||||
"four" => Some(4),
|
||||
"five" => Some(5),
|
||||
"six" => Some(6),
|
||||
"seven" => Some(7),
|
||||
"eight" => Some(8),
|
||||
"nine" => Some(9),
|
||||
"ten" => Some(10),
|
||||
_ => s.parse().ok(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect relative temporal phrases in text
|
||||
#[must_use]
|
||||
pub fn detect_relative_phrases(text: &str) -> Vec<(String, usize, usize)> {
|
||||
let mut phrases = Vec::new();
|
||||
|
||||
// Relative year patterns
|
||||
for caps in RELATIVE_YEAR_PATTERN.captures_iter(text) {
|
||||
let m = caps.get(0).expect("full match");
|
||||
phrases.push((m.as_str().to_string(), m.start(), m.len()));
|
||||
}
|
||||
|
||||
// Relative month patterns
|
||||
for caps in RELATIVE_MONTH_PATTERN.captures_iter(text) {
|
||||
let m = caps.get(0).expect("full match");
|
||||
phrases.push((m.as_str().to_string(), m.start(), m.len()));
|
||||
}
|
||||
|
||||
// Relative week patterns
|
||||
for caps in RELATIVE_WEEK_PATTERN.captures_iter(text) {
|
||||
let m = caps.get(0).expect("full match");
|
||||
phrases.push((m.as_str().to_string(), m.start(), m.len()));
|
||||
}
|
||||
|
||||
// "N days/weeks/months/years ago" patterns
|
||||
for caps in AGO_PATTERN.captures_iter(text) {
|
||||
let m = caps.get(0).expect("full match");
|
||||
phrases.push((m.as_str().to_string(), m.start(), m.len()));
|
||||
}
|
||||
|
||||
// "in N days/weeks/months/years" patterns
|
||||
for caps in IN_FUTURE_PATTERN.captures_iter(text) {
|
||||
let m = caps.get(0).expect("full match");
|
||||
phrases.push((m.as_str().to_string(), m.start(), m.len()));
|
||||
}
|
||||
|
||||
// yesterday/today/tomorrow
|
||||
for caps in RELATIVE_DAY_PATTERN.captures_iter(text) {
|
||||
let m = caps.get(0).expect("full match");
|
||||
phrases.push((m.as_str().to_string(), m.start(), m.len()));
|
||||
}
|
||||
|
||||
// last/next weekday
|
||||
for caps in RELATIVE_WEEKDAY_PATTERN.captures_iter(text) {
|
||||
let m = caps.get(0).expect("full match");
|
||||
phrases.push((m.as_str().to_string(), m.start(), m.len()));
|
||||
}
|
||||
|
||||
// Sort by position
|
||||
phrases.sort_by_key(|(_, pos, _)| *pos);
|
||||
|
||||
phrases
|
||||
}
|
||||
|
||||
/// Resolve a relative phrase to an absolute date using an anchor
|
||||
#[must_use]
|
||||
pub fn resolve_relative_phrase(phrase: &str, anchor: NaiveDate) -> Option<ResolvedTemporal> {
|
||||
let lower = phrase.to_lowercase();
|
||||
|
||||
// Relative year
|
||||
if lower.contains("last year") {
|
||||
return Some(ResolvedTemporal::Year(anchor.year() - 1));
|
||||
}
|
||||
if lower.contains("this year") {
|
||||
return Some(ResolvedTemporal::Year(anchor.year()));
|
||||
}
|
||||
if lower.contains("next year") {
|
||||
return Some(ResolvedTemporal::Year(anchor.year() + 1));
|
||||
}
|
||||
|
||||
// Relative month
|
||||
if lower.contains("last month") {
|
||||
let (y, m) = if anchor.month() == 1 {
|
||||
(anchor.year() - 1, 12)
|
||||
} else {
|
||||
(anchor.year(), anchor.month() - 1)
|
||||
};
|
||||
return Some(ResolvedTemporal::Month { year: y, month: m });
|
||||
}
|
||||
if lower.contains("this month") {
|
||||
return Some(ResolvedTemporal::Month {
|
||||
year: anchor.year(),
|
||||
month: anchor.month(),
|
||||
});
|
||||
}
|
||||
if lower.contains("next month") {
|
||||
let (y, m) = if anchor.month() == 12 {
|
||||
(anchor.year() + 1, 1)
|
||||
} else {
|
||||
(anchor.year(), anchor.month() + 1)
|
||||
};
|
||||
return Some(ResolvedTemporal::Month { year: y, month: m });
|
||||
}
|
||||
|
||||
// Relative week - return date range
|
||||
if lower.contains("last week") {
|
||||
let start =
|
||||
anchor - chrono::Duration::days(7 + anchor.weekday().num_days_from_monday() as i64);
|
||||
let end = start + chrono::Duration::days(6);
|
||||
return Some(ResolvedTemporal::DateRange { start, end });
|
||||
}
|
||||
if lower.contains("this week") {
|
||||
let start = anchor - chrono::Duration::days(anchor.weekday().num_days_from_monday() as i64);
|
||||
let end = start + chrono::Duration::days(6);
|
||||
return Some(ResolvedTemporal::DateRange { start, end });
|
||||
}
|
||||
if lower.contains("next week") {
|
||||
let start =
|
||||
anchor + chrono::Duration::days(7 - anchor.weekday().num_days_from_monday() as i64);
|
||||
let end = start + chrono::Duration::days(6);
|
||||
return Some(ResolvedTemporal::DateRange { start, end });
|
||||
}
|
||||
|
||||
// yesterday/today/tomorrow
|
||||
if lower == "yesterday" {
|
||||
return Some(ResolvedTemporal::Date(anchor - chrono::Duration::days(1)));
|
||||
}
|
||||
if lower == "today" {
|
||||
return Some(ResolvedTemporal::Date(anchor));
|
||||
}
|
||||
if lower == "tomorrow" {
|
||||
return Some(ResolvedTemporal::Date(anchor + chrono::Duration::days(1)));
|
||||
}
|
||||
|
||||
// N units ago
|
||||
if let Some(caps) = AGO_PATTERN.captures(&lower) {
|
||||
let count = parse_number_word(&caps[1])?;
|
||||
let unit = &caps[2];
|
||||
|
||||
return match unit {
|
||||
u if u.starts_with("day") => Some(ResolvedTemporal::Date(
|
||||
anchor - chrono::Duration::days(count as i64),
|
||||
)),
|
||||
u if u.starts_with("week") => Some(ResolvedTemporal::Date(
|
||||
anchor - chrono::Duration::weeks(count as i64),
|
||||
)),
|
||||
u if u.starts_with("month") => {
|
||||
let total_months = anchor.year() * 12 + anchor.month() as i32 - count;
|
||||
let new_year = (total_months - 1) / 12;
|
||||
let new_month = ((total_months - 1) % 12 + 1) as u32;
|
||||
NaiveDate::from_ymd_opt(new_year, new_month, anchor.day().min(28))
|
||||
.map(ResolvedTemporal::Date)
|
||||
}
|
||||
u if u.starts_with("year") => Some(ResolvedTemporal::Year(anchor.year() - count)),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
|
||||
// in N units
|
||||
if let Some(caps) = IN_FUTURE_PATTERN.captures(&lower) {
|
||||
let count = parse_number_word(&caps[1])?;
|
||||
let unit = &caps[2];
|
||||
|
||||
return match unit {
|
||||
u if u.starts_with("day") => Some(ResolvedTemporal::Date(
|
||||
anchor + chrono::Duration::days(count as i64),
|
||||
)),
|
||||
u if u.starts_with("week") => Some(ResolvedTemporal::Date(
|
||||
anchor + chrono::Duration::weeks(count as i64),
|
||||
)),
|
||||
u if u.starts_with("month") => {
|
||||
let total_months = anchor.year() * 12 + anchor.month() as i32 + count;
|
||||
let new_year = (total_months - 1) / 12;
|
||||
let new_month = ((total_months - 1) % 12 + 1) as u32;
|
||||
NaiveDate::from_ymd_opt(new_year, new_month, anchor.day().min(28))
|
||||
.map(ResolvedTemporal::Date)
|
||||
}
|
||||
u if u.starts_with("year") => Some(ResolvedTemporal::Year(anchor.year() + count)),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
|
||||
// Relative weekday (last Monday, next Friday, etc.)
|
||||
if let Some(caps) = RELATIVE_WEEKDAY_PATTERN.captures(&lower) {
|
||||
let direction = &caps[1];
|
||||
let weekday_name = &caps[2];
|
||||
|
||||
let target_weekday = match weekday_name.to_lowercase().as_str() {
|
||||
"monday" => chrono::Weekday::Mon,
|
||||
"tuesday" => chrono::Weekday::Tue,
|
||||
"wednesday" => chrono::Weekday::Wed,
|
||||
"thursday" => chrono::Weekday::Thu,
|
||||
"friday" => chrono::Weekday::Fri,
|
||||
"saturday" => chrono::Weekday::Sat,
|
||||
"sunday" => chrono::Weekday::Sun,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let current_weekday = anchor.weekday();
|
||||
let days_diff = (target_weekday.num_days_from_monday() as i64)
|
||||
- (current_weekday.num_days_from_monday() as i64);
|
||||
|
||||
let result_date = match direction.to_lowercase().as_str() {
|
||||
"last" => {
|
||||
let mut offset = days_diff;
|
||||
if offset >= 0 {
|
||||
offset -= 7;
|
||||
}
|
||||
anchor + chrono::Duration::days(offset)
|
||||
}
|
||||
"this" => anchor + chrono::Duration::days(days_diff),
|
||||
"next" => {
|
||||
let mut offset = days_diff;
|
||||
if offset <= 0 {
|
||||
offset += 7;
|
||||
}
|
||||
anchor + chrono::Duration::days(offset)
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
return Some(ResolvedTemporal::Date(result_date));
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Enrich a chunk of text with temporal context
|
||||
///
|
||||
/// This function:
|
||||
/// 1. Detects any anchor dates in the text
|
||||
/// 2. Finds all relative temporal phrases
|
||||
/// 3. Resolves relative phrases using the anchor
|
||||
/// 4. Generates a context block for embedding
|
||||
#[must_use]
|
||||
pub fn enrich_chunk(text: &str, tracker: &mut TemporalAnchorTracker) -> TemporalEnrichment {
|
||||
let mut result = TemporalEnrichment::default();
|
||||
|
||||
// Process each line to detect anchors
|
||||
let mut char_offset = 0;
|
||||
for line in text.lines() {
|
||||
if let Some(anchor_info) = tracker.process_line(line, char_offset) {
|
||||
result.anchor = Some(anchor_info);
|
||||
}
|
||||
char_offset += line.len() + 1; // +1 for newline
|
||||
}
|
||||
|
||||
// If no anchor in this chunk, inherit from tracker
|
||||
if result.anchor.is_none() {
|
||||
result.anchor = tracker.anchor_info();
|
||||
}
|
||||
|
||||
// Detect relative phrases
|
||||
let phrases = detect_relative_phrases(text);
|
||||
|
||||
// Resolve phrases if we have an anchor
|
||||
if let Some(ref anchor_info) = result.anchor {
|
||||
for (phrase, offset, len) in phrases {
|
||||
let resolved = resolve_relative_phrase(&phrase, anchor_info.date);
|
||||
result.relative_phrases.push(RelativePhrase {
|
||||
phrase,
|
||||
char_offset: offset,
|
||||
length: len,
|
||||
resolved,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// No anchor, can't resolve
|
||||
for (phrase, offset, len) in phrases {
|
||||
result.relative_phrases.push(RelativePhrase {
|
||||
phrase,
|
||||
char_offset: offset,
|
||||
length: len,
|
||||
resolved: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Generate context block if we have resolutions
|
||||
let resolved_phrases: Vec<_> = result
|
||||
.relative_phrases
|
||||
.iter()
|
||||
.filter_map(|p| p.resolved.as_ref().map(|r| (p.phrase.clone(), r.clone())))
|
||||
.collect();
|
||||
|
||||
if !resolved_phrases.is_empty() {
|
||||
let mut context_parts = Vec::new();
|
||||
|
||||
if let Some(ref anchor) = result.anchor {
|
||||
context_parts.push(format!(
|
||||
"Document date context: {}",
|
||||
anchor.date.format("%B %d, %Y")
|
||||
));
|
||||
}
|
||||
|
||||
context_parts.push("Temporal references:".to_string());
|
||||
for (phrase, resolved) in &resolved_phrases {
|
||||
context_parts.push(format!(
|
||||
"- \"{}\" refers to {}",
|
||||
phrase,
|
||||
resolved.to_display_string()
|
||||
));
|
||||
}
|
||||
|
||||
result.context_block = Some(context_parts.join(" "));
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Enrich a full document, returning enriched text with context blocks
|
||||
#[must_use]
|
||||
pub fn enrich_document(text: &str, document_date: Option<NaiveDate>) -> String {
|
||||
let mut tracker = match document_date {
|
||||
Some(date) => TemporalAnchorTracker::with_document_date(date),
|
||||
None => TemporalAnchorTracker::new(),
|
||||
};
|
||||
|
||||
let enrichment = enrich_chunk(text, &mut tracker);
|
||||
|
||||
// Append context block to text if we have temporal resolutions
|
||||
if let Some(context) = enrichment.context_block {
|
||||
format!("{text}\n\n[Temporal Context: {context}]")
|
||||
} else {
|
||||
text.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Batch enrich multiple chunks, maintaining anchor state across chunks
|
||||
pub fn enrich_chunks(
|
||||
chunks: &[String],
|
||||
document_date: Option<NaiveDate>,
|
||||
) -> Vec<(String, TemporalEnrichment)> {
|
||||
let mut tracker = match document_date {
|
||||
Some(date) => TemporalAnchorTracker::with_document_date(date),
|
||||
None => TemporalAnchorTracker::new(),
|
||||
};
|
||||
|
||||
chunks
|
||||
.iter()
|
||||
.map(|chunk| {
|
||||
let enrichment = enrich_chunk(chunk, &mut tracker);
|
||||
let enriched_text = if let Some(ref context) = enrichment.context_block {
|
||||
format!("{chunk}\n\n[Temporal Context: {context}]")
|
||||
} else {
|
||||
chunk.clone()
|
||||
};
|
||||
(enriched_text, enrichment)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_anchor_detection_session_header() {
|
||||
let mut tracker = TemporalAnchorTracker::new();
|
||||
let line = "=== Session 5 (May 7, 2023) ===";
|
||||
let info = tracker.process_line(line, 0);
|
||||
assert!(info.is_some());
|
||||
let info = info.expect("anchor info");
|
||||
assert_eq!(
|
||||
info.date,
|
||||
NaiveDate::from_ymd_opt(2023, 5, 7).expect("valid date")
|
||||
);
|
||||
assert_eq!(info.source, AnchorSource::ExplicitHeader);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_anchor_detection_iso_date() {
|
||||
let mut tracker = TemporalAnchorTracker::new();
|
||||
let line = "Event occurred on 2023-05-07 at noon.";
|
||||
let info = tracker.process_line(line, 0);
|
||||
assert!(info.is_some());
|
||||
let info = info.expect("anchor info");
|
||||
assert_eq!(
|
||||
info.date,
|
||||
NaiveDate::from_ymd_opt(2023, 5, 7).expect("valid date")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_relative_phrase_detection() {
|
||||
let text = "I did this last year. We'll meet next week. Two days ago was fun.";
|
||||
let phrases = detect_relative_phrases(text);
|
||||
assert_eq!(phrases.len(), 3);
|
||||
assert_eq!(phrases[0].0, "last year");
|
||||
assert_eq!(phrases[1].0, "next week");
|
||||
assert_eq!(phrases[2].0, "Two days ago");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_last_year() {
|
||||
let anchor = NaiveDate::from_ymd_opt(2023, 5, 7).expect("valid date");
|
||||
let resolved = resolve_relative_phrase("last year", anchor);
|
||||
assert!(resolved.is_some());
|
||||
if let Some(ResolvedTemporal::Year(y)) = resolved {
|
||||
assert_eq!(y, 2022);
|
||||
} else {
|
||||
panic!("Expected Year resolution");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_last_week() {
|
||||
let anchor = NaiveDate::from_ymd_opt(2023, 5, 10).expect("valid date"); // Wednesday
|
||||
let resolved = resolve_relative_phrase("last week", anchor);
|
||||
assert!(resolved.is_some());
|
||||
if let Some(ResolvedTemporal::DateRange { start, end }) = resolved {
|
||||
assert_eq!(
|
||||
start,
|
||||
NaiveDate::from_ymd_opt(2023, 5, 1).expect("valid date")
|
||||
); // Previous Monday
|
||||
assert_eq!(
|
||||
end,
|
||||
NaiveDate::from_ymd_opt(2023, 5, 7).expect("valid date")
|
||||
); // Previous Sunday
|
||||
} else {
|
||||
panic!("Expected DateRange resolution");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_enrich_chunk() {
|
||||
let mut tracker = TemporalAnchorTracker::new();
|
||||
let text = "=== Session 1 (May 7, 2023) ===\n\nI painted a sunrise last year.";
|
||||
let enrichment = enrich_chunk(text, &mut tracker);
|
||||
|
||||
assert!(enrichment.anchor.is_some());
|
||||
assert_eq!(enrichment.relative_phrases.len(), 1);
|
||||
assert!(enrichment.relative_phrases[0].resolved.is_some());
|
||||
assert!(enrichment.context_block.is_some());
|
||||
|
||||
let context = enrichment.context_block.as_ref().expect("context");
|
||||
assert!(context.contains("2022"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_enrich_document() {
|
||||
let text = "Session 1 (May 7, 2023)\n\nMelanie: I painted a sunrise last year.";
|
||||
let enriched = enrich_document(text, None);
|
||||
assert!(enriched.contains("[Temporal Context:"));
|
||||
assert!(enriched.contains("2022"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_anchor_propagation() {
|
||||
let chunks = vec![
|
||||
"Session 1 (May 7, 2023)\n\nHello!".to_string(),
|
||||
"This happened last year.".to_string(), // Should inherit anchor
|
||||
];
|
||||
let results = enrich_chunks(&chunks, None);
|
||||
|
||||
// Second chunk should have resolution because anchor propagated
|
||||
assert!(results[1].1.anchor.is_some());
|
||||
assert!(!results[1].1.relative_phrases.is_empty());
|
||||
assert!(results[1].1.relative_phrases[0].resolved.is_some());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,596 @@
|
||||
//! API-based embedding providers (OpenAI, Anthropic, etc.)
|
||||
//!
|
||||
//! This module provides cloud API embedding generation, enabling semantic search
|
||||
//! using external embedding services. Requires the `api_embed` feature.
|
||||
//!
|
||||
//! # Example
|
||||
//!
|
||||
//! ```ignore
|
||||
//! use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
|
||||
//! use memvid_core::types::embedding::EmbeddingProvider;
|
||||
//!
|
||||
//! // Requires OPENAI_API_KEY environment variable
|
||||
//! let config = OpenAIConfig::default();
|
||||
//! let embedder = OpenAIEmbedder::new(config)?;
|
||||
//!
|
||||
//! let embedding = embedder.embed_text("Hello, world!")?;
|
||||
//! println!("Embedding dimension: {}", embedding.len());
|
||||
//! ```
|
||||
|
||||
use crate::error::{MemvidError, Result};
|
||||
use crate::types::embedding::EmbeddingProvider;
|
||||
use reqwest::blocking::Client;
|
||||
use reqwest::header::{AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
|
||||
// ============================================================================
|
||||
// OpenAI Models Registry
|
||||
// ============================================================================
|
||||
|
||||
/// OpenAI embedding model information
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OpenAIModelInfo {
|
||||
/// Model identifier (e.g., "text-embedding-3-small")
|
||||
pub name: &'static str,
|
||||
/// Output embedding dimension
|
||||
pub dimension: usize,
|
||||
/// Maximum input tokens
|
||||
pub max_tokens: usize,
|
||||
/// Maximum texts per batch request
|
||||
pub max_batch_size: usize,
|
||||
/// Whether this is the default model
|
||||
pub is_default: bool,
|
||||
}
|
||||
|
||||
/// Available OpenAI embedding models
|
||||
pub static OPENAI_MODELS: &[OpenAIModelInfo] = &[
|
||||
OpenAIModelInfo {
|
||||
name: "text-embedding-3-small",
|
||||
dimension: 1536,
|
||||
max_tokens: 8191,
|
||||
max_batch_size: 2048,
|
||||
is_default: true,
|
||||
},
|
||||
OpenAIModelInfo {
|
||||
name: "text-embedding-3-large",
|
||||
dimension: 3072,
|
||||
max_tokens: 8191,
|
||||
max_batch_size: 2048,
|
||||
is_default: false,
|
||||
},
|
||||
OpenAIModelInfo {
|
||||
name: "text-embedding-ada-002",
|
||||
dimension: 1536,
|
||||
max_tokens: 8191,
|
||||
max_batch_size: 2048,
|
||||
is_default: false,
|
||||
},
|
||||
];
|
||||
|
||||
/// Get model info by name, defaults to text-embedding-3-small
|
||||
#[must_use]
|
||||
pub fn get_openai_model_info(name: &str) -> &'static OpenAIModelInfo {
|
||||
OPENAI_MODELS
|
||||
.iter()
|
||||
.find(|m| m.name == name)
|
||||
.unwrap_or_else(|| OPENAI_MODELS.iter().find(|m| m.is_default).unwrap())
|
||||
}
|
||||
|
||||
/// Get the default model info
|
||||
#[must_use]
|
||||
pub fn default_openai_model_info() -> &'static OpenAIModelInfo {
|
||||
OPENAI_MODELS.iter().find(|m| m.is_default).unwrap()
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Configuration
|
||||
// ============================================================================
|
||||
|
||||
/// Configuration for OpenAI embedding provider
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OpenAIConfig {
|
||||
/// Model name (e.g., "text-embedding-3-small")
|
||||
pub model: String,
|
||||
/// Environment variable name for API key (default: "OPENAI_API_KEY")
|
||||
pub api_key_env: String,
|
||||
/// Custom API base URL (for Azure OpenAI, proxies, etc.)
|
||||
/// Default: "https://api.openai.com/v1"
|
||||
pub base_url: String,
|
||||
/// Request timeout in seconds
|
||||
pub timeout_secs: u64,
|
||||
/// Maximum retries on rate limit (429) errors
|
||||
pub max_retries: u32,
|
||||
/// Initial backoff in milliseconds for exponential retry
|
||||
pub initial_backoff_ms: u64,
|
||||
}
|
||||
|
||||
impl Default for OpenAIConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
model: "text-embedding-3-small".to_string(),
|
||||
api_key_env: "OPENAI_API_KEY".to_string(),
|
||||
base_url: "https://api.openai.com/v1".to_string(),
|
||||
timeout_secs: 30,
|
||||
max_retries: 3,
|
||||
initial_backoff_ms: 1000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl OpenAIConfig {
|
||||
/// Create config for text-embedding-3-small (default, fastest)
|
||||
#[must_use]
|
||||
pub fn small() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Create config for text-embedding-3-large (highest quality)
|
||||
#[must_use]
|
||||
pub fn large() -> Self {
|
||||
Self {
|
||||
model: "text-embedding-3-large".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Create config for text-embedding-ada-002 (legacy)
|
||||
#[must_use]
|
||||
pub fn ada() -> Self {
|
||||
Self {
|
||||
model: "text-embedding-ada-002".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Set custom base URL (for Azure OpenAI or proxies)
|
||||
#[must_use]
|
||||
pub fn with_base_url(mut self, url: impl Into<String>) -> Self {
|
||||
self.base_url = url.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set custom API key environment variable name
|
||||
#[must_use]
|
||||
pub fn with_api_key_env(mut self, env_var: impl Into<String>) -> Self {
|
||||
self.api_key_env = env_var.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set request timeout
|
||||
#[must_use]
|
||||
pub fn with_timeout(mut self, secs: u64) -> Self {
|
||||
self.timeout_secs = secs;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// API Request/Response Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct EmbeddingRequest<'a> {
|
||||
model: &'a str,
|
||||
input: Vec<&'a str>,
|
||||
encoding_format: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EmbeddingResponse {
|
||||
data: Vec<EmbeddingData>,
|
||||
#[allow(dead_code)]
|
||||
usage: Usage,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EmbeddingData {
|
||||
embedding: Vec<f32>,
|
||||
#[allow(dead_code)]
|
||||
index: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Usage {
|
||||
#[allow(dead_code)]
|
||||
prompt_tokens: usize,
|
||||
#[allow(dead_code)]
|
||||
total_tokens: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ApiError {
|
||||
error: ApiErrorDetail,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ApiErrorDetail {
|
||||
message: String,
|
||||
#[serde(rename = "type")]
|
||||
error_type: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// OpenAI Embedder
|
||||
// ============================================================================
|
||||
|
||||
/// OpenAI embedding provider
|
||||
///
|
||||
/// Generates embeddings using OpenAI's embedding API. Requires the `OPENAI_API_KEY`
|
||||
/// environment variable to be set (or a custom env var via config).
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
|
||||
/// use memvid_core::types::embedding::EmbeddingProvider;
|
||||
///
|
||||
/// let embedder = OpenAIEmbedder::new(OpenAIConfig::default())?;
|
||||
/// let embedding = embedder.embed_text("Hello, world!")?;
|
||||
/// ```
|
||||
pub struct OpenAIEmbedder {
|
||||
config: OpenAIConfig,
|
||||
model_info: &'static OpenAIModelInfo,
|
||||
client: Client,
|
||||
api_key: String,
|
||||
}
|
||||
|
||||
impl OpenAIEmbedder {
|
||||
/// Create a new OpenAI embedder
|
||||
///
|
||||
/// Reads the API key from the environment variable specified in config.
|
||||
/// Returns an error if the API key is not set.
|
||||
pub fn new(config: OpenAIConfig) -> Result<Self> {
|
||||
let api_key =
|
||||
std::env::var(&config.api_key_env).map_err(|_| MemvidError::EmbeddingFailed {
|
||||
reason: format!(
|
||||
"API key not found. Set the {} environment variable.",
|
||||
config.api_key_env
|
||||
)
|
||||
.into(),
|
||||
})?;
|
||||
|
||||
if api_key.is_empty() {
|
||||
return Err(MemvidError::EmbeddingFailed {
|
||||
reason: format!("{} environment variable is empty", config.api_key_env).into(),
|
||||
});
|
||||
}
|
||||
|
||||
let model_info = get_openai_model_info(&config.model);
|
||||
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(config.timeout_secs))
|
||||
.build()
|
||||
.map_err(|e| MemvidError::EmbeddingFailed {
|
||||
reason: format!("Failed to create HTTP client: {}", e).into(),
|
||||
})?;
|
||||
|
||||
tracing::info!(
|
||||
model = %model_info.name,
|
||||
dimension = model_info.dimension,
|
||||
"OpenAI embedder initialized"
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
model_info,
|
||||
client,
|
||||
api_key,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get model info
|
||||
#[must_use]
|
||||
pub fn model_info(&self) -> &'static OpenAIModelInfo {
|
||||
self.model_info
|
||||
}
|
||||
|
||||
/// Make an embedding request with retry logic
|
||||
fn request_embeddings(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
|
||||
let url = format!("{}/embeddings", self.config.base_url);
|
||||
|
||||
let request_body = EmbeddingRequest {
|
||||
model: &self.config.model,
|
||||
input: texts.to_vec(),
|
||||
encoding_format: "float",
|
||||
};
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
headers.insert(
|
||||
AUTHORIZATION,
|
||||
HeaderValue::from_str(&format!("Bearer {}", self.api_key)).map_err(|_| {
|
||||
MemvidError::EmbeddingFailed {
|
||||
reason: "Invalid API key format".into(),
|
||||
}
|
||||
})?,
|
||||
);
|
||||
|
||||
let mut backoff_ms = self.config.initial_backoff_ms;
|
||||
let mut last_error = None;
|
||||
|
||||
for attempt in 0..=self.config.max_retries {
|
||||
if attempt > 0 {
|
||||
tracing::warn!(
|
||||
attempt = attempt,
|
||||
backoff_ms = backoff_ms,
|
||||
"Retrying OpenAI request after rate limit"
|
||||
);
|
||||
std::thread::sleep(Duration::from_millis(backoff_ms));
|
||||
backoff_ms *= 2; // Exponential backoff
|
||||
}
|
||||
|
||||
let response = self
|
||||
.client
|
||||
.post(&url)
|
||||
.headers(headers.clone())
|
||||
.json(&request_body)
|
||||
.send();
|
||||
|
||||
match response {
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
|
||||
if status.is_success() {
|
||||
let embedding_response: EmbeddingResponse =
|
||||
resp.json().map_err(|e| MemvidError::EmbeddingFailed {
|
||||
reason: format!("Failed to parse response: {}", e).into(),
|
||||
})?;
|
||||
|
||||
// Sort by index to ensure correct order
|
||||
let mut data = embedding_response.data;
|
||||
data.sort_by_key(|d| d.index);
|
||||
|
||||
let embeddings: Vec<Vec<f32>> =
|
||||
data.into_iter().map(|d| d.embedding).collect();
|
||||
|
||||
tracing::debug!(
|
||||
texts = texts.len(),
|
||||
dimension = embeddings.first().map(|e| e.len()).unwrap_or(0),
|
||||
"Generated OpenAI embeddings"
|
||||
);
|
||||
|
||||
return Ok(embeddings);
|
||||
}
|
||||
|
||||
// Handle rate limiting
|
||||
if status.as_u16() == 429 {
|
||||
last_error = Some(MemvidError::EmbeddingFailed {
|
||||
reason: "Rate limit exceeded".into(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parse error response
|
||||
let error_text = resp.text().unwrap_or_default();
|
||||
let error_msg =
|
||||
if let Ok(api_error) = serde_json::from_str::<ApiError>(&error_text) {
|
||||
format!(
|
||||
"OpenAI API error ({}): {}",
|
||||
api_error.error.error_type.unwrap_or_default(),
|
||||
api_error.error.message
|
||||
)
|
||||
} else {
|
||||
format!("OpenAI API error ({}): {}", status, error_text)
|
||||
};
|
||||
|
||||
return Err(MemvidError::EmbeddingFailed {
|
||||
reason: error_msg.into(),
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
// Network error - might be transient
|
||||
last_error = Some(MemvidError::EmbeddingFailed {
|
||||
reason: format!("Request failed: {}", e).into(),
|
||||
});
|
||||
|
||||
if e.is_timeout() || e.is_connect() {
|
||||
continue; // Retry on timeout or connection errors
|
||||
}
|
||||
|
||||
return Err(last_error.unwrap());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// All retries exhausted
|
||||
Err(last_error.unwrap_or_else(|| MemvidError::EmbeddingFailed {
|
||||
reason: "Max retries exceeded".into(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for OpenAIEmbedder {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("OpenAIEmbedder")
|
||||
.field("config", &self.config)
|
||||
.field("model_info", &self.model_info)
|
||||
.field("api_key", &"[REDACTED]")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// EmbeddingProvider Implementation
|
||||
// ============================================================================
|
||||
|
||||
impl EmbeddingProvider for OpenAIEmbedder {
|
||||
fn kind(&self) -> &str {
|
||||
"openai"
|
||||
}
|
||||
|
||||
fn model(&self) -> &str {
|
||||
self.model_info.name
|
||||
}
|
||||
|
||||
fn dimension(&self) -> usize {
|
||||
self.model_info.dimension
|
||||
}
|
||||
|
||||
fn embed_text(&self, text: &str) -> Result<Vec<f32>> {
|
||||
let embeddings = self.request_embeddings(&[text])?;
|
||||
embeddings
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| MemvidError::EmbeddingFailed {
|
||||
reason: "No embedding returned".into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
|
||||
if texts.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// Split into chunks respecting API batch size limit
|
||||
let max_batch = self.model_info.max_batch_size;
|
||||
let mut all_embeddings = Vec::with_capacity(texts.len());
|
||||
|
||||
for chunk in texts.chunks(max_batch) {
|
||||
let chunk_embeddings = self.request_embeddings(chunk)?;
|
||||
all_embeddings.extend(chunk_embeddings);
|
||||
}
|
||||
|
||||
Ok(all_embeddings)
|
||||
}
|
||||
|
||||
fn is_ready(&self) -> bool {
|
||||
// We have an API key, so we're ready
|
||||
!self.api_key.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_model_registry() {
|
||||
assert_eq!(OPENAI_MODELS.len(), 3);
|
||||
|
||||
let default = default_openai_model_info();
|
||||
assert_eq!(default.name, "text-embedding-3-small");
|
||||
assert_eq!(default.dimension, 1536);
|
||||
assert!(default.is_default);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_model_info() {
|
||||
let small = get_openai_model_info("text-embedding-3-small");
|
||||
assert_eq!(small.dimension, 1536);
|
||||
|
||||
let large = get_openai_model_info("text-embedding-3-large");
|
||||
assert_eq!(large.dimension, 3072);
|
||||
|
||||
let ada = get_openai_model_info("text-embedding-ada-002");
|
||||
assert_eq!(ada.dimension, 1536);
|
||||
|
||||
// Unknown model should return default
|
||||
let unknown = get_openai_model_info("unknown-model");
|
||||
assert_eq!(unknown.name, "text-embedding-3-small");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_defaults() {
|
||||
let config = OpenAIConfig::default();
|
||||
assert_eq!(config.model, "text-embedding-3-small");
|
||||
assert_eq!(config.api_key_env, "OPENAI_API_KEY");
|
||||
assert_eq!(config.base_url, "https://api.openai.com/v1");
|
||||
assert_eq!(config.timeout_secs, 30);
|
||||
assert_eq!(config.max_retries, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_builders() {
|
||||
let small = OpenAIConfig::small();
|
||||
assert_eq!(small.model, "text-embedding-3-small");
|
||||
|
||||
let large = OpenAIConfig::large();
|
||||
assert_eq!(large.model, "text-embedding-3-large");
|
||||
|
||||
let ada = OpenAIConfig::ada();
|
||||
assert_eq!(ada.model, "text-embedding-ada-002");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_with_methods() {
|
||||
let config = OpenAIConfig::default()
|
||||
.with_base_url("https://custom.api.com")
|
||||
.with_api_key_env("MY_API_KEY")
|
||||
.with_timeout(60);
|
||||
|
||||
assert_eq!(config.base_url, "https://custom.api.com");
|
||||
assert_eq!(config.api_key_env, "MY_API_KEY");
|
||||
assert_eq!(config.timeout_secs, 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_embedder_requires_api_key() {
|
||||
// Use a custom env var name that doesn't exist
|
||||
let config = OpenAIConfig::default().with_api_key_env("NONEXISTENT_API_KEY_12345");
|
||||
let result = OpenAIEmbedder::new(config);
|
||||
assert!(result.is_err());
|
||||
|
||||
let err_msg = format!("{:?}", result.unwrap_err());
|
||||
assert!(err_msg.contains("NONEXISTENT_API_KEY_12345"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_embedder_validates_config() {
|
||||
// Test that config with custom env var works correctly
|
||||
let config = OpenAIConfig::default()
|
||||
.with_api_key_env("CUSTOM_KEY_VAR")
|
||||
.with_base_url("https://custom.openai.com/v1");
|
||||
|
||||
assert_eq!(config.api_key_env, "CUSTOM_KEY_VAR");
|
||||
assert_eq!(config.base_url, "https://custom.openai.com/v1");
|
||||
|
||||
// Creating embedder should fail since CUSTOM_KEY_VAR is not set
|
||||
let result = OpenAIEmbedder::new(config);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
/// Integration test - requires OPENAI_API_KEY to be set
|
||||
/// Run with: cargo test --features api_embed test_openai_integration -- --ignored
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn test_openai_integration() {
|
||||
let config = OpenAIConfig::default();
|
||||
let embedder = OpenAIEmbedder::new(config).expect("Failed to create embedder");
|
||||
|
||||
// Test single embedding
|
||||
let embedding = embedder
|
||||
.embed_text("Hello, world!")
|
||||
.expect("Failed to embed text");
|
||||
assert_eq!(embedding.len(), 1536);
|
||||
|
||||
// Test batch embedding
|
||||
let texts = vec!["Hello", "World", "Test"];
|
||||
let embeddings = embedder.embed_batch(&texts).expect("Failed to batch embed");
|
||||
assert_eq!(embeddings.len(), 3);
|
||||
assert!(embeddings.iter().all(|e| e.len() == 1536));
|
||||
|
||||
// Test EmbeddingProvider trait
|
||||
assert_eq!(embedder.kind(), "openai");
|
||||
assert_eq!(embedder.model(), "text-embedding-3-small");
|
||||
assert_eq!(embedder.dimension(), 1536);
|
||||
assert!(embedder.is_ready());
|
||||
}
|
||||
|
||||
/// Integration test with text-embedding-3-large
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn test_openai_large_integration() {
|
||||
let config = OpenAIConfig::large();
|
||||
let embedder = OpenAIEmbedder::new(config).expect("Failed to create embedder");
|
||||
|
||||
let embedding = embedder
|
||||
.embed_text("Test with large model")
|
||||
.expect("Failed to embed text");
|
||||
assert_eq!(embedding.len(), 3072);
|
||||
}
|
||||
}
|
||||
+1754
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
/// File magic for `.mv2` memories.
|
||||
pub const MAGIC: [u8; 4] = *b"MV2\0";
|
||||
/// Logical header size (4 KiB) reserving space for future upgrades.
|
||||
pub const HEADER_SIZE: usize = 4096;
|
||||
/// Magic bytes for the time index track.
|
||||
pub const TIME_INDEX_MAGIC: [u8; 4] = *b"MVTI";
|
||||
#[cfg(feature = "temporal_track")]
|
||||
/// Magic bytes for the temporal mentions track.
|
||||
pub const TEMPORAL_TRACK_MAGIC: [u8; 4] = *b"MVTN";
|
||||
#[cfg(feature = "temporal_track")]
|
||||
/// Initial on-disk version for the temporal mentions track header.
|
||||
pub const TEMPORAL_TRACK_VERSION: u16 = 1;
|
||||
/// Specification major version.
|
||||
pub const SPEC_MAJOR: u8 = 2;
|
||||
/// Specification minor version.
|
||||
pub const SPEC_MINOR: u8 = 1;
|
||||
/// Combined two-byte specification version encoded in headers.
|
||||
pub const SPEC_VERSION: u16 = ((SPEC_MAJOR as u16) << 8) | SPEC_MINOR as u16;
|
||||
/// Binary format schema version.
|
||||
pub const FORMAT_VERSION: u16 = 1;
|
||||
|
||||
/// Embedded WAL begins immediately after the fixed header.
|
||||
pub const WAL_OFFSET: u64 = HEADER_SIZE as u64;
|
||||
/// Minimal WAL size for empty/small memories (auto-grows on demand).
|
||||
pub const WAL_SIZE_TINY: u64 = 64 * 1024;
|
||||
/// WAL size tiers based on requested capacity (<100 MB).
|
||||
pub const WAL_SIZE_SMALL: u64 = 1024 * 1024;
|
||||
/// WAL size for memories under 1 GB.
|
||||
pub const WAL_SIZE_MEDIUM: u64 = 4 * 1024 * 1024;
|
||||
/// WAL size for memories under 10 GB.
|
||||
pub const WAL_SIZE_LARGE: u64 = 16 * 1024 * 1024;
|
||||
/// WAL size for larger memories.
|
||||
pub const WAL_SIZE_XLARGE: u64 = 64 * 1024 * 1024;
|
||||
/// Trigger checkpoints when the WAL exceeds 75 % occupancy.
|
||||
pub const WAL_CHECKPOINT_THRESHOLD: f64 = 0.75;
|
||||
/// Additional checkpoint every N transactions (PRD default).
|
||||
pub const WAL_CHECKPOINT_PERIOD: u64 = 1_000;
|
||||
|
||||
/// Memvid's Ed25519 public key for verifying signed tickets.
|
||||
/// This key is used to verify that tickets were issued by the official Memvid control plane.
|
||||
/// The corresponding private key is held securely on the Memvid dashboard.
|
||||
pub const MEMVID_TICKET_PUBKEY: &str = "DFKNhP/yO5i1b9aKL+aHeBaGunz9sMfOF736fzYws4Q=";
|
||||
@@ -0,0 +1,146 @@
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use atomic_write_file::AtomicWriteFile;
|
||||
use zeroize::Zeroize;
|
||||
|
||||
use crate::encryption::capsule_stream::{lock_file_stream, unlock_file_stream};
|
||||
use crate::encryption::crypto::{decrypt, derive_key};
|
||||
use crate::encryption::error::EncryptionError;
|
||||
use crate::encryption::types::Mv2eHeader;
|
||||
|
||||
/// Lock (encrypt) an `.mv2` file into a `.mv2e` capsule.
|
||||
pub fn lock_file(
|
||||
input: impl AsRef<Path>,
|
||||
output: Option<&Path>,
|
||||
password: &[u8],
|
||||
) -> Result<PathBuf, EncryptionError> {
|
||||
lock_file_stream(input, output, password)
|
||||
}
|
||||
|
||||
/// Unlock (decrypt) an `.mv2e` capsule into an `.mv2` file.
|
||||
pub fn unlock_file(
|
||||
input: impl AsRef<Path>,
|
||||
output: Option<&Path>,
|
||||
password: &[u8],
|
||||
) -> Result<PathBuf, EncryptionError> {
|
||||
let input = input.as_ref();
|
||||
|
||||
let mut file = File::open(input).map_err(|source| EncryptionError::Io {
|
||||
source,
|
||||
path: Some(input.to_path_buf()),
|
||||
})?;
|
||||
|
||||
let mut header_bytes = [0u8; Mv2eHeader::SIZE];
|
||||
file.read_exact(&mut header_bytes)
|
||||
.map_err(|source| EncryptionError::Io {
|
||||
source,
|
||||
path: Some(input.to_path_buf()),
|
||||
})?;
|
||||
let header = Mv2eHeader::decode(&header_bytes)?;
|
||||
|
||||
if header.reserved[0] == 0x01 {
|
||||
unlock_file_stream(input, output, password)
|
||||
} else {
|
||||
unlock_file_oneshot(input, output, password, header)
|
||||
}
|
||||
}
|
||||
|
||||
fn unlock_file_oneshot(
|
||||
input: &Path,
|
||||
output: Option<&Path>,
|
||||
password: &[u8],
|
||||
header: Mv2eHeader,
|
||||
) -> Result<PathBuf, EncryptionError> {
|
||||
let mut file = File::open(input).map_err(|source| EncryptionError::Io {
|
||||
source,
|
||||
path: Some(input.to_path_buf()),
|
||||
})?;
|
||||
|
||||
file.seek(SeekFrom::Start(Mv2eHeader::SIZE as u64))
|
||||
.map_err(|source| EncryptionError::Io {
|
||||
source,
|
||||
path: Some(input.to_path_buf()),
|
||||
})?;
|
||||
|
||||
let mut ciphertext = Vec::new();
|
||||
file.read_to_end(&mut ciphertext)
|
||||
.map_err(|source| EncryptionError::Io {
|
||||
source,
|
||||
path: Some(input.to_path_buf()),
|
||||
})?;
|
||||
|
||||
let mut key = derive_key(password, &header.salt)?;
|
||||
let plaintext = decrypt(&ciphertext, &key, &header.nonce)?;
|
||||
key.zeroize();
|
||||
|
||||
if plaintext.len() as u64 != header.original_size {
|
||||
return Err(EncryptionError::SizeMismatch {
|
||||
expected: header.original_size,
|
||||
actual: plaintext.len() as u64,
|
||||
});
|
||||
}
|
||||
|
||||
validate_mv2_bytes(&plaintext)?;
|
||||
|
||||
let output_path = output
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| input.with_extension("mv2"));
|
||||
|
||||
write_atomic(&output_path, |writer| -> Result<(), EncryptionError> {
|
||||
writer.write_all(&plaintext)?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
Ok(output_path)
|
||||
}
|
||||
|
||||
pub fn validate_mv2_file(path: &Path) -> Result<(), EncryptionError> {
|
||||
let mut file = File::open(path).map_err(|source| EncryptionError::Io {
|
||||
source,
|
||||
path: Some(path.to_path_buf()),
|
||||
})?;
|
||||
|
||||
let mut magic = [0u8; 4];
|
||||
file.read_exact(&mut magic)
|
||||
.map_err(|source| EncryptionError::Io {
|
||||
source,
|
||||
path: Some(path.to_path_buf()),
|
||||
})?;
|
||||
|
||||
// Plain `.mv2` files start with "MV2\0".
|
||||
if magic != *b"MV2\0" {
|
||||
return Err(EncryptionError::NotMv2File {
|
||||
path: path.to_path_buf(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_mv2_bytes(bytes: &[u8]) -> Result<(), EncryptionError> {
|
||||
if bytes.len() < 4 || &bytes[0..4] != b"MV2\0" {
|
||||
return Err(EncryptionError::CorruptedDecryption);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn write_atomic<F, E>(path: &Path, write_fn: F) -> Result<(), E>
|
||||
where
|
||||
F: FnOnce(&mut File) -> Result<(), E>,
|
||||
E: From<std::io::Error>,
|
||||
{
|
||||
let mut options = AtomicWriteFile::options();
|
||||
options.read(false);
|
||||
let mut atomic = options.open(path)?;
|
||||
|
||||
let file = atomic.as_file_mut();
|
||||
file.set_len(0)?;
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
write_fn(file)?;
|
||||
file.flush()?;
|
||||
file.sync_all()?;
|
||||
atomic.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
use rand::RngCore;
|
||||
use rand::rngs::OsRng;
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, BufWriter, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use zeroize::Zeroize;
|
||||
|
||||
use crate::encryption::capsule::{validate_mv2_file, write_atomic};
|
||||
use crate::encryption::constants::{MV2E_MAGIC, MV2E_VERSION, NONCE_SIZE, SALT_SIZE};
|
||||
use crate::encryption::crypto::{decrypt, derive_key, encrypt};
|
||||
use crate::encryption::error::EncryptionError;
|
||||
use crate::encryption::types::{CipherAlgorithm, KdfAlgorithm, Mv2eHeader};
|
||||
|
||||
const CHUNK_SIZE: usize = 1024 * 1024;
|
||||
|
||||
// format: [header][len0][chunk0][len1][chunk1]...
|
||||
// reserved[0] == 0x01 => streaming framed format
|
||||
|
||||
pub fn lock_file_stream(
|
||||
input: impl AsRef<Path>,
|
||||
output: Option<&Path>,
|
||||
password: &[u8],
|
||||
) -> Result<PathBuf, EncryptionError> {
|
||||
let input = input.as_ref();
|
||||
validate_mv2_file(input)?;
|
||||
|
||||
let metadata = std::fs::metadata(input)?;
|
||||
|
||||
let mut salt = [0u8; SALT_SIZE];
|
||||
let mut base_nonce = [0u8; NONCE_SIZE];
|
||||
OsRng.fill_bytes(&mut salt);
|
||||
OsRng.fill_bytes(&mut base_nonce);
|
||||
|
||||
let mut key = derive_key(password, &salt)?;
|
||||
|
||||
let header = Mv2eHeader {
|
||||
magic: MV2E_MAGIC,
|
||||
version: MV2E_VERSION,
|
||||
kdf_algorithm: KdfAlgorithm::Argon2id,
|
||||
cipher_algorithm: CipherAlgorithm::Aes256Gcm,
|
||||
salt,
|
||||
nonce: base_nonce,
|
||||
original_size: metadata.len(),
|
||||
reserved: [0x01, 0, 0, 0],
|
||||
};
|
||||
|
||||
let output_path = output
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| input.with_extension("mv2e"));
|
||||
|
||||
let input_file = File::open(input)?;
|
||||
let mut reader = BufReader::new(input_file);
|
||||
|
||||
write_atomic(&output_path, |file| -> Result<(), EncryptionError> {
|
||||
let mut writer = BufWriter::new(file);
|
||||
writer.write_all(&header.encode())?;
|
||||
|
||||
let mut buffer = vec![0u8; CHUNK_SIZE];
|
||||
let mut chunk_index: u64 = 0;
|
||||
|
||||
loop {
|
||||
let n = reader.read(&mut buffer)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let mut nonce = base_nonce;
|
||||
nonce[NONCE_SIZE - 8..].copy_from_slice(&chunk_index.to_be_bytes());
|
||||
|
||||
let ciphertext = encrypt(&buffer[..n], &key, &nonce)?;
|
||||
|
||||
let chunk_len = ciphertext.len() as u32;
|
||||
writer.write_all(&chunk_len.to_le_bytes())?;
|
||||
writer.write_all(&ciphertext)?;
|
||||
|
||||
chunk_index += 1;
|
||||
}
|
||||
|
||||
writer.flush()?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
key.zeroize();
|
||||
Ok(output_path)
|
||||
}
|
||||
|
||||
pub fn unlock_file_stream(
|
||||
input: impl AsRef<Path>,
|
||||
output: Option<&Path>,
|
||||
password: &[u8],
|
||||
) -> Result<PathBuf, EncryptionError> {
|
||||
let input = input.as_ref();
|
||||
|
||||
let input_file = File::open(input)?;
|
||||
let mut reader = BufReader::new(input_file);
|
||||
|
||||
let mut header_bytes = [0u8; Mv2eHeader::SIZE];
|
||||
reader.read_exact(&mut header_bytes)?;
|
||||
|
||||
let header = Mv2eHeader::decode(&header_bytes)?;
|
||||
let mut key = derive_key(password, &header.salt)?;
|
||||
|
||||
let output_path = output
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| input.with_extension("mv2"));
|
||||
|
||||
write_atomic(&output_path, |file| -> Result<(), EncryptionError> {
|
||||
let mut writer = BufWriter::new(file);
|
||||
let mut chunk_index: u64 = 0;
|
||||
|
||||
loop {
|
||||
let mut len_bytes = [0u8; 4];
|
||||
match reader.read_exact(&mut len_bytes) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
let chunk_len = u32::from_le_bytes(len_bytes) as usize;
|
||||
|
||||
let mut ciphertext = vec![0u8; chunk_len];
|
||||
reader.read_exact(&mut ciphertext)?;
|
||||
|
||||
let mut nonce = header.nonce;
|
||||
nonce[NONCE_SIZE - 8..].copy_from_slice(&chunk_index.to_be_bytes());
|
||||
|
||||
let plaintext = decrypt(&ciphertext, &key, &nonce)?;
|
||||
writer.write_all(&plaintext)?;
|
||||
|
||||
chunk_index += 1;
|
||||
}
|
||||
|
||||
writer.flush()?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
key.zeroize();
|
||||
Ok(output_path)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Constants for the `.mv2e` encrypted capsule format.
|
||||
|
||||
/// Magic bytes identifying an encrypted capsule file.
|
||||
pub const MV2E_MAGIC: [u8; 4] = *b"MV2E";
|
||||
|
||||
/// Current `.mv2e` format version.
|
||||
pub const MV2E_VERSION: u16 = 1;
|
||||
|
||||
/// Fixed header size for `.mv2e`.
|
||||
pub const MV2E_HEADER_SIZE: usize = 64;
|
||||
|
||||
/// KDF algorithm identifiers.
|
||||
pub const KDF_ARGON2ID: u8 = 1;
|
||||
|
||||
/// Cipher algorithm identifiers.
|
||||
pub const CIPHER_AES_256_GCM: u8 = 1;
|
||||
|
||||
/// Cryptographic parameter sizes.
|
||||
pub const SALT_SIZE: usize = 32;
|
||||
pub const NONCE_SIZE: usize = 12;
|
||||
pub const TAG_SIZE: usize = 16;
|
||||
pub const KEY_SIZE: usize = 32;
|
||||
|
||||
/// Argon2id parameters (OWASP 2024 recommendations).
|
||||
pub const ARGON2_MEMORY_KIB: u32 = 64 * 1024; // 64 MiB
|
||||
pub const ARGON2_ITERATIONS: u32 = 3;
|
||||
pub const ARGON2_PARALLELISM: u32 = 4;
|
||||
@@ -0,0 +1,64 @@
|
||||
use aes_gcm::aead::Aead;
|
||||
use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
|
||||
use argon2::{Algorithm, Argon2, Params, Version};
|
||||
|
||||
use crate::encryption::constants::{
|
||||
ARGON2_ITERATIONS, ARGON2_MEMORY_KIB, ARGON2_PARALLELISM, KEY_SIZE, NONCE_SIZE, SALT_SIZE,
|
||||
};
|
||||
use crate::encryption::error::EncryptionError;
|
||||
|
||||
pub fn derive_key(
|
||||
password: &[u8],
|
||||
salt: &[u8; SALT_SIZE],
|
||||
) -> Result<[u8; KEY_SIZE], EncryptionError> {
|
||||
let params = Params::new(
|
||||
ARGON2_MEMORY_KIB,
|
||||
ARGON2_ITERATIONS,
|
||||
ARGON2_PARALLELISM,
|
||||
Some(KEY_SIZE),
|
||||
)
|
||||
.map_err(|e| EncryptionError::KeyDerivation {
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let argon2 = Argon2::new(Algorithm::Argon2id, Version::V0x13, params);
|
||||
|
||||
let mut key = [0u8; KEY_SIZE];
|
||||
argon2
|
||||
.hash_password_into(password, salt, &mut key)
|
||||
.map_err(|e| EncryptionError::KeyDerivation {
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
pub fn encrypt(
|
||||
plaintext: &[u8],
|
||||
key: &[u8; KEY_SIZE],
|
||||
nonce: &[u8; NONCE_SIZE],
|
||||
) -> Result<Vec<u8>, EncryptionError> {
|
||||
let cipher = Aes256Gcm::new_from_slice(key).map_err(|e| EncryptionError::CipherInit {
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
cipher
|
||||
.encrypt(Nonce::from_slice(nonce), plaintext)
|
||||
.map_err(|e| EncryptionError::Encryption {
|
||||
reason: e.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn decrypt(
|
||||
ciphertext: &[u8],
|
||||
key: &[u8; KEY_SIZE],
|
||||
nonce: &[u8; NONCE_SIZE],
|
||||
) -> Result<Vec<u8>, EncryptionError> {
|
||||
let cipher = Aes256Gcm::new_from_slice(key).map_err(|e| EncryptionError::CipherInit {
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
cipher
|
||||
.decrypt(Nonce::from_slice(nonce), ciphertext)
|
||||
.map_err(|e| EncryptionError::Decryption {
|
||||
reason: e.to_string(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum EncryptionError {
|
||||
#[error("I/O error: {source}")]
|
||||
Io {
|
||||
source: std::io::Error,
|
||||
path: Option<PathBuf>,
|
||||
},
|
||||
|
||||
#[error("Invalid magic header: expected {expected:?}, found {found:?}")]
|
||||
InvalidMagic { expected: [u8; 4], found: [u8; 4] },
|
||||
|
||||
#[error("Unsupported encryption format version: {version}")]
|
||||
UnsupportedVersion { version: u16 },
|
||||
|
||||
#[error("Unsupported key derivation function: {id}")]
|
||||
UnsupportedKdf { id: u8 },
|
||||
|
||||
#[error("Unsupported cipher algorithm: {id}")]
|
||||
UnsupportedCipher { id: u8 },
|
||||
|
||||
#[error("Key derivation failed: {reason}")]
|
||||
KeyDerivation { reason: String },
|
||||
|
||||
#[error("Cipher initialization failed: {reason}")]
|
||||
CipherInit { reason: String },
|
||||
|
||||
#[error("Encryption failed: {reason}")]
|
||||
Encryption { reason: String },
|
||||
|
||||
#[error("Decryption failed - invalid password or corrupted file")]
|
||||
Decryption { reason: String },
|
||||
|
||||
#[error("Size mismatch: expected {expected} bytes, got {actual}")]
|
||||
SizeMismatch { expected: u64, actual: u64 },
|
||||
|
||||
#[error("Not an MV2 file: {path}")]
|
||||
NotMv2File { path: PathBuf },
|
||||
|
||||
#[error("Corrupted decryption - output is not a valid MV2 file")]
|
||||
CorruptedDecryption,
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for EncryptionError {
|
||||
fn from(source: std::io::Error) -> Self {
|
||||
EncryptionError::Io { source, path: None }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//! Password-based encryption capsules for `.mv2` files (`.mv2e`).
|
||||
//!
|
||||
//! This module is feature-gated (`encryption`) to keep the default memvid-core
|
||||
//! binary size small and avoid pulling crypto dependencies into users that don't
|
||||
//! need encrypted-at-rest capsules.
|
||||
|
||||
mod capsule;
|
||||
mod capsule_stream;
|
||||
mod constants;
|
||||
mod crypto;
|
||||
mod error;
|
||||
mod types;
|
||||
|
||||
pub use capsule::{lock_file, unlock_file};
|
||||
pub use capsule_stream::{lock_file_stream, unlock_file_stream};
|
||||
pub use constants::*;
|
||||
pub use error::EncryptionError;
|
||||
pub use types::{CipherAlgorithm, KdfAlgorithm, Mv2eHeader};
|
||||
@@ -0,0 +1,96 @@
|
||||
use crate::encryption::constants::{
|
||||
CIPHER_AES_256_GCM, KDF_ARGON2ID, MV2E_HEADER_SIZE, MV2E_MAGIC, MV2E_VERSION, NONCE_SIZE,
|
||||
SALT_SIZE,
|
||||
};
|
||||
use crate::encryption::error::EncryptionError;
|
||||
|
||||
/// MV2E file header (fixed-size, 64 bytes).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Mv2eHeader {
|
||||
pub magic: [u8; 4],
|
||||
pub version: u16,
|
||||
pub kdf_algorithm: KdfAlgorithm,
|
||||
pub cipher_algorithm: CipherAlgorithm,
|
||||
pub salt: [u8; SALT_SIZE],
|
||||
pub nonce: [u8; NONCE_SIZE],
|
||||
pub original_size: u64,
|
||||
pub reserved: [u8; 4],
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum KdfAlgorithm {
|
||||
Argon2id = KDF_ARGON2ID,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum CipherAlgorithm {
|
||||
Aes256Gcm = CIPHER_AES_256_GCM,
|
||||
}
|
||||
|
||||
impl Mv2eHeader {
|
||||
pub const SIZE: usize = MV2E_HEADER_SIZE;
|
||||
|
||||
#[must_use]
|
||||
pub fn encode(&self) -> [u8; Self::SIZE] {
|
||||
let mut buf = [0u8; Self::SIZE];
|
||||
buf[0..4].copy_from_slice(&self.magic);
|
||||
buf[4..6].copy_from_slice(&self.version.to_le_bytes());
|
||||
buf[6] = self.kdf_algorithm as u8;
|
||||
buf[7] = self.cipher_algorithm as u8;
|
||||
buf[8..40].copy_from_slice(&self.salt);
|
||||
buf[40..52].copy_from_slice(&self.nonce);
|
||||
buf[52..60].copy_from_slice(&self.original_size.to_le_bytes());
|
||||
buf[60..64].copy_from_slice(&self.reserved);
|
||||
buf
|
||||
}
|
||||
|
||||
pub fn decode(bytes: &[u8; Self::SIZE]) -> Result<Self, EncryptionError> {
|
||||
let magic = [bytes[0], bytes[1], bytes[2], bytes[3]];
|
||||
if magic != MV2E_MAGIC {
|
||||
return Err(EncryptionError::InvalidMagic {
|
||||
expected: MV2E_MAGIC,
|
||||
found: magic,
|
||||
});
|
||||
}
|
||||
|
||||
let version = u16::from_le_bytes([bytes[4], bytes[5]]);
|
||||
if version != MV2E_VERSION {
|
||||
return Err(EncryptionError::UnsupportedVersion { version });
|
||||
}
|
||||
|
||||
let kdf_algorithm = match bytes[6] {
|
||||
KDF_ARGON2ID => KdfAlgorithm::Argon2id,
|
||||
other => return Err(EncryptionError::UnsupportedKdf { id: other }),
|
||||
};
|
||||
|
||||
let cipher_algorithm = match bytes[7] {
|
||||
CIPHER_AES_256_GCM => CipherAlgorithm::Aes256Gcm,
|
||||
other => return Err(EncryptionError::UnsupportedCipher { id: other }),
|
||||
};
|
||||
|
||||
let mut salt = [0u8; SALT_SIZE];
|
||||
salt.copy_from_slice(&bytes[8..40]);
|
||||
|
||||
let mut nonce = [0u8; NONCE_SIZE];
|
||||
nonce.copy_from_slice(&bytes[40..52]);
|
||||
|
||||
let original_size = u64::from_le_bytes([
|
||||
bytes[52], bytes[53], bytes[54], bytes[55], bytes[56], bytes[57], bytes[58], bytes[59],
|
||||
]);
|
||||
|
||||
let reserved = [bytes[60], bytes[61], bytes[62], bytes[63]];
|
||||
|
||||
Ok(Self {
|
||||
magic,
|
||||
version,
|
||||
kdf_algorithm,
|
||||
cipher_algorithm,
|
||||
salt,
|
||||
nonce,
|
||||
original_size,
|
||||
reserved,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
//! Enrichment engine trait and context types.
|
||||
//!
|
||||
//! The `EnrichmentEngine` trait defines the interface that all enrichment
|
||||
//! engines must implement. Each engine processes frames and produces
|
||||
//! structured memory cards.
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::types::{FrameId, MemoryCard};
|
||||
|
||||
/// Context provided to enrichment engines during processing.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EnrichmentContext {
|
||||
/// The frame ID being processed.
|
||||
pub frame_id: FrameId,
|
||||
/// The frame's URI (e.g., "<mv2://session-1/msg-5>").
|
||||
pub uri: String,
|
||||
/// The frame's text content.
|
||||
pub text: String,
|
||||
/// The frame's title (if any).
|
||||
pub title: Option<String>,
|
||||
/// The frame's timestamp (Unix seconds).
|
||||
pub timestamp: i64,
|
||||
/// Optional metadata from the frame.
|
||||
pub metadata: Option<String>,
|
||||
}
|
||||
|
||||
impl EnrichmentContext {
|
||||
/// Create a new enrichment context.
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
frame_id: FrameId,
|
||||
uri: String,
|
||||
text: String,
|
||||
title: Option<String>,
|
||||
timestamp: i64,
|
||||
metadata: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
frame_id,
|
||||
uri,
|
||||
text,
|
||||
title,
|
||||
timestamp,
|
||||
metadata,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of running an enrichment engine on a frame.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct EnrichmentResult {
|
||||
/// Memory cards extracted from the frame.
|
||||
pub cards: Vec<MemoryCard>,
|
||||
/// Whether the engine successfully processed the frame.
|
||||
/// Even if no cards were extracted, this can be true.
|
||||
pub success: bool,
|
||||
/// Optional error message if processing failed.
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
impl EnrichmentResult {
|
||||
/// Create a successful result with cards.
|
||||
#[must_use]
|
||||
pub fn success(cards: Vec<MemoryCard>) -> Self {
|
||||
Self {
|
||||
cards,
|
||||
success: true,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create an empty successful result (no cards extracted).
|
||||
#[must_use]
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
cards: Vec::new(),
|
||||
success: true,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a failed result.
|
||||
#[must_use]
|
||||
pub fn failed(error: impl Into<String>) -> Self {
|
||||
Self {
|
||||
cards: Vec::new(),
|
||||
success: false,
|
||||
error: Some(error.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait for enrichment engines that process frames and extract memory cards.
|
||||
///
|
||||
/// Engines are identified by a kind (e.g., "rules", "llm:phi-3.5-mini") and
|
||||
/// a version string. The combination allows tracking which frames have been
|
||||
/// processed by which engine versions.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// use memvid_core::enrich::{EnrichmentEngine, EnrichmentContext, EnrichmentResult};
|
||||
///
|
||||
/// struct MyEngine;
|
||||
///
|
||||
/// impl EnrichmentEngine for MyEngine {
|
||||
/// fn kind(&self) -> &str { "my-engine" }
|
||||
/// fn version(&self) -> &str { "1.0.0" }
|
||||
///
|
||||
/// fn enrich(&self, ctx: &EnrichmentContext) -> EnrichmentResult {
|
||||
/// // Extract memory cards from ctx.text
|
||||
/// EnrichmentResult::empty()
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
pub trait EnrichmentEngine: Send + Sync {
|
||||
/// Return the engine kind identifier (e.g., "rules", "llm:phi-3.5-mini").
|
||||
fn kind(&self) -> &str;
|
||||
|
||||
/// Return the engine version string (e.g., "1.0.0").
|
||||
fn version(&self) -> &str;
|
||||
|
||||
/// Process a frame and extract memory cards.
|
||||
///
|
||||
/// The engine receives the frame's text content and metadata via the
|
||||
/// `EnrichmentContext` and should return any extracted memory cards.
|
||||
fn enrich(&self, ctx: &EnrichmentContext) -> EnrichmentResult;
|
||||
|
||||
/// Initialize the engine (e.g., load models).
|
||||
///
|
||||
/// This is called once before processing begins. Engines that need
|
||||
/// to load models or other resources should do so here.
|
||||
fn init(&mut self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if the engine is ready for processing.
|
||||
///
|
||||
/// Returns true if `init()` has been called and the engine is ready.
|
||||
fn is_ready(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
struct TestEngine;
|
||||
|
||||
impl EnrichmentEngine for TestEngine {
|
||||
fn kind(&self) -> &'static str {
|
||||
"test"
|
||||
}
|
||||
fn version(&self) -> &'static str {
|
||||
"1.0.0"
|
||||
}
|
||||
fn enrich(&self, _ctx: &EnrichmentContext) -> EnrichmentResult {
|
||||
EnrichmentResult::empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_enrichment_context() {
|
||||
let ctx = EnrichmentContext::new(
|
||||
42,
|
||||
"mv2://test/msg-1".to_string(),
|
||||
"Hello, I work at Anthropic.".to_string(),
|
||||
Some("Test".to_string()),
|
||||
1700000000,
|
||||
None,
|
||||
);
|
||||
assert_eq!(ctx.frame_id, 42);
|
||||
assert_eq!(ctx.uri, "mv2://test/msg-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_enrichment_result() {
|
||||
let success = EnrichmentResult::success(vec![]);
|
||||
assert!(success.success);
|
||||
assert!(success.error.is_none());
|
||||
|
||||
let empty = EnrichmentResult::empty();
|
||||
assert!(empty.success);
|
||||
assert!(empty.cards.is_empty());
|
||||
|
||||
let failed = EnrichmentResult::failed("test error");
|
||||
assert!(!failed.success);
|
||||
assert_eq!(failed.error, Some("test error".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_engine_trait() {
|
||||
let engine = TestEngine;
|
||||
assert_eq!(engine.kind(), "test");
|
||||
assert_eq!(engine.version(), "1.0.0");
|
||||
assert!(engine.is_ready());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
//! Enrichment engine framework for extracting memory cards from frames.
|
||||
//!
|
||||
//! This module provides the trait and utilities for building enrichment engines
|
||||
//! that process MV2 frames and extract structured memory cards.
|
||||
|
||||
pub mod engine;
|
||||
pub mod rules;
|
||||
|
||||
pub use engine::{EnrichmentContext, EnrichmentEngine, EnrichmentResult};
|
||||
pub use rules::RulesEngine;
|
||||
+1244
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,594 @@
|
||||
//! Background enrichment worker for progressive ingestion.
|
||||
//!
|
||||
//! Processes frames in the enrichment queue asynchronously:
|
||||
//! - Re-extracts full text for skim extractions
|
||||
//! - Generates embeddings with batching and checkpointing
|
||||
//! - Updates Tantivy index with enriched content
|
||||
//! - Marks frames as Enriched when complete
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::types::{EnrichmentTask, FrameId, VecEmbedder};
|
||||
|
||||
/// Configuration for the enrichment worker.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EnrichmentWorkerConfig {
|
||||
/// Batch size for embedding generation.
|
||||
pub embedding_batch_size: usize,
|
||||
/// Checkpoint interval (persist progress every N embeddings).
|
||||
pub checkpoint_interval: usize,
|
||||
/// Delay between processing tasks (to avoid blocking writers).
|
||||
pub task_delay_ms: u64,
|
||||
/// Maximum time to spend on a single task before yielding.
|
||||
pub max_task_time_ms: u64,
|
||||
}
|
||||
|
||||
impl Default for EnrichmentWorkerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
embedding_batch_size: 32,
|
||||
checkpoint_interval: 100,
|
||||
task_delay_ms: 50,
|
||||
max_task_time_ms: 5000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Statistics for the enrichment worker.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct EnrichmentWorkerStats {
|
||||
/// Total frames processed.
|
||||
pub frames_processed: u64,
|
||||
/// Total embeddings generated.
|
||||
pub embeddings_generated: u64,
|
||||
/// Total re-extractions performed.
|
||||
pub re_extractions: u64,
|
||||
/// Total errors encountered.
|
||||
pub errors: u64,
|
||||
/// Current queue depth.
|
||||
pub queue_depth: usize,
|
||||
/// Whether worker is currently running.
|
||||
pub is_running: bool,
|
||||
}
|
||||
|
||||
/// Handle for controlling the background enrichment worker.
|
||||
pub struct EnrichmentWorkerHandle {
|
||||
/// Signal to stop the worker.
|
||||
stop_signal: Arc<AtomicBool>,
|
||||
/// Counter for frames processed.
|
||||
frames_processed: Arc<AtomicU64>,
|
||||
/// Counter for embeddings generated.
|
||||
embeddings_generated: Arc<AtomicU64>,
|
||||
/// Counter for re-extractions.
|
||||
re_extractions: Arc<AtomicU64>,
|
||||
/// Counter for errors.
|
||||
errors: Arc<AtomicU64>,
|
||||
/// Running state.
|
||||
is_running: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl EnrichmentWorkerHandle {
|
||||
/// Create a new worker handle.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
stop_signal: Arc::new(AtomicBool::new(false)),
|
||||
frames_processed: Arc::new(AtomicU64::new(0)),
|
||||
embeddings_generated: Arc::new(AtomicU64::new(0)),
|
||||
re_extractions: Arc::new(AtomicU64::new(0)),
|
||||
errors: Arc::new(AtomicU64::new(0)),
|
||||
is_running: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Signal the worker to stop.
|
||||
pub fn stop(&self) {
|
||||
self.stop_signal.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// Check if stop was requested.
|
||||
#[must_use]
|
||||
pub fn should_stop(&self) -> bool {
|
||||
self.stop_signal.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Check if worker is currently running.
|
||||
#[must_use]
|
||||
pub fn is_running(&self) -> bool {
|
||||
self.is_running.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Get current statistics.
|
||||
#[must_use]
|
||||
pub fn stats(&self) -> EnrichmentWorkerStats {
|
||||
EnrichmentWorkerStats {
|
||||
frames_processed: self.frames_processed.load(Ordering::Relaxed),
|
||||
embeddings_generated: self.embeddings_generated.load(Ordering::Relaxed),
|
||||
re_extractions: self.re_extractions.load(Ordering::Relaxed),
|
||||
errors: self.errors.load(Ordering::Relaxed),
|
||||
queue_depth: 0, // Will be updated by caller
|
||||
is_running: self.is_running.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Increment frames processed counter.
|
||||
pub(crate) fn inc_frames_processed(&self) {
|
||||
self.frames_processed.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Increment embeddings generated counter.
|
||||
pub(crate) fn inc_embeddings(&self, count: u64) {
|
||||
self.embeddings_generated
|
||||
.fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Increment re-extractions counter.
|
||||
pub(crate) fn inc_re_extractions(&self) {
|
||||
self.re_extractions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Increment errors counter.
|
||||
pub(crate) fn inc_errors(&self) {
|
||||
self.errors.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Set running state.
|
||||
pub(crate) fn set_running(&self, running: bool) {
|
||||
self.is_running.store(running, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// Clone the handle for sharing with the worker thread.
|
||||
#[must_use]
|
||||
pub fn clone_handle(&self) -> Self {
|
||||
Self {
|
||||
stop_signal: Arc::clone(&self.stop_signal),
|
||||
frames_processed: Arc::clone(&self.frames_processed),
|
||||
embeddings_generated: Arc::clone(&self.embeddings_generated),
|
||||
re_extractions: Arc::clone(&self.re_extractions),
|
||||
errors: Arc::clone(&self.errors),
|
||||
is_running: Arc::clone(&self.is_running),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EnrichmentWorkerHandle {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of processing a single enrichment task.
|
||||
#[derive(Debug)]
|
||||
pub struct TaskResult {
|
||||
/// Frame ID that was processed.
|
||||
pub frame_id: FrameId,
|
||||
/// Whether full re-extraction was performed.
|
||||
pub re_extracted: bool,
|
||||
/// Number of embeddings generated.
|
||||
pub embeddings_generated: usize,
|
||||
/// Time spent processing.
|
||||
pub elapsed_ms: u64,
|
||||
/// Error if processing failed.
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Batched embedding generator for efficient embedding creation.
|
||||
///
|
||||
/// Collects text chunks and generates embeddings in batches to minimize
|
||||
/// API calls and improve throughput.
|
||||
pub struct EmbeddingBatcher<E: VecEmbedder> {
|
||||
/// The embedder to use for generating embeddings.
|
||||
embedder: E,
|
||||
/// Batch size for embedding generation.
|
||||
batch_size: usize,
|
||||
/// Pending texts to embed.
|
||||
pending_texts: Vec<(FrameId, String)>,
|
||||
/// Generated embeddings ready to store.
|
||||
ready_embeddings: Vec<(FrameId, Vec<f32>)>,
|
||||
}
|
||||
|
||||
impl<E: VecEmbedder> EmbeddingBatcher<E> {
|
||||
/// Create a new embedding batcher.
|
||||
pub fn new(embedder: E, batch_size: usize) -> Self {
|
||||
Self {
|
||||
embedder,
|
||||
batch_size: batch_size.max(1),
|
||||
pending_texts: Vec::new(),
|
||||
ready_embeddings: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a frame's text for embedding.
|
||||
pub fn add(&mut self, frame_id: FrameId, text: String) {
|
||||
self.pending_texts.push((frame_id, text));
|
||||
}
|
||||
|
||||
/// Get the number of pending texts.
|
||||
pub fn pending_count(&self) -> usize {
|
||||
self.pending_texts.len()
|
||||
}
|
||||
|
||||
/// Get the number of ready embeddings.
|
||||
pub fn ready_count(&self) -> usize {
|
||||
self.ready_embeddings.len()
|
||||
}
|
||||
|
||||
/// Check if a batch is ready to process.
|
||||
pub fn should_flush(&self) -> bool {
|
||||
self.pending_texts.len() >= self.batch_size
|
||||
}
|
||||
|
||||
/// Process pending texts and generate embeddings.
|
||||
///
|
||||
/// Returns the number of embeddings generated.
|
||||
pub fn flush(&mut self) -> Result<usize> {
|
||||
if self.pending_texts.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Take all pending texts
|
||||
let pending: Vec<_> = std::mem::take(&mut self.pending_texts);
|
||||
let count = pending.len();
|
||||
|
||||
// Extract texts for batch embedding
|
||||
let texts: Vec<&str> = pending.iter().map(|(_, text)| text.as_str()).collect();
|
||||
|
||||
// Generate embeddings in batch
|
||||
let embeddings = self.embedder.embed_chunks(&texts)?;
|
||||
|
||||
// Store results
|
||||
for ((frame_id, _), embedding) in pending.into_iter().zip(embeddings.into_iter()) {
|
||||
self.ready_embeddings.push((frame_id, embedding));
|
||||
}
|
||||
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// Take all ready embeddings.
|
||||
pub fn take_embeddings(&mut self) -> Vec<(FrameId, Vec<f32>)> {
|
||||
std::mem::take(&mut self.ready_embeddings)
|
||||
}
|
||||
|
||||
/// Get embedding dimension from the embedder.
|
||||
pub fn dimension(&self) -> usize {
|
||||
self.embedder.embedding_dimension()
|
||||
}
|
||||
}
|
||||
|
||||
/// Enrichment task processor (stateless, operates on Memvid instance).
|
||||
pub struct EnrichmentProcessor {
|
||||
/// Worker configuration.
|
||||
pub config: EnrichmentWorkerConfig,
|
||||
}
|
||||
|
||||
impl EnrichmentProcessor {
|
||||
/// Create a new enrichment processor.
|
||||
#[must_use]
|
||||
pub fn new(config: EnrichmentWorkerConfig) -> Self {
|
||||
Self { config }
|
||||
}
|
||||
|
||||
/// Process a single enrichment task.
|
||||
///
|
||||
/// This method:
|
||||
/// 1. Reads the frame from the memory
|
||||
/// 2. If frame needs re-extraction (skim), performs full extraction
|
||||
/// 3. If frame needs embeddings, generates them with batching
|
||||
/// 4. Updates the Tantivy index
|
||||
/// 5. Returns the result
|
||||
///
|
||||
/// The caller is responsible for:
|
||||
/// - Acquiring write lock on the memory
|
||||
/// - Updating the enrichment queue
|
||||
/// - Persisting changes
|
||||
pub fn process_task<F, E, R>(
|
||||
&self,
|
||||
task: &EnrichmentTask,
|
||||
read_frame: F,
|
||||
extract_full: E,
|
||||
update_index: R,
|
||||
) -> TaskResult
|
||||
where
|
||||
F: FnOnce(FrameId) -> Option<(String, bool, bool)>, // (text, is_skim, needs_embedding)
|
||||
E: FnOnce(FrameId) -> Result<String>, // Full extraction
|
||||
R: FnOnce(FrameId, &str) -> Result<()>, // Update index
|
||||
{
|
||||
let start = Instant::now();
|
||||
let mut result = TaskResult {
|
||||
frame_id: task.frame_id,
|
||||
re_extracted: false,
|
||||
embeddings_generated: 0,
|
||||
elapsed_ms: 0,
|
||||
error: None,
|
||||
};
|
||||
|
||||
// Read current frame state
|
||||
let (text, is_skim, _needs_embedding) = if let Some(data) = read_frame(task.frame_id) {
|
||||
data
|
||||
} else {
|
||||
result.error = Some("Frame not found".to_string());
|
||||
result.elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
|
||||
return result;
|
||||
};
|
||||
|
||||
// Re-extract if this was a skim
|
||||
let final_text = if is_skim {
|
||||
match extract_full(task.frame_id) {
|
||||
Ok(full_text) => {
|
||||
result.re_extracted = true;
|
||||
full_text
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
frame_id = task.frame_id,
|
||||
?err,
|
||||
"re-extraction failed, using skim text"
|
||||
);
|
||||
text
|
||||
}
|
||||
}
|
||||
} else {
|
||||
text
|
||||
};
|
||||
|
||||
// Update index with enriched content
|
||||
if let Err(err) = update_index(task.frame_id, &final_text) {
|
||||
result.error = Some(format!("Index update failed: {err}"));
|
||||
}
|
||||
|
||||
result.elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the enrichment worker loop.
|
||||
///
|
||||
/// This function should be called from a background thread.
|
||||
/// It processes tasks from the enrichment queue until stopped.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `handle` - Worker handle for control and statistics
|
||||
/// * `config` - Worker configuration
|
||||
/// * `get_next_task` - Closure to get the next task from the queue
|
||||
/// * `process_task` - Closure to process a single task
|
||||
/// * `mark_complete` - Closure to mark a task as complete
|
||||
/// * `checkpoint` - Closure to save progress
|
||||
pub fn run_worker_loop<G, P, M, C>(
|
||||
handle: &EnrichmentWorkerHandle,
|
||||
config: &EnrichmentWorkerConfig,
|
||||
mut get_next_task: G,
|
||||
mut process_task: P,
|
||||
mut mark_complete: M,
|
||||
mut checkpoint: C,
|
||||
) where
|
||||
G: FnMut() -> Option<EnrichmentTask>,
|
||||
P: FnMut(&EnrichmentTask) -> TaskResult,
|
||||
M: FnMut(FrameId),
|
||||
C: FnMut(),
|
||||
{
|
||||
handle.set_running(true);
|
||||
tracing::info!("enrichment worker started");
|
||||
|
||||
let mut tasks_since_checkpoint = 0;
|
||||
|
||||
while !handle.should_stop() {
|
||||
// Get next task
|
||||
let task = if let Some(task) = get_next_task() {
|
||||
task
|
||||
} else {
|
||||
// Queue is empty, wait and check again
|
||||
std::thread::sleep(Duration::from_millis(config.task_delay_ms * 10));
|
||||
continue;
|
||||
};
|
||||
|
||||
// Process the task
|
||||
let result = process_task(&task);
|
||||
|
||||
// Update statistics
|
||||
handle.inc_frames_processed();
|
||||
if result.re_extracted {
|
||||
handle.inc_re_extractions();
|
||||
}
|
||||
if result.embeddings_generated > 0 {
|
||||
handle.inc_embeddings(result.embeddings_generated as u64);
|
||||
}
|
||||
if result.error.is_some() {
|
||||
handle.inc_errors();
|
||||
tracing::warn!(
|
||||
frame_id = task.frame_id,
|
||||
error = ?result.error,
|
||||
"enrichment task failed"
|
||||
);
|
||||
} else {
|
||||
tracing::debug!(
|
||||
frame_id = task.frame_id,
|
||||
re_extracted = result.re_extracted,
|
||||
embeddings = result.embeddings_generated,
|
||||
elapsed_ms = result.elapsed_ms,
|
||||
"enrichment task complete"
|
||||
);
|
||||
}
|
||||
|
||||
// Mark task complete (remove from queue)
|
||||
mark_complete(task.frame_id);
|
||||
tasks_since_checkpoint += 1;
|
||||
|
||||
// Checkpoint periodically
|
||||
if tasks_since_checkpoint >= config.checkpoint_interval {
|
||||
checkpoint();
|
||||
tasks_since_checkpoint = 0;
|
||||
}
|
||||
|
||||
// Yield to other threads
|
||||
std::thread::sleep(Duration::from_millis(config.task_delay_ms));
|
||||
}
|
||||
|
||||
// Final checkpoint
|
||||
if tasks_since_checkpoint > 0 {
|
||||
checkpoint();
|
||||
}
|
||||
|
||||
handle.set_running(false);
|
||||
tracing::info!(
|
||||
frames_processed = handle.frames_processed.load(Ordering::Relaxed),
|
||||
"enrichment worker stopped"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Mock embedder for testing
|
||||
struct MockEmbedder {
|
||||
dimension: usize,
|
||||
}
|
||||
|
||||
impl MockEmbedder {
|
||||
fn new(dimension: usize) -> Self {
|
||||
Self { dimension }
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::types::VecEmbedder for MockEmbedder {
|
||||
fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
|
||||
// Generate deterministic embedding based on text length
|
||||
let seed = text.len() as f32;
|
||||
Ok((0..self.dimension)
|
||||
.map(|i| (seed + i as f32) * 0.1)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn embedding_dimension(&self) -> usize {
|
||||
self.dimension
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_embedding_batcher_basic() {
|
||||
let embedder = MockEmbedder::new(4);
|
||||
let mut batcher = EmbeddingBatcher::new(embedder, 2);
|
||||
|
||||
assert_eq!(batcher.pending_count(), 0);
|
||||
assert_eq!(batcher.ready_count(), 0);
|
||||
assert!(!batcher.should_flush());
|
||||
|
||||
// Add one item - shouldn't trigger flush yet
|
||||
batcher.add(1, "hello".to_string());
|
||||
assert_eq!(batcher.pending_count(), 1);
|
||||
assert!(!batcher.should_flush());
|
||||
|
||||
// Add second item - should trigger flush
|
||||
batcher.add(2, "world".to_string());
|
||||
assert_eq!(batcher.pending_count(), 2);
|
||||
assert!(batcher.should_flush());
|
||||
|
||||
// Flush the batch
|
||||
let count = batcher.flush().expect("flush should succeed");
|
||||
assert_eq!(count, 2);
|
||||
assert_eq!(batcher.pending_count(), 0);
|
||||
assert_eq!(batcher.ready_count(), 2);
|
||||
|
||||
// Take embeddings
|
||||
let embeddings = batcher.take_embeddings();
|
||||
assert_eq!(embeddings.len(), 2);
|
||||
assert_eq!(embeddings[0].0, 1); // frame_id
|
||||
assert_eq!(embeddings[0].1.len(), 4); // dimension
|
||||
assert_eq!(embeddings[1].0, 2);
|
||||
assert_eq!(embeddings[1].1.len(), 4);
|
||||
|
||||
// After take, ready should be empty
|
||||
assert_eq!(batcher.ready_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_embedding_batcher_dimension() {
|
||||
let embedder = MockEmbedder::new(128);
|
||||
let batcher = EmbeddingBatcher::new(embedder, 32);
|
||||
assert_eq!(batcher.dimension(), 128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_embedding_batcher_flush_empty() {
|
||||
let embedder = MockEmbedder::new(4);
|
||||
let mut batcher = EmbeddingBatcher::new(embedder, 2);
|
||||
|
||||
// Flushing empty batcher should return 0
|
||||
let count = batcher.flush().expect("flush should succeed");
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_worker_handle() {
|
||||
let handle = EnrichmentWorkerHandle::new();
|
||||
assert!(!handle.is_running());
|
||||
assert!(!handle.should_stop());
|
||||
|
||||
handle.set_running(true);
|
||||
assert!(handle.is_running());
|
||||
|
||||
handle.stop();
|
||||
assert!(handle.should_stop());
|
||||
|
||||
handle.inc_frames_processed();
|
||||
handle.inc_embeddings(10);
|
||||
handle.inc_re_extractions();
|
||||
handle.inc_errors();
|
||||
|
||||
let stats = handle.stats();
|
||||
assert_eq!(stats.frames_processed, 1);
|
||||
assert_eq!(stats.embeddings_generated, 10);
|
||||
assert_eq!(stats.re_extractions, 1);
|
||||
assert_eq!(stats.errors, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_processor() {
|
||||
let processor = EnrichmentProcessor::new(EnrichmentWorkerConfig::default());
|
||||
let task = EnrichmentTask {
|
||||
frame_id: 1,
|
||||
created_at: 0,
|
||||
chunks_done: 0,
|
||||
chunks_total: 0,
|
||||
};
|
||||
|
||||
let result = processor.process_task(
|
||||
&task,
|
||||
|_| Some(("test content".to_string(), false, false)),
|
||||
|_| Ok("full content".to_string()),
|
||||
|_, _| Ok(()),
|
||||
);
|
||||
|
||||
assert_eq!(result.frame_id, 1);
|
||||
assert!(!result.re_extracted); // Not a skim
|
||||
assert!(result.error.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_processor_with_skim() {
|
||||
let processor = EnrichmentProcessor::new(EnrichmentWorkerConfig::default());
|
||||
let task = EnrichmentTask {
|
||||
frame_id: 2,
|
||||
created_at: 0,
|
||||
chunks_done: 0,
|
||||
chunks_total: 0,
|
||||
};
|
||||
|
||||
let result = processor.process_task(
|
||||
&task,
|
||||
|_| Some(("skim content".to_string(), true, false)), // is_skim = true
|
||||
|_| Ok("full extracted content".to_string()),
|
||||
|_, text| {
|
||||
assert_eq!(text, "full extracted content");
|
||||
Ok(())
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(result.frame_id, 2);
|
||||
assert!(result.re_extracted); // Re-extraction happened
|
||||
assert!(result.error.is_none());
|
||||
}
|
||||
}
|
||||
+237
@@ -0,0 +1,237 @@
|
||||
use std::borrow::Cow;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// Result alias used throughout the core crate.
|
||||
pub type Result<T> = std::result::Result<T, MemvidError>;
|
||||
|
||||
/// Process metadata for a lock holder used to produce human readable diagnostics.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LockOwnerHint {
|
||||
pub pid: Option<u32>,
|
||||
pub cmd: Option<String>,
|
||||
pub started_at: Option<String>,
|
||||
pub file_path: Option<PathBuf>,
|
||||
pub file_id: Option<String>,
|
||||
pub last_heartbeat: Option<String>,
|
||||
pub heartbeat_ms: Option<u64>,
|
||||
}
|
||||
|
||||
/// Structured error returned when a `.mv2` is locked by another writer.
|
||||
#[derive(Debug, Error, Clone)]
|
||||
#[error("{message}")]
|
||||
pub struct LockedError {
|
||||
pub file: PathBuf,
|
||||
pub message: String,
|
||||
pub owner: Option<LockOwnerHint>,
|
||||
pub stale: bool,
|
||||
}
|
||||
|
||||
impl LockedError {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
file: PathBuf,
|
||||
message: impl Into<String>,
|
||||
owner: Option<LockOwnerHint>,
|
||||
stale: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
file,
|
||||
message: message.into(),
|
||||
owner,
|
||||
stale,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical error surface for memvid-core.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum MemvidError {
|
||||
#[error("I/O error: {source}")]
|
||||
Io {
|
||||
source: std::io::Error,
|
||||
path: Option<PathBuf>,
|
||||
},
|
||||
|
||||
#[error("Serialization error: {0}")]
|
||||
Encode(#[from] bincode::error::EncodeError),
|
||||
|
||||
#[error("Deserialization error: {0}")]
|
||||
Decode(#[from] bincode::error::DecodeError),
|
||||
|
||||
#[error("Lock acquisition failed: {0}")]
|
||||
Lock(String),
|
||||
|
||||
#[error(transparent)]
|
||||
Locked(#[from] Box<LockedError>),
|
||||
|
||||
#[error("Checksum mismatch while validating {context}")]
|
||||
ChecksumMismatch { context: &'static str },
|
||||
|
||||
#[error("Header validation failed: {reason}")]
|
||||
InvalidHeader { reason: Cow<'static, str> },
|
||||
|
||||
#[error("This file is encrypted: {path}\n{hint}")]
|
||||
EncryptedFile { path: PathBuf, hint: String },
|
||||
|
||||
#[error("Table of contents validation failed: {reason}")]
|
||||
InvalidToc { reason: Cow<'static, str> },
|
||||
|
||||
#[error("Time index track is invalid: {reason}")]
|
||||
InvalidTimeIndex { reason: Cow<'static, str> },
|
||||
|
||||
#[error("Sketch track is invalid: {reason}")]
|
||||
InvalidSketchTrack { reason: Cow<'static, str> },
|
||||
|
||||
#[cfg(feature = "temporal_track")]
|
||||
#[error("Temporal track is invalid: {reason}")]
|
||||
InvalidTemporalTrack { reason: Cow<'static, str> },
|
||||
|
||||
#[error("Logic-Mesh is invalid: {reason}")]
|
||||
InvalidLogicMesh { reason: Cow<'static, str> },
|
||||
|
||||
#[error("Logic-Mesh is not enabled")]
|
||||
LogicMeshNotEnabled,
|
||||
|
||||
#[error("NER model not available: {reason}")]
|
||||
NerModelNotAvailable { reason: Cow<'static, str> },
|
||||
|
||||
#[error("Unsupported tier requested")]
|
||||
InvalidTier,
|
||||
|
||||
#[error("Lexical index is not enabled")]
|
||||
LexNotEnabled,
|
||||
|
||||
#[error("Vector index is not enabled")]
|
||||
VecNotEnabled,
|
||||
|
||||
#[error("CLIP index is not enabled")]
|
||||
ClipNotEnabled,
|
||||
|
||||
#[error("Vector dimension mismatch (expected {expected}, got {actual})")]
|
||||
VecDimensionMismatch { expected: u32, actual: usize },
|
||||
|
||||
#[error("Auxiliary file detected: {path:?}")]
|
||||
AuxiliaryFileDetected { path: PathBuf },
|
||||
|
||||
#[error("Embedded WAL is corrupted at offset {offset}: {reason}")]
|
||||
WalCorruption {
|
||||
offset: u64,
|
||||
reason: Cow<'static, str>,
|
||||
},
|
||||
|
||||
#[error("Manifest WAL is corrupted at offset {offset}: {reason}")]
|
||||
ManifestWalCorrupted { offset: u64, reason: &'static str },
|
||||
|
||||
#[error("Unable to checkpoint embedded WAL: {reason}")]
|
||||
CheckpointFailed { reason: String },
|
||||
|
||||
#[error("Ticket sequence is out of order (expected > {expected}, got {actual})")]
|
||||
TicketSequence { expected: i64, actual: i64 },
|
||||
|
||||
#[error("Apply a ticket before mutating this memory (tier {tier:?})")]
|
||||
TicketRequired { tier: crate::types::Tier },
|
||||
|
||||
#[error(
|
||||
"Capacity exceeded. Current: {current} bytes, Limit: {limit} bytes, Required: {required} bytes"
|
||||
)]
|
||||
CapacityExceeded {
|
||||
current: u64,
|
||||
limit: u64,
|
||||
required: u64,
|
||||
},
|
||||
|
||||
#[error("API key required for files larger than {limit} bytes. File size: {file_size} bytes")]
|
||||
ApiKeyRequired { file_size: u64, limit: u64 },
|
||||
|
||||
#[error(
|
||||
"Memory already bound to '{existing_memory_name}' ({existing_memory_id}). Bound at: {bound_at}"
|
||||
)]
|
||||
MemoryAlreadyBound {
|
||||
existing_memory_id: uuid::Uuid,
|
||||
existing_memory_name: String,
|
||||
bound_at: String,
|
||||
},
|
||||
|
||||
#[error("Operation requires a sealed memory")]
|
||||
RequiresSealed,
|
||||
|
||||
#[error("Operation requires an open memory")]
|
||||
RequiresOpen,
|
||||
|
||||
#[error("Doctor command requires at least one operation")]
|
||||
DoctorNoOp,
|
||||
|
||||
#[error("Doctor operation failed: {reason}")]
|
||||
Doctor { reason: String },
|
||||
|
||||
#[error("Feature '{feature}' is not available in this build")]
|
||||
FeatureUnavailable { feature: &'static str },
|
||||
|
||||
#[error("Invalid search cursor: {reason}")]
|
||||
InvalidCursor { reason: &'static str },
|
||||
|
||||
#[error("Invalid frame {frame_id}: {reason}")]
|
||||
InvalidFrame {
|
||||
frame_id: crate::types::FrameId,
|
||||
reason: &'static str,
|
||||
},
|
||||
|
||||
#[error("Frame {frame_id} was not found")]
|
||||
FrameNotFound { frame_id: crate::types::FrameId },
|
||||
|
||||
#[error("Frame with uri '{uri}' was not found")]
|
||||
FrameNotFoundByUri { uri: String },
|
||||
|
||||
#[error("Ticket signature verification failed: {reason}")]
|
||||
TicketSignatureInvalid { reason: Box<str> },
|
||||
|
||||
#[error("Model signature verification failed: {reason}")]
|
||||
ModelSignatureInvalid { reason: Box<str> },
|
||||
|
||||
#[error("Model manifest invalid: {reason}")]
|
||||
ModelManifestInvalid { reason: Box<str> },
|
||||
|
||||
#[error("Model integrity check failed: {reason}")]
|
||||
ModelIntegrity { reason: Box<str> },
|
||||
|
||||
#[error("Extraction failed: {reason}")]
|
||||
ExtractionFailed { reason: Box<str> },
|
||||
|
||||
#[error("Embedding failed: {reason}")]
|
||||
EmbeddingFailed { reason: Box<str> },
|
||||
|
||||
#[error("Reranking failed: {reason}")]
|
||||
RerankFailed { reason: Box<str> },
|
||||
|
||||
#[error("Model mismatch: Index is bound to '{expected}', but requested model was '{actual}'")]
|
||||
ModelMismatch { expected: String, actual: String },
|
||||
|
||||
#[error("Invalid query: {reason}")]
|
||||
InvalidQuery { reason: String },
|
||||
|
||||
#[error("Tantivy error: {reason}")]
|
||||
Tantivy { reason: String },
|
||||
|
||||
#[error("Table extraction failed: {reason}")]
|
||||
TableExtraction { reason: String },
|
||||
|
||||
#[error("Schema validation failed: {reason}")]
|
||||
SchemaValidation { reason: String },
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for MemvidError {
|
||||
fn from(source: std::io::Error) -> Self {
|
||||
Self::Io { source, path: None }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "lex")]
|
||||
impl From<tantivy::TantivyError> for MemvidError {
|
||||
fn from(value: tantivy::TantivyError) -> Self {
|
||||
Self::Tantivy {
|
||||
reason: value.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
+1086
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,643 @@
|
||||
//! Time-budgeted text extraction for instant indexing.
|
||||
//!
|
||||
//! Extracts representative text from documents within a time budget,
|
||||
//! enabling sub-second ingestion for large files. Uses pdf-extract (when available)
|
||||
//! for fast full-document extraction, with lopdf as fallback for per-page extraction.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use lopdf::Document as LopdfDocument;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{MemvidError, Result};
|
||||
// Use SymSpell-based cleanup when feature is enabled, otherwise fall back to heuristic
|
||||
#[cfg(feature = "symspell_cleanup")]
|
||||
use crate::symspell_cleanup::fix_pdf_text as fix_pdf_spacing;
|
||||
#[cfg(not(feature = "symspell_cleanup"))]
|
||||
use crate::text::fix_pdf_spacing;
|
||||
|
||||
/// Default extraction time budget in milliseconds.
|
||||
/// Calibrated for sub-second total ingestion (<1s including I/O + indexing).
|
||||
pub const DEFAULT_EXTRACTION_BUDGET_MS: u64 = 350;
|
||||
|
||||
/// Configuration for time-budgeted extraction.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct ExtractionBudget {
|
||||
/// Maximum time to spend on extraction.
|
||||
pub budget: Duration,
|
||||
/// Maximum characters to extract (safety limit).
|
||||
pub max_chars: usize,
|
||||
/// Sample interval for middle pages (extract every Nth page).
|
||||
pub sample_interval: usize,
|
||||
}
|
||||
|
||||
impl Default for ExtractionBudget {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
budget: Duration::from_millis(DEFAULT_EXTRACTION_BUDGET_MS),
|
||||
max_chars: 100_000,
|
||||
sample_interval: 20, // Every 20th page
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ExtractionBudget {
|
||||
/// Create a budget with custom milliseconds.
|
||||
#[must_use]
|
||||
pub fn with_ms(ms: u64) -> Self {
|
||||
Self {
|
||||
budget: Duration::from_millis(ms),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Create an unlimited budget (extract everything).
|
||||
#[must_use]
|
||||
pub fn unlimited() -> Self {
|
||||
Self {
|
||||
budget: Duration::from_secs(3600), // 1 hour = effectively unlimited
|
||||
max_chars: usize::MAX,
|
||||
sample_interval: 1, // Every page
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of time-budgeted extraction.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BudgetedExtractionResult {
|
||||
/// Extracted text (may be partial).
|
||||
pub text: String,
|
||||
/// Number of pages/sections extracted.
|
||||
pub sections_extracted: usize,
|
||||
/// Total pages/sections in document.
|
||||
pub sections_total: usize,
|
||||
/// Whether extraction completed within budget.
|
||||
pub completed: bool,
|
||||
/// Time spent on extraction.
|
||||
pub elapsed_ms: u64,
|
||||
/// Coverage ratio (0.0 to 1.0).
|
||||
pub coverage: f32,
|
||||
}
|
||||
|
||||
impl BudgetedExtractionResult {
|
||||
/// Check if we got meaningful content.
|
||||
#[must_use]
|
||||
pub fn has_content(&self) -> bool {
|
||||
!self.text.trim().is_empty()
|
||||
}
|
||||
|
||||
/// Check if this is a skim (partial) extraction.
|
||||
#[must_use]
|
||||
pub fn is_skim(&self) -> bool {
|
||||
!self.completed && self.sections_extracted < self.sections_total
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract text from a PDF with time budget.
|
||||
///
|
||||
/// Strategy:
|
||||
/// 1. Try extractous first (best quality, handles all fonts including ligatures)
|
||||
/// 2. Try pdf-extract (fast, but panics on some fonts with ligatures)
|
||||
/// 3. Fall back to lopdf per-page (basic extraction, poor word spacing)
|
||||
pub fn extract_pdf_budgeted(
|
||||
bytes: &[u8],
|
||||
budget: ExtractionBudget,
|
||||
) -> Result<BudgetedExtractionResult> {
|
||||
let start = Instant::now();
|
||||
|
||||
// Try extractous first - best quality, handles all fonts including ligatures
|
||||
#[cfg(feature = "extractous")]
|
||||
{
|
||||
use extractous::Extractor;
|
||||
|
||||
// Write bytes to temp file (extractous needs a file path)
|
||||
if let Ok(mut temp_file) = tempfile::NamedTempFile::new() {
|
||||
use std::io::Write;
|
||||
if temp_file.write_all(bytes).is_ok() {
|
||||
let extractor = Extractor::new();
|
||||
let path_str = temp_file.path().to_string_lossy();
|
||||
if let Ok((text, _metadata)) = extractor.extract_file_to_string(&path_str) {
|
||||
let trimmed = text.trim();
|
||||
if !trimmed.is_empty() {
|
||||
tracing::debug!("extractous successfully extracted PDF text");
|
||||
// Estimate page count from text (rough heuristic: ~3000 chars per page)
|
||||
let estimated_pages = (trimmed.len() / 3000).max(1);
|
||||
let text_len = trimmed.len();
|
||||
|
||||
// Truncate if needed
|
||||
let final_text = if text_len > budget.max_chars {
|
||||
truncate_at_boundary(trimmed, budget.max_chars)
|
||||
} else {
|
||||
trimmed.to_string()
|
||||
};
|
||||
|
||||
let completed = final_text.len() == text_len;
|
||||
|
||||
return Ok(BudgetedExtractionResult {
|
||||
text: final_text,
|
||||
sections_extracted: estimated_pages,
|
||||
sections_total: estimated_pages,
|
||||
completed,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
});
|
||||
}
|
||||
tracing::debug!("extractous returned empty text, trying pdf-extract");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try pdf-extract - fast full document extraction
|
||||
// Wrap in catch_unwind because cff-parser may panic on certain fonts (ligatures like fi/fl)
|
||||
#[cfg(feature = "pdf_extract")]
|
||||
{
|
||||
let bytes_clone = bytes.to_vec();
|
||||
let extract_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
pdf_extract::extract_text_from_mem(&bytes_clone)
|
||||
}));
|
||||
|
||||
match extract_result {
|
||||
Ok(Ok(text)) => {
|
||||
// Clean up character spacing issues from PDF extraction
|
||||
let cleaned = fix_pdf_spacing(text.trim());
|
||||
if !cleaned.is_empty() {
|
||||
// Estimate page count from text (rough heuristic: ~3000 chars per page)
|
||||
let estimated_pages = (cleaned.len() / 3000).max(1);
|
||||
let cleaned_len = cleaned.len();
|
||||
|
||||
// Truncate if needed
|
||||
let final_text = if cleaned_len > budget.max_chars {
|
||||
truncate_at_boundary(&cleaned, budget.max_chars)
|
||||
} else {
|
||||
cleaned
|
||||
};
|
||||
|
||||
let completed = final_text.len() == cleaned_len;
|
||||
|
||||
return Ok(BudgetedExtractionResult {
|
||||
text: final_text,
|
||||
sections_extracted: estimated_pages,
|
||||
sections_total: estimated_pages,
|
||||
completed,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
});
|
||||
}
|
||||
// pdf-extract returned empty, fall through to lopdf
|
||||
tracing::debug!("pdf-extract returned empty text, trying lopdf");
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
// pdf-extract failed, fall through to lopdf
|
||||
tracing::debug!(?e, "pdf-extract failed, trying lopdf");
|
||||
}
|
||||
Err(_) => {
|
||||
// pdf-extract panicked (cff-parser font issue), fall through to lopdf
|
||||
tracing::warn!(
|
||||
"pdf-extract panicked (likely font parsing issue), falling back to lopdf"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to lopdf per-page extraction
|
||||
extract_pdf_budgeted_lopdf(bytes, budget, start)
|
||||
}
|
||||
|
||||
/// Truncate text at a word/sentence boundary
|
||||
fn truncate_at_boundary(text: &str, max_chars: usize) -> String {
|
||||
if text.len() <= max_chars {
|
||||
return text.to_string();
|
||||
}
|
||||
|
||||
// Find last space before max_chars
|
||||
let truncate_at = text[..max_chars]
|
||||
.rfind(|c: char| c.is_whitespace())
|
||||
.unwrap_or(max_chars);
|
||||
|
||||
text[..truncate_at].to_string()
|
||||
}
|
||||
|
||||
/// lopdf-based per-page extraction (slower but works as fallback)
|
||||
fn extract_pdf_budgeted_lopdf(
|
||||
bytes: &[u8],
|
||||
budget: ExtractionBudget,
|
||||
start: Instant,
|
||||
) -> Result<BudgetedExtractionResult> {
|
||||
let deadline = start + budget.budget;
|
||||
|
||||
// Load PDF
|
||||
let mut document =
|
||||
LopdfDocument::load_mem(bytes).map_err(|err| MemvidError::ExtractionFailed {
|
||||
reason: format!("failed to load PDF: {err}").into(),
|
||||
})?;
|
||||
|
||||
// Handle encryption
|
||||
if document.is_encrypted() && document.decrypt("").is_err() {
|
||||
return Err(MemvidError::ExtractionFailed {
|
||||
reason: "cannot decrypt password-protected PDF".into(),
|
||||
});
|
||||
}
|
||||
|
||||
// Decompress for better extraction
|
||||
let () = document.decompress();
|
||||
|
||||
// Get page numbers
|
||||
let mut page_numbers: Vec<u32> = document.get_pages().keys().copied().collect();
|
||||
if page_numbers.is_empty() {
|
||||
return Ok(BudgetedExtractionResult {
|
||||
text: String::new(),
|
||||
sections_extracted: 0,
|
||||
sections_total: 0,
|
||||
completed: true,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
});
|
||||
}
|
||||
page_numbers.sort_unstable();
|
||||
|
||||
let page_count = page_numbers.len();
|
||||
let mut extracted_pages: Vec<(u32, String)> = Vec::new();
|
||||
let mut total_chars = 0usize;
|
||||
|
||||
// Priority pages: first and last
|
||||
let priority_pages: Vec<u32> = {
|
||||
let mut pages = vec![page_numbers[0]]; // First page
|
||||
if page_count > 1 {
|
||||
pages.push(page_numbers[page_count - 1]); // Last page
|
||||
}
|
||||
pages
|
||||
};
|
||||
|
||||
// Extract priority pages first (always)
|
||||
for &page_num in &priority_pages {
|
||||
if total_chars >= budget.max_chars {
|
||||
break;
|
||||
}
|
||||
|
||||
if let Ok(text) = document.extract_text(&[page_num]) {
|
||||
let trimmed = text.trim();
|
||||
if !trimmed.is_empty() {
|
||||
total_chars += trimmed.len();
|
||||
extracted_pages.push((page_num, trimmed.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we still have time
|
||||
if Instant::now() >= deadline || total_chars >= budget.max_chars {
|
||||
return finish_extraction(extracted_pages, page_count, start, false);
|
||||
}
|
||||
|
||||
// Sample middle pages until deadline
|
||||
let sample_interval = budget.sample_interval.max(1);
|
||||
let middle_pages: Vec<u32> = page_numbers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, page)| {
|
||||
// Skip priority pages, sample every Nth page
|
||||
!priority_pages.contains(page) && (*i % sample_interval == 0)
|
||||
})
|
||||
.map(|(_, page)| *page)
|
||||
.collect();
|
||||
|
||||
for &page_num in &middle_pages {
|
||||
// Check deadline before each page
|
||||
if Instant::now() >= deadline {
|
||||
return finish_extraction(extracted_pages, page_count, start, false);
|
||||
}
|
||||
|
||||
if total_chars >= budget.max_chars {
|
||||
return finish_extraction(extracted_pages, page_count, start, false);
|
||||
}
|
||||
|
||||
if let Ok(text) = document.extract_text(&[page_num]) {
|
||||
let trimmed = text.trim();
|
||||
if !trimmed.is_empty() {
|
||||
total_chars += trimmed.len();
|
||||
extracted_pages.push((page_num, trimmed.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we got here, we extracted all sampled pages without hitting limits
|
||||
let completed = extracted_pages.len() >= page_count;
|
||||
finish_extraction(extracted_pages, page_count, start, completed)
|
||||
}
|
||||
|
||||
/// Extract text from plain text/markdown with time budget.
|
||||
/// Text formats are fast - we typically get everything.
|
||||
pub fn extract_text_budgeted(
|
||||
bytes: &[u8],
|
||||
budget: ExtractionBudget,
|
||||
) -> Result<BudgetedExtractionResult> {
|
||||
let start = Instant::now();
|
||||
|
||||
// Try UTF-8 decode, using lossy conversion for non-UTF8
|
||||
let text: String = match std::str::from_utf8(bytes) {
|
||||
Ok(s) => s.to_string(),
|
||||
Err(_) => String::from_utf8_lossy(bytes).into_owned(),
|
||||
};
|
||||
|
||||
// Truncate if needed, respecting character boundaries
|
||||
let truncated = if text.len() > budget.max_chars {
|
||||
// Find a valid character boundary
|
||||
let mut end = budget.max_chars;
|
||||
while end > 0 && !text.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
text[..end].to_string()
|
||||
} else {
|
||||
text
|
||||
};
|
||||
|
||||
// Count "sections" as paragraphs (double newlines)
|
||||
let sections = truncated.split("\n\n").count();
|
||||
|
||||
Ok(BudgetedExtractionResult {
|
||||
text: truncated,
|
||||
sections_extracted: sections,
|
||||
sections_total: sections,
|
||||
completed: true,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Check if MIME type is an Office Open XML format (xlsx, docx, pptx).
|
||||
fn is_ooxml_mime(mime: Option<&str>) -> bool {
|
||||
let Some(m) = mime else { return false };
|
||||
let m = m.to_lowercase();
|
||||
m.contains("spreadsheetml")
|
||||
|| m.contains("wordprocessingml")
|
||||
|| m.contains("presentationml")
|
||||
|| m == "application/vnd.ms-excel"
|
||||
|| m == "application/msword"
|
||||
|| m == "application/vnd.ms-powerpoint"
|
||||
}
|
||||
|
||||
/// Extract OOXML and legacy Office documents using the reader registry.
|
||||
fn extract_ooxml_budgeted(
|
||||
bytes: &[u8],
|
||||
mime: Option<&str>,
|
||||
uri: Option<&str>,
|
||||
) -> Result<BudgetedExtractionResult> {
|
||||
use crate::reader::{DocumentFormat, ReaderHint, ReaderRegistry};
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
// Determine the document format from MIME first, then fall back to extension
|
||||
let format = match mime.map(str::to_lowercase).as_deref() {
|
||||
Some(m) if m.contains("spreadsheetml") => Some(DocumentFormat::Xlsx),
|
||||
Some(m) if m.contains("wordprocessingml") => Some(DocumentFormat::Docx),
|
||||
Some(m) if m.contains("presentationml") => Some(DocumentFormat::Pptx),
|
||||
Some("application/vnd.ms-excel") => Some(DocumentFormat::Xls),
|
||||
_ => {
|
||||
// Fall back to extension-based detection
|
||||
uri.and_then(|u| {
|
||||
let lower = u.to_lowercase();
|
||||
if lower.ends_with(".xlsx") {
|
||||
Some(DocumentFormat::Xlsx)
|
||||
} else if lower.ends_with(".docx") {
|
||||
Some(DocumentFormat::Docx)
|
||||
} else if lower.ends_with(".pptx") {
|
||||
Some(DocumentFormat::Pptx)
|
||||
} else if lower.ends_with(".xls") {
|
||||
Some(DocumentFormat::Xls)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
let hint = ReaderHint::new(mime, format).with_uri(uri);
|
||||
let registry = ReaderRegistry::default();
|
||||
|
||||
if let Some(reader) = registry.find_reader(&hint) {
|
||||
match reader.extract(bytes, &hint) {
|
||||
Ok(output) => {
|
||||
let text = output.document.text.unwrap_or_default();
|
||||
let sections = text
|
||||
.split("\n\n")
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.count()
|
||||
.max(1);
|
||||
Ok(BudgetedExtractionResult {
|
||||
text,
|
||||
sections_extracted: sections,
|
||||
sections_total: sections,
|
||||
completed: true,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
})
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
} else {
|
||||
// No reader found, return empty
|
||||
Ok(BudgetedExtractionResult {
|
||||
text: String::new(),
|
||||
sections_extracted: 0,
|
||||
sections_total: 0,
|
||||
completed: true,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Determine document type and extract with budget.
|
||||
pub fn extract_with_budget(
|
||||
bytes: &[u8],
|
||||
mime: Option<&str>,
|
||||
uri: Option<&str>,
|
||||
budget: ExtractionBudget,
|
||||
) -> Result<BudgetedExtractionResult> {
|
||||
// Check if PDF
|
||||
let is_pdf = mime.is_some_and(|m| m.contains("pdf")) || is_pdf_magic(bytes);
|
||||
|
||||
if is_pdf {
|
||||
extract_pdf_budgeted(bytes, budget)
|
||||
} else if is_ooxml_mime(mime) || is_ooxml_by_extension(uri) || is_ooxml_by_magic(bytes, uri) {
|
||||
// Handle Office Open XML formats (xlsx, docx, pptx) via reader registry
|
||||
extract_ooxml_budgeted(bytes, mime, uri)
|
||||
} else if is_binary_mime(mime) || is_binary_content(bytes) {
|
||||
// Skip extraction for binary content (video, audio, images, etc.)
|
||||
Ok(BudgetedExtractionResult {
|
||||
text: String::new(),
|
||||
sections_extracted: 0,
|
||||
sections_total: 0,
|
||||
completed: true,
|
||||
elapsed_ms: 0,
|
||||
coverage: 1.0,
|
||||
})
|
||||
} else {
|
||||
// Treat as text
|
||||
extract_text_budgeted(bytes, budget)
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if file extension indicates OOXML format
|
||||
fn is_ooxml_by_extension(uri: Option<&str>) -> bool {
|
||||
let Some(u) = uri else { return false };
|
||||
let lower = u.to_lowercase();
|
||||
lower.ends_with(".docx")
|
||||
|| lower.ends_with(".xlsx")
|
||||
|| lower.ends_with(".pptx")
|
||||
|| lower.ends_with(".doc")
|
||||
|| lower.ends_with(".xls")
|
||||
|| lower.ends_with(".ppt")
|
||||
}
|
||||
|
||||
/// Check if bytes start with ZIP magic and extension indicates OOXML
|
||||
fn is_ooxml_by_magic(bytes: &[u8], uri: Option<&str>) -> bool {
|
||||
// ZIP magic bytes: PK\x03\x04
|
||||
if bytes.len() >= 4 && bytes.starts_with(&[0x50, 0x4B, 0x03, 0x04]) {
|
||||
// It's a ZIP file - check if extension suggests Office format
|
||||
is_ooxml_by_extension(uri)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if mime type indicates binary content
|
||||
fn is_binary_mime(mime: Option<&str>) -> bool {
|
||||
let Some(m) = mime else { return false };
|
||||
let m = m.to_lowercase();
|
||||
m.starts_with("video/")
|
||||
|| m.starts_with("audio/")
|
||||
|| m.starts_with("image/")
|
||||
|| m == "application/octet-stream"
|
||||
|| m.contains("zip")
|
||||
|| m.contains("gzip")
|
||||
|| m.contains("tar")
|
||||
|| m.contains("rar")
|
||||
|| m.contains("7z")
|
||||
}
|
||||
|
||||
/// Check if content appears to be binary (high ratio of non-printable bytes)
|
||||
fn is_binary_content(bytes: &[u8]) -> bool {
|
||||
if bytes.is_empty() {
|
||||
return false;
|
||||
}
|
||||
// Sample first 8KB to detect binary content
|
||||
let sample_size = bytes.len().min(8192);
|
||||
let sample = &bytes[..sample_size];
|
||||
|
||||
let non_text_count = sample
|
||||
.iter()
|
||||
.filter(|&&b| {
|
||||
// Non-text: null bytes, or bytes outside printable ASCII/UTF-8 range
|
||||
// Allow: tab, newline, carriage return, and printable ASCII
|
||||
b == 0 || (b < 32 && b != 9 && b != 10 && b != 13)
|
||||
})
|
||||
.count();
|
||||
|
||||
// If >30% of bytes are non-text, treat as binary
|
||||
non_text_count * 100 / sample_size > 30
|
||||
}
|
||||
|
||||
/// Check PDF magic bytes.
|
||||
fn is_pdf_magic(bytes: &[u8]) -> bool {
|
||||
if bytes.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let mut slice = bytes;
|
||||
// Skip BOM
|
||||
if slice.starts_with(&[0xEF, 0xBB, 0xBF]) {
|
||||
slice = &slice[3..];
|
||||
}
|
||||
// Skip whitespace
|
||||
while let Some((first, rest)) = slice.split_first() {
|
||||
if *first == 0 || first.is_ascii_whitespace() {
|
||||
slice = rest;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
slice.starts_with(b"%PDF")
|
||||
}
|
||||
|
||||
/// Finish extraction and build result.
|
||||
fn finish_extraction(
|
||||
mut pages: Vec<(u32, String)>,
|
||||
total_pages: usize,
|
||||
start: Instant,
|
||||
completed: bool,
|
||||
) -> Result<BudgetedExtractionResult> {
|
||||
// Sort by page number for coherent output
|
||||
pages.sort_by_key(|(num, _)| *num);
|
||||
|
||||
let sections_extracted = pages.len();
|
||||
let text = pages
|
||||
.into_iter()
|
||||
.map(|(_, text)| fix_pdf_spacing(&text)) // Clean up character spacing
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n");
|
||||
|
||||
let coverage = if total_pages > 0 {
|
||||
sections_extracted as f32 / total_pages as f32
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
|
||||
Ok(BudgetedExtractionResult {
|
||||
text,
|
||||
sections_extracted,
|
||||
sections_total: total_pages,
|
||||
completed,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_text_extraction_budget() {
|
||||
let text = b"Hello world.\n\nThis is a test.\n\nAnother paragraph.";
|
||||
let result = extract_text_budgeted(text, ExtractionBudget::default()).unwrap();
|
||||
|
||||
assert!(result.completed);
|
||||
assert!(result.has_content());
|
||||
assert_eq!(result.sections_extracted, 3);
|
||||
assert_eq!(result.coverage, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_truncation() {
|
||||
let text = "x".repeat(200_000);
|
||||
let budget = ExtractionBudget {
|
||||
max_chars: 1000,
|
||||
..Default::default()
|
||||
};
|
||||
let result = extract_text_budgeted(text.as_bytes(), budget).unwrap();
|
||||
|
||||
assert_eq!(result.text.len(), 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pdf_magic_detection() {
|
||||
assert!(is_pdf_magic(b"%PDF-1.7"));
|
||||
assert!(is_pdf_magic(b" \n%PDF-1.4"));
|
||||
assert!(!is_pdf_magic(b"Hello world"));
|
||||
assert!(!is_pdf_magic(b""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_budget_config() {
|
||||
let default = ExtractionBudget::default();
|
||||
assert_eq!(default.budget.as_millis(), 350);
|
||||
|
||||
let custom = ExtractionBudget::with_ms(500);
|
||||
assert_eq!(custom.budget.as_millis(), 500);
|
||||
|
||||
let unlimited = ExtractionBudget::unlimited();
|
||||
assert_eq!(unlimited.sample_interval, 1);
|
||||
}
|
||||
|
||||
// PDF spacing cleanup tests are in text.rs (fix_pdf_spacing function)
|
||||
}
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
use std::convert::TryInto;
|
||||
|
||||
use blake3::Hasher;
|
||||
use memchr::memrchr;
|
||||
|
||||
/// Magic trailer marker appended to every committed TOC.
|
||||
pub const FOOTER_MAGIC: &[u8; 8] = b"MV2FOOT!";
|
||||
|
||||
/// Total size of a commit footer in bytes.
|
||||
pub const FOOTER_SIZE: usize = FOOTER_MAGIC.len() + 8 + 32 + 8;
|
||||
|
||||
/// Parsed representation of the footer trailer.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CommitFooter {
|
||||
pub toc_len: u64,
|
||||
pub toc_hash: [u8; 32],
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
impl CommitFooter {
|
||||
/// Serialises the footer into a fixed-size byte array.
|
||||
#[must_use]
|
||||
pub fn encode(&self) -> [u8; FOOTER_SIZE] {
|
||||
let mut buf = [0u8; FOOTER_SIZE];
|
||||
buf[..FOOTER_MAGIC.len()].copy_from_slice(FOOTER_MAGIC);
|
||||
buf[FOOTER_MAGIC.len()..FOOTER_MAGIC.len() + 8]
|
||||
.copy_from_slice(&self.toc_len.to_le_bytes());
|
||||
buf[FOOTER_MAGIC.len() + 8..FOOTER_MAGIC.len() + 40].copy_from_slice(&self.toc_hash);
|
||||
buf[FOOTER_MAGIC.len() + 40..].copy_from_slice(&self.generation.to_le_bytes());
|
||||
buf
|
||||
}
|
||||
|
||||
/// Attempts to decode a footer from a byte slice.
|
||||
#[must_use]
|
||||
pub fn decode(bytes: &[u8]) -> Option<Self> {
|
||||
if bytes.len() != FOOTER_SIZE {
|
||||
return None;
|
||||
}
|
||||
if &bytes[..FOOTER_MAGIC.len()] != FOOTER_MAGIC {
|
||||
return None;
|
||||
}
|
||||
let toc_len = u64::from_le_bytes(
|
||||
bytes[FOOTER_MAGIC.len()..FOOTER_MAGIC.len() + 8]
|
||||
.try_into()
|
||||
.ok()?,
|
||||
);
|
||||
let mut toc_hash = [0u8; 32];
|
||||
toc_hash.copy_from_slice(&bytes[FOOTER_MAGIC.len() + 8..FOOTER_MAGIC.len() + 40]);
|
||||
let generation = u64::from_le_bytes(bytes[FOOTER_MAGIC.len() + 40..].try_into().ok()?);
|
||||
Some(Self {
|
||||
toc_len,
|
||||
toc_hash,
|
||||
generation,
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn hash_matches(&self, toc_bytes: &[u8]) -> bool {
|
||||
let mut hasher = Hasher::new();
|
||||
hasher.update(toc_bytes);
|
||||
hasher.finalize().as_bytes() == &self.toc_hash
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of scanning a file for the last valid commit footer.
|
||||
#[derive(Debug)]
|
||||
pub struct FooterSlice<'a> {
|
||||
pub footer_offset: usize,
|
||||
pub toc_offset: usize,
|
||||
pub footer: CommitFooter,
|
||||
pub toc_bytes: &'a [u8],
|
||||
}
|
||||
|
||||
/// Scan the provided bytes backwards to locate the most recent valid footer.
|
||||
#[must_use]
|
||||
pub fn find_last_valid_footer(bytes: &[u8]) -> Option<FooterSlice<'_>> {
|
||||
if bytes.len() < FOOTER_SIZE {
|
||||
return None;
|
||||
}
|
||||
|
||||
let total_len = bytes.len();
|
||||
let mut search_end = bytes.len();
|
||||
while let Some(pos) = memrchr(FOOTER_MAGIC[0], &bytes[..search_end]) {
|
||||
if pos + FOOTER_SIZE > total_len {
|
||||
if pos == 0 {
|
||||
break;
|
||||
}
|
||||
search_end = pos;
|
||||
continue;
|
||||
}
|
||||
let candidate = &bytes[pos..pos + FOOTER_SIZE];
|
||||
if let Some(footer) = CommitFooter::decode(candidate) {
|
||||
let toc_end = pos;
|
||||
let toc_len = usize::try_from(footer.toc_len).unwrap_or(0);
|
||||
if toc_len == 0 || toc_len > toc_end {
|
||||
search_end = pos;
|
||||
continue;
|
||||
}
|
||||
let toc_offset = toc_end - toc_len;
|
||||
let toc_bytes = &bytes[toc_offset..toc_end];
|
||||
if !footer.hash_matches(toc_bytes) {
|
||||
search_end = pos;
|
||||
continue;
|
||||
}
|
||||
return Some(FooterSlice {
|
||||
footer_offset: pos,
|
||||
toc_offset,
|
||||
footer,
|
||||
toc_bytes,
|
||||
});
|
||||
}
|
||||
if pos == 0 {
|
||||
break;
|
||||
}
|
||||
search_end = pos;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn build_sample_bytes(generation: u64, toc: &[u8]) -> Vec<u8> {
|
||||
let mut footer_hash = Hasher::new();
|
||||
footer_hash.update(toc);
|
||||
let mut buffer = Vec::new();
|
||||
buffer.extend_from_slice(toc);
|
||||
let footer = CommitFooter {
|
||||
toc_len: toc.len() as u64,
|
||||
toc_hash: *footer_hash.finalize().as_bytes(),
|
||||
generation,
|
||||
};
|
||||
buffer.extend_from_slice(&footer.encode());
|
||||
buffer
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_decode_roundtrip() {
|
||||
let footer = CommitFooter {
|
||||
toc_len: 123,
|
||||
toc_hash: [0xAB; 32],
|
||||
generation: 99,
|
||||
};
|
||||
let encoded = footer.encode();
|
||||
let decoded = CommitFooter::decode(&encoded).expect("decode");
|
||||
assert_eq!(footer, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_finds_footer() {
|
||||
let toc = vec![0xAA, 0xBB, 0xCC];
|
||||
let bytes = build_sample_bytes(7, &toc);
|
||||
let slice = find_last_valid_footer(&bytes).expect("footer present");
|
||||
assert_eq!(slice.footer.generation, 7);
|
||||
assert_eq!(slice.toc_bytes, toc);
|
||||
assert_eq!(
|
||||
&bytes[slice.footer_offset..slice.footer_offset + FOOTER_SIZE],
|
||||
&slice.footer.encode()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_skips_corrupt_footer() {
|
||||
let toc = vec![1u8, 2, 3, 4];
|
||||
let mut bytes = build_sample_bytes(1, &toc);
|
||||
// Corrupt the hash of the first footer.
|
||||
let idx = bytes.len() - FOOTER_SIZE + FOOTER_MAGIC.len() + 12;
|
||||
bytes[idx] ^= 0xFF;
|
||||
// Append a valid second footer.
|
||||
let mut extra_toc = vec![9u8; 10];
|
||||
extra_toc.push(42);
|
||||
let appended = build_sample_bytes(2, &extra_toc);
|
||||
bytes.extend_from_slice(&appended);
|
||||
let slice = find_last_valid_footer(&bytes).expect("footer present");
|
||||
assert_eq!(slice.footer.generation, 2);
|
||||
assert_eq!(slice.toc_bytes, &extra_toc);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,509 @@
|
||||
//! Graph-aware search combining MemoryCards/Logic-Mesh with vector search.
|
||||
//!
|
||||
//! This module provides hybrid retrieval that can:
|
||||
//! 1. Parse natural language queries for relational patterns
|
||||
//! 2. Match patterns against entity state (`MemoryCards`) or graph (Logic-Mesh)
|
||||
//! 3. Combine graph-filtered candidates with vector ranking
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::types::{
|
||||
GraphMatchResult, GraphPattern, HybridSearchHit, PatternTerm, QueryPlan, SearchRequest,
|
||||
TriplePattern,
|
||||
};
|
||||
use crate::{FrameId, Memvid, Result};
|
||||
|
||||
/// Query planner that analyzes queries and creates execution plans.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct QueryPlanner {
|
||||
/// Patterns for detecting relational queries
|
||||
entity_patterns: Vec<EntityPattern>,
|
||||
}
|
||||
|
||||
/// Pattern for detecting entity-related queries.
|
||||
#[derive(Debug, Clone)]
|
||||
struct EntityPattern {
|
||||
/// Keywords that trigger this pattern
|
||||
keywords: Vec<&'static str>,
|
||||
/// Slot to query
|
||||
slot: &'static str,
|
||||
/// Whether the pattern looks for a specific value
|
||||
needs_value: bool,
|
||||
}
|
||||
|
||||
impl QueryPlanner {
|
||||
/// Create a new query planner.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
let mut planner = Self::default();
|
||||
planner.init_patterns();
|
||||
planner
|
||||
}
|
||||
|
||||
fn init_patterns(&mut self) {
|
||||
// Location patterns
|
||||
self.entity_patterns.push(EntityPattern {
|
||||
keywords: vec![
|
||||
"who lives in",
|
||||
"people in",
|
||||
"users in",
|
||||
"from",
|
||||
"located in",
|
||||
"based in",
|
||||
],
|
||||
slot: "location",
|
||||
needs_value: true,
|
||||
});
|
||||
|
||||
// Employer/workplace patterns
|
||||
self.entity_patterns.push(EntityPattern {
|
||||
keywords: vec![
|
||||
"who works at",
|
||||
"employees of",
|
||||
"people at",
|
||||
"works for",
|
||||
"employed by",
|
||||
],
|
||||
slot: "workplace", // OpenAI enrichment uses "workplace" not "employer"
|
||||
needs_value: true,
|
||||
});
|
||||
|
||||
// Preference patterns
|
||||
self.entity_patterns.push(EntityPattern {
|
||||
keywords: vec![
|
||||
"who likes",
|
||||
"who loves",
|
||||
"fans of",
|
||||
"people who like",
|
||||
"people who love",
|
||||
],
|
||||
slot: "preference",
|
||||
needs_value: true,
|
||||
});
|
||||
|
||||
// Entity state patterns
|
||||
self.entity_patterns.push(EntityPattern {
|
||||
keywords: vec!["what is", "where does", "who is", "what does"],
|
||||
slot: "",
|
||||
needs_value: false,
|
||||
});
|
||||
}
|
||||
|
||||
/// Analyze a query and produce an execution plan.
|
||||
#[must_use]
|
||||
pub fn plan(&self, query: &str, top_k: usize) -> QueryPlan {
|
||||
let query_lower = query.to_lowercase();
|
||||
|
||||
// Try to detect relational patterns
|
||||
if let Some(pattern) = self.detect_pattern(&query_lower, query) {
|
||||
if pattern.triples.is_empty() {
|
||||
// No specific pattern found, use vector search
|
||||
QueryPlan::vector_only(Some(query.to_string()), None, top_k)
|
||||
} else {
|
||||
// Found relational pattern - use hybrid search
|
||||
QueryPlan::hybrid(pattern, Some(query.to_string()), None, top_k)
|
||||
}
|
||||
} else {
|
||||
// Default to vector-only search
|
||||
QueryPlan::vector_only(Some(query.to_string()), None, top_k)
|
||||
}
|
||||
}
|
||||
|
||||
fn detect_pattern(&self, query_lower: &str, _original: &str) -> Option<GraphPattern> {
|
||||
let mut pattern = GraphPattern::new();
|
||||
|
||||
for ep in &self.entity_patterns {
|
||||
for keyword in &ep.keywords {
|
||||
if query_lower.contains(keyword) {
|
||||
// Extract the value after the keyword
|
||||
if let Some(pos) = query_lower.find(keyword) {
|
||||
let after = &query_lower[pos + keyword.len()..];
|
||||
let value = extract_value(after);
|
||||
|
||||
if !value.is_empty() && ep.needs_value {
|
||||
// Create pattern: ?entity :slot "value"
|
||||
pattern.add(TriplePattern::any_slot_value("entity", ep.slot, &value));
|
||||
return Some(pattern);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for entity-specific queries like "alice's employer" or "what is alice's job"
|
||||
if let Some((entity, slot)) = extract_possessive_query(query_lower) {
|
||||
pattern.add(TriplePattern::entity_slot_any(&entity, &slot, "value"));
|
||||
return Some(pattern);
|
||||
}
|
||||
|
||||
Some(pattern)
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract a value from text after a keyword.
|
||||
fn extract_value(text: &str) -> String {
|
||||
let trimmed = text.trim();
|
||||
// Take words until we hit a common query continuation
|
||||
let stop_words = ["and", "or", "who", "what", "that", "?"];
|
||||
let mut words = Vec::new();
|
||||
|
||||
for word in trimmed.split_whitespace() {
|
||||
let clean = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-');
|
||||
if stop_words.contains(&clean.to_lowercase().as_str()) {
|
||||
break;
|
||||
}
|
||||
if !clean.is_empty() {
|
||||
words.push(clean);
|
||||
}
|
||||
// Stop after a few words
|
||||
if words.len() >= 3 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
words.join(" ")
|
||||
}
|
||||
|
||||
/// Extract entity and slot from possessive queries like "alice's employer".
|
||||
fn extract_possessive_query(query: &str) -> Option<(String, String)> {
|
||||
// Pattern: "X's Y" or "X's Y is"
|
||||
if let Some(pos) = query.find("'s ") {
|
||||
let entity = query[..pos].split_whitespace().last()?;
|
||||
let after = &query[pos + 3..];
|
||||
let slot = after.split_whitespace().next()?;
|
||||
|
||||
// Map common slot aliases
|
||||
let slot = match slot {
|
||||
"job" | "work" | "employer" | "role" | "company" => "workplace",
|
||||
"home" | "city" | "address" => "location",
|
||||
"favorite" => "preference",
|
||||
"wife" | "husband" | "spouse" | "partner" => "spouse",
|
||||
other => other,
|
||||
};
|
||||
|
||||
return Some((entity.to_string(), slot.to_string()));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Graph matcher that executes patterns against `MemoryCards`.
|
||||
pub struct GraphMatcher<'a> {
|
||||
memvid: &'a Memvid,
|
||||
}
|
||||
|
||||
impl<'a> GraphMatcher<'a> {
|
||||
/// Create a new graph matcher.
|
||||
#[must_use]
|
||||
pub fn new(memvid: &'a Memvid) -> Self {
|
||||
Self { memvid }
|
||||
}
|
||||
|
||||
/// Execute a graph pattern and return matching results.
|
||||
#[must_use]
|
||||
pub fn execute(&self, pattern: &GraphPattern) -> Vec<GraphMatchResult> {
|
||||
let mut results = Vec::new();
|
||||
|
||||
for triple in &pattern.triples {
|
||||
let matches = self.match_triple(triple);
|
||||
results.extend(matches);
|
||||
}
|
||||
|
||||
// Deduplicate by entity
|
||||
let mut seen = HashSet::new();
|
||||
results.retain(|r| seen.insert(r.entity.clone()));
|
||||
|
||||
results
|
||||
}
|
||||
|
||||
fn match_triple(&self, triple: &TriplePattern) -> Vec<GraphMatchResult> {
|
||||
let mut results = Vec::new();
|
||||
|
||||
match (&triple.subject, &triple.predicate, &triple.object) {
|
||||
// Pattern: ?entity :slot "value" - find entities with this slot value
|
||||
(
|
||||
PatternTerm::Variable(var),
|
||||
PatternTerm::Literal(slot),
|
||||
PatternTerm::Literal(value),
|
||||
) => {
|
||||
// Iterate all entities and check for matching slot value
|
||||
for entity in self.memvid.memory_entities() {
|
||||
let cards = self.memvid.get_entity_memories(&entity);
|
||||
for card in cards {
|
||||
if card.slot.to_lowercase() == *slot
|
||||
&& card.value.to_lowercase().contains(&value.to_lowercase())
|
||||
{
|
||||
let mut result = GraphMatchResult::new(
|
||||
entity.clone(),
|
||||
vec![card.source_frame_id],
|
||||
1.0,
|
||||
);
|
||||
result.bind(var, entity.clone());
|
||||
results.push(result);
|
||||
break; // One match per entity
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pattern: "entity" :slot ?value - get entity's slot value
|
||||
(
|
||||
PatternTerm::Literal(entity),
|
||||
PatternTerm::Literal(slot),
|
||||
PatternTerm::Variable(var),
|
||||
) => {
|
||||
if let Some(card) = self.memvid.get_current_memory(entity, slot) {
|
||||
let mut result =
|
||||
GraphMatchResult::new(entity.clone(), vec![card.source_frame_id], 1.0);
|
||||
result.bind(var, card.value.clone());
|
||||
results.push(result);
|
||||
}
|
||||
}
|
||||
|
||||
// Pattern: "entity" :slot "value" - check if entity has this exact value
|
||||
(
|
||||
PatternTerm::Literal(entity),
|
||||
PatternTerm::Literal(slot),
|
||||
PatternTerm::Literal(value),
|
||||
) => {
|
||||
if let Some(card) = self.memvid.get_current_memory(entity, slot) {
|
||||
if card.value.to_lowercase().contains(&value.to_lowercase()) {
|
||||
let result =
|
||||
GraphMatchResult::new(entity.clone(), vec![card.source_frame_id], 1.0);
|
||||
results.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_ => {
|
||||
// Other patterns not yet implemented
|
||||
}
|
||||
}
|
||||
|
||||
results
|
||||
}
|
||||
|
||||
/// Get frame IDs from graph matches for use in vector search filtering.
|
||||
#[must_use]
|
||||
pub fn get_candidate_frames(&self, matches: &[GraphMatchResult]) -> Vec<FrameId> {
|
||||
let mut frame_ids: Vec<FrameId> = matches
|
||||
.iter()
|
||||
.flat_map(|m| m.frame_ids.iter().copied())
|
||||
.collect();
|
||||
frame_ids.sort_unstable();
|
||||
frame_ids.dedup();
|
||||
frame_ids
|
||||
}
|
||||
|
||||
/// Get matched entities for context.
|
||||
#[must_use]
|
||||
pub fn get_matched_entities(&self, matches: &[GraphMatchResult]) -> HashMap<FrameId, String> {
|
||||
let mut map = HashMap::new();
|
||||
for m in matches {
|
||||
for &fid in &m.frame_ids {
|
||||
map.insert(fid, m.entity.clone());
|
||||
}
|
||||
}
|
||||
map
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute a hybrid search: graph filter + vector ranking.
|
||||
pub fn hybrid_search(memvid: &mut Memvid, plan: &QueryPlan) -> Result<Vec<HybridSearchHit>> {
|
||||
match plan {
|
||||
QueryPlan::VectorOnly {
|
||||
query_text, top_k, ..
|
||||
} => {
|
||||
// Fall back to regular lexical search
|
||||
let query = query_text.as_deref().unwrap_or("");
|
||||
let request = SearchRequest {
|
||||
query: query.to_string(),
|
||||
top_k: *top_k,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: crate::types::AclEnforcementMode::Audit,
|
||||
};
|
||||
let response = memvid.search(request)?;
|
||||
Ok(response
|
||||
.hits
|
||||
.iter()
|
||||
.map(|h| {
|
||||
let score = h.score.unwrap_or(0.0);
|
||||
HybridSearchHit {
|
||||
frame_id: h.frame_id,
|
||||
score,
|
||||
graph_score: 0.0,
|
||||
vector_score: score,
|
||||
matched_entity: None,
|
||||
preview: Some(h.text.clone()),
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
QueryPlan::GraphOnly { pattern, limit } => {
|
||||
let matcher = GraphMatcher::new(memvid);
|
||||
let matches = matcher.execute(pattern);
|
||||
|
||||
Ok(matches
|
||||
.into_iter()
|
||||
.take(*limit)
|
||||
.map(|m| HybridSearchHit {
|
||||
frame_id: m.frame_ids.first().copied().unwrap_or(0),
|
||||
score: m.confidence,
|
||||
graph_score: m.confidence,
|
||||
vector_score: 0.0,
|
||||
matched_entity: Some(m.entity),
|
||||
preview: None,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
QueryPlan::Hybrid {
|
||||
graph_filter,
|
||||
query_text,
|
||||
top_k,
|
||||
..
|
||||
} => {
|
||||
// Step 1: Execute graph pattern to get candidate frames
|
||||
let matcher = GraphMatcher::new(memvid);
|
||||
let matches = matcher.execute(graph_filter);
|
||||
let entity_map = matcher.get_matched_entities(&matches);
|
||||
let candidate_frames = matcher.get_candidate_frames(&matches);
|
||||
|
||||
if candidate_frames.is_empty() {
|
||||
// No graph matches - fall back to lexical search
|
||||
let query = query_text.as_deref().unwrap_or("");
|
||||
let request = SearchRequest {
|
||||
query: query.to_string(),
|
||||
top_k: *top_k,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: crate::types::AclEnforcementMode::Audit,
|
||||
};
|
||||
let response = memvid.search(request)?;
|
||||
return Ok(response
|
||||
.hits
|
||||
.iter()
|
||||
.map(|h| {
|
||||
let score = h.score.unwrap_or(0.0);
|
||||
HybridSearchHit {
|
||||
frame_id: h.frame_id,
|
||||
score,
|
||||
graph_score: 0.0,
|
||||
vector_score: score,
|
||||
matched_entity: None,
|
||||
preview: Some(h.text.clone()),
|
||||
}
|
||||
})
|
||||
.collect());
|
||||
}
|
||||
|
||||
// Step 2: Return graph matches directly with frame previews
|
||||
// Graph matching already found the relevant frames - return them
|
||||
let mut hybrid_hits: Vec<HybridSearchHit> = Vec::new();
|
||||
|
||||
for &frame_id in &candidate_frames {
|
||||
let matched_entity = entity_map.get(&frame_id).cloned();
|
||||
|
||||
// Get frame preview if possible
|
||||
let preview = memvid.frame_preview_by_id(frame_id).ok();
|
||||
|
||||
hybrid_hits.push(HybridSearchHit {
|
||||
frame_id,
|
||||
score: 1.0, // Graph match score
|
||||
graph_score: 1.0,
|
||||
vector_score: 0.0,
|
||||
matched_entity,
|
||||
preview,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(hybrid_hits.into_iter().take(*top_k).collect())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_query_planner_detects_location() {
|
||||
let planner = QueryPlanner::new();
|
||||
let plan = planner.plan("who lives in San Francisco", 10);
|
||||
|
||||
match plan {
|
||||
QueryPlan::Hybrid { graph_filter, .. } => {
|
||||
assert!(!graph_filter.is_empty());
|
||||
let triple = &graph_filter.triples[0];
|
||||
assert!(matches!(&triple.predicate, PatternTerm::Literal(s) if s == "location"));
|
||||
}
|
||||
_ => panic!("Expected hybrid plan for location query"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_planner_detects_workplace() {
|
||||
let planner = QueryPlanner::new();
|
||||
let plan = planner.plan("who works at Google", 10);
|
||||
|
||||
match plan {
|
||||
QueryPlan::Hybrid { graph_filter, .. } => {
|
||||
assert!(!graph_filter.is_empty());
|
||||
let triple = &graph_filter.triples[0];
|
||||
assert!(matches!(&triple.predicate, PatternTerm::Literal(s) if s == "workplace"));
|
||||
}
|
||||
_ => panic!("Expected hybrid plan for workplace query"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_planner_possessive() {
|
||||
let planner = QueryPlanner::new();
|
||||
let plan = planner.plan("what is alice's employer", 10);
|
||||
|
||||
match plan {
|
||||
QueryPlan::Hybrid { graph_filter, .. } => {
|
||||
assert!(!graph_filter.is_empty());
|
||||
let triple = &graph_filter.triples[0];
|
||||
assert!(matches!(&triple.subject, PatternTerm::Literal(s) if s == "alice"));
|
||||
}
|
||||
_ => panic!("Expected hybrid plan for possessive query"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_value() {
|
||||
assert_eq!(extract_value("San Francisco and"), "San Francisco");
|
||||
assert_eq!(extract_value("Google who"), "Google");
|
||||
assert_eq!(extract_value("New York City"), "New York City");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_possessive() {
|
||||
assert_eq!(
|
||||
extract_possessive_query("what is alice's job"),
|
||||
Some(("alice".to_string(), "workplace".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
extract_possessive_query("bob's location"),
|
||||
Some(("bob".to_string(), "location".to_string()))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
use std::{
|
||||
convert::TryInto,
|
||||
io::{Read, Seek, SeekFrom, Write},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
constants::{HEADER_SIZE, MAGIC, SPEC_MAJOR, SPEC_MINOR, WAL_OFFSET},
|
||||
error::{MemvidError, Result},
|
||||
types::Header,
|
||||
};
|
||||
|
||||
const VERSION_OFFSET: usize = 4;
|
||||
const SPEC_BYTES_OFFSET: usize = 6;
|
||||
const FOOTER_OFFSET_POS: usize = 8;
|
||||
const WAL_OFFSET_POS: usize = 16;
|
||||
const WAL_SIZE_POS: usize = 24;
|
||||
const WAL_CHECKPOINT_POS: usize = 32;
|
||||
const WAL_SEQUENCE_POS: usize = 40;
|
||||
const TOC_CHECKSUM_POS: usize = 48;
|
||||
const TOC_CHECKSUM_END: usize = 80;
|
||||
// Legacy lock metadata occupied bytes 80..140 within the header padding.
|
||||
const LEGACY_LOCK_REGION_START: usize = TOC_CHECKSUM_END;
|
||||
const LEGACY_LOCK_REGION_END: usize = LEGACY_LOCK_REGION_START + 60;
|
||||
const EXPECTED_VERSION: u16 = ((SPEC_MAJOR as u16) << 8) | SPEC_MINOR as u16;
|
||||
|
||||
/// Deterministic encoder/decoder for the fixed-size header region.
|
||||
pub struct HeaderCodec;
|
||||
|
||||
impl HeaderCodec {
|
||||
/// Writes the header back to the beginning of the file, zero-filling any unused bytes.
|
||||
pub fn write<W: Write + Seek>(mut writer: W, header: &Header) -> Result<()> {
|
||||
let bytes = Self::encode(header)?;
|
||||
writer.seek(SeekFrom::Start(0))?;
|
||||
writer.write_all(&bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reads and decodes the header from the start of the file. If legacy lock metadata is present
|
||||
/// in the reserved padding, it is cleared in-place before decoding to maintain forward
|
||||
/// compatibility with older files.
|
||||
pub fn read<R: Read + Write + Seek>(mut reader: R) -> Result<Header> {
|
||||
let mut buf = [0u8; HEADER_SIZE];
|
||||
reader.seek(SeekFrom::Start(0))?;
|
||||
reader.read_exact(&mut buf)?;
|
||||
if clear_legacy_lock_metadata(&mut buf) {
|
||||
reader.seek(SeekFrom::Start(0))?;
|
||||
reader.write_all(&buf)?;
|
||||
reader.flush()?;
|
||||
}
|
||||
Self::decode(&buf)
|
||||
}
|
||||
|
||||
/// Encodes a header into the canonical 4 KB byte representation.
|
||||
pub fn encode(header: &Header) -> Result<[u8; HEADER_SIZE]> {
|
||||
if header.magic != MAGIC {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "magic mismatch".into(),
|
||||
});
|
||||
}
|
||||
if header.version != EXPECTED_VERSION {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "unsupported version".into(),
|
||||
});
|
||||
}
|
||||
if header.wal_offset < WAL_OFFSET {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "wal_offset precedes data region".into(),
|
||||
});
|
||||
}
|
||||
if header.wal_size == 0 {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "wal_size must be non-zero".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut buf = [0u8; HEADER_SIZE];
|
||||
buf[..MAGIC.len()].copy_from_slice(&header.magic);
|
||||
buf[VERSION_OFFSET..VERSION_OFFSET + 2].copy_from_slice(&header.version.to_le_bytes());
|
||||
buf[SPEC_BYTES_OFFSET] = SPEC_MAJOR;
|
||||
buf[SPEC_BYTES_OFFSET + 1] = SPEC_MINOR;
|
||||
buf[FOOTER_OFFSET_POS..FOOTER_OFFSET_POS + 8]
|
||||
.copy_from_slice(&header.footer_offset.to_le_bytes());
|
||||
buf[WAL_OFFSET_POS..WAL_OFFSET_POS + 8].copy_from_slice(&header.wal_offset.to_le_bytes());
|
||||
buf[WAL_SIZE_POS..WAL_SIZE_POS + 8].copy_from_slice(&header.wal_size.to_le_bytes());
|
||||
buf[WAL_CHECKPOINT_POS..WAL_CHECKPOINT_POS + 8]
|
||||
.copy_from_slice(&header.wal_checkpoint_pos.to_le_bytes());
|
||||
buf[WAL_SEQUENCE_POS..WAL_SEQUENCE_POS + 8]
|
||||
.copy_from_slice(&header.wal_sequence.to_le_bytes());
|
||||
buf[TOC_CHECKSUM_POS..TOC_CHECKSUM_END].copy_from_slice(&header.toc_checksum);
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Decodes the canonical header bytes into a strongly typed struct after validation.
|
||||
pub fn decode(bytes: &[u8; HEADER_SIZE]) -> Result<Header> {
|
||||
// Extract fixed-size arrays from the header buffer
|
||||
// All indices are compile-time constants, so these slices are guaranteed to fit
|
||||
let magic: [u8; 4] = extract_array(bytes, 0)?;
|
||||
if magic != MAGIC {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "magic mismatch".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let version = u16::from_le_bytes(extract_array(bytes, VERSION_OFFSET)?);
|
||||
if version != EXPECTED_VERSION {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "unsupported version".into(),
|
||||
});
|
||||
}
|
||||
|
||||
if bytes[SPEC_BYTES_OFFSET] != SPEC_MAJOR || bytes[SPEC_BYTES_OFFSET + 1] != SPEC_MINOR {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "spec byte mismatch".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let footer_offset = u64::from_le_bytes(extract_array(bytes, FOOTER_OFFSET_POS)?);
|
||||
let wal_offset = u64::from_le_bytes(extract_array(bytes, WAL_OFFSET_POS)?);
|
||||
if wal_offset < WAL_OFFSET {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "wal_offset precedes data region".into(),
|
||||
});
|
||||
}
|
||||
let wal_size = u64::from_le_bytes(extract_array(bytes, WAL_SIZE_POS)?);
|
||||
if wal_size == 0 {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "wal_size must be non-zero".into(),
|
||||
});
|
||||
}
|
||||
let wal_checkpoint_pos = u64::from_le_bytes(extract_array(bytes, WAL_CHECKPOINT_POS)?);
|
||||
let wal_sequence = u64::from_le_bytes(extract_array(bytes, WAL_SEQUENCE_POS)?);
|
||||
let toc_checksum: [u8; 32] = extract_array(bytes, TOC_CHECKSUM_POS)?;
|
||||
|
||||
Ok(Header {
|
||||
magic,
|
||||
version,
|
||||
footer_offset,
|
||||
wal_offset,
|
||||
wal_size,
|
||||
wal_checkpoint_pos,
|
||||
wal_sequence,
|
||||
toc_checksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Extracts a fixed-size array from a byte slice at the given offset.
|
||||
/// Returns an error if the slice is too short (should never happen with valid headers).
|
||||
#[inline]
|
||||
fn extract_array<const N: usize>(bytes: &[u8], offset: usize) -> Result<[u8; N]> {
|
||||
bytes
|
||||
.get(offset..offset + N)
|
||||
.and_then(|s| s.try_into().ok())
|
||||
.ok_or_else(|| MemvidError::InvalidHeader {
|
||||
reason: "header truncated".into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn clear_legacy_lock_metadata(buf: &mut [u8; HEADER_SIZE]) -> bool {
|
||||
let region = &mut buf[LEGACY_LOCK_REGION_START..LEGACY_LOCK_REGION_END];
|
||||
if region.iter().any(|byte| *byte != 0) {
|
||||
region.fill(0);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
|
||||
fn sample_header() -> Header {
|
||||
Header {
|
||||
magic: MAGIC,
|
||||
version: EXPECTED_VERSION,
|
||||
footer_offset: 1_048_576,
|
||||
wal_offset: WAL_OFFSET,
|
||||
wal_size: 4 * 1024 * 1024,
|
||||
wal_checkpoint_pos: 0,
|
||||
wal_sequence: 42,
|
||||
toc_checksum: [0xAB; 32],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_encode_decode() {
|
||||
let header = sample_header();
|
||||
let encoded = HeaderCodec::encode(&header).expect("encode header");
|
||||
let decoded = HeaderCodec::decode(&encoded).expect("decode header");
|
||||
assert_eq!(decoded.magic, MAGIC);
|
||||
assert_eq!(decoded.version, EXPECTED_VERSION);
|
||||
assert_eq!(decoded.footer_offset, header.footer_offset);
|
||||
assert_eq!(decoded.wal_offset, WAL_OFFSET);
|
||||
assert_eq!(decoded.toc_checksum, header.toc_checksum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_write_from_cursor() {
|
||||
let header = sample_header();
|
||||
let mut cursor = Cursor::new(vec![0u8; HEADER_SIZE]);
|
||||
HeaderCodec::write(&mut cursor, &header).expect("write header");
|
||||
cursor.set_position(0);
|
||||
let decoded = HeaderCodec::read(&mut cursor).expect("read header");
|
||||
assert_eq!(decoded.wal_size, header.wal_size);
|
||||
assert_eq!(decoded.wal_sequence, header.wal_sequence);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clears_legacy_lock_metadata() {
|
||||
let header = sample_header();
|
||||
let mut encoded = HeaderCodec::encode(&header).expect("encode header");
|
||||
encoded[LEGACY_LOCK_REGION_START..LEGACY_LOCK_REGION_END].fill(0xAA);
|
||||
let mut cursor = Cursor::new(encoded.to_vec());
|
||||
HeaderCodec::read(&mut cursor).expect("read header with legacy metadata");
|
||||
let sanitized = cursor.into_inner();
|
||||
assert!(
|
||||
sanitized[LEGACY_LOCK_REGION_START..LEGACY_LOCK_REGION_END]
|
||||
.iter()
|
||||
.all(|byte| *byte == 0)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reject_invalid_magic() {
|
||||
let mut header = sample_header();
|
||||
header.magic = *b"BAD!";
|
||||
let err = HeaderCodec::encode(&header).expect_err("should fail");
|
||||
matches!(err, MemvidError::InvalidHeader { .. });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reject_short_wal_size() {
|
||||
let mut header = sample_header();
|
||||
header.wal_size = 0;
|
||||
let err = HeaderCodec::encode(&header).expect_err("should fail");
|
||||
matches!(err, MemvidError::InvalidHeader { .. });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reject_decoding_with_bad_version() {
|
||||
let header = sample_header();
|
||||
let mut encoded = HeaderCodec::encode(&header).expect("encode header");
|
||||
encoded[VERSION_OFFSET] = 0xFF;
|
||||
let err = HeaderCodec::decode(&encoded).expect_err("decode should fail");
|
||||
matches!(err, MemvidError::InvalidHeader { .. });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
//! Durable manifest WAL for append-only segment descriptors (parallel ingestion).
|
||||
//!
|
||||
//! This WAL stores [`IndexSegmentRef`] entries using a simple length-prefixed format:
|
||||
//!
|
||||
//! ```text
|
||||
//! file header: [ magic (8 bytes) | version (u32) ]
|
||||
//! record: [ len (u32 LE) | checksum (32 bytes) | payload (len bytes bincode) ]
|
||||
//! ```
|
||||
//!
|
||||
//! On startup we validate the header, stream all intact records, and truncate any
|
||||
//! trailing partial record caused by a crash. Writes append records and rely on the
|
||||
//! caller to `flush()` when durability is required.
|
||||
|
||||
use std::{
|
||||
fs::{File, OpenOptions},
|
||||
io::{ErrorKind, Read, Seek, SeekFrom, Write},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use bincode::serde::{decode_from_slice, encode_to_vec};
|
||||
use blake3::hash;
|
||||
|
||||
use crate::{Result, error::MemvidError, types::IndexSegmentRef};
|
||||
|
||||
const FILE_MAGIC: [u8; 8] = *b"MVSGWAL1";
|
||||
const FILE_VERSION: u32 = 1;
|
||||
const FILE_HEADER_SIZE: usize = FILE_MAGIC.len() + 4;
|
||||
const RECORD_HEADER_SIZE: usize = 4 + 32; // length (u32) + checksum
|
||||
const MAX_RECORD_BYTES: usize = 4 * 1024 * 1024; // 4 MiB segments metadata upper bound
|
||||
|
||||
fn wal_config() -> impl bincode::config::Config {
|
||||
bincode::config::standard()
|
||||
.with_fixed_int_encoding()
|
||||
.with_little_endian()
|
||||
}
|
||||
|
||||
/// Crash-safe manifest log backing the upcoming parallel segment builder.
|
||||
pub struct ManifestWal {
|
||||
#[allow(dead_code)]
|
||||
path: PathBuf,
|
||||
file: File,
|
||||
entries: Vec<IndexSegmentRef>,
|
||||
write_offset: u64,
|
||||
}
|
||||
|
||||
impl ManifestWal {
|
||||
/// Opens the WAL at `path`, creating it if necessary and replaying intact entries.
|
||||
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
|
||||
let path_buf = path.as_ref().to_path_buf();
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.open(&path_buf)?;
|
||||
|
||||
let mut wal = Self {
|
||||
path: path_buf,
|
||||
file,
|
||||
entries: Vec::new(),
|
||||
write_offset: FILE_HEADER_SIZE as u64,
|
||||
};
|
||||
wal.bootstrap()?;
|
||||
Ok(wal)
|
||||
}
|
||||
|
||||
/// Appends a batch of segment references.
|
||||
pub fn append_segments(&mut self, segments: &[IndexSegmentRef]) -> Result<()> {
|
||||
for segment in segments {
|
||||
self.append_one(segment)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns a copy of the replayed entries.
|
||||
pub fn replay(&self) -> Result<Vec<IndexSegmentRef>> {
|
||||
Ok(self.entries.clone())
|
||||
}
|
||||
|
||||
/// Flushes the WAL to durable storage (fsync).
|
||||
pub fn flush(&mut self) -> Result<()> {
|
||||
self.file.sync_data()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Truncates the WAL back to just the header after entries are materialised.
|
||||
pub fn truncate(&mut self) -> Result<()> {
|
||||
self.truncate_at(FILE_HEADER_SIZE as u64)?;
|
||||
self.entries.clear();
|
||||
self.write_offset = FILE_HEADER_SIZE as u64;
|
||||
self.file.seek(SeekFrom::Start(self.write_offset))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn append_one(&mut self, segment: &IndexSegmentRef) -> Result<()> {
|
||||
let payload = encode_to_vec(segment, wal_config())?;
|
||||
if payload.len() > MAX_RECORD_BYTES {
|
||||
return Err(MemvidError::CheckpointFailed {
|
||||
reason: "manifest wal payload exceeds limit".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let checksum = hash(&payload);
|
||||
self.file.seek(SeekFrom::Start(self.write_offset))?;
|
||||
// Safe: validated payload.len() <= MAX_RECORD_BYTES (4MB) on line 96
|
||||
self.file
|
||||
.write_all(&(u32::try_from(payload.len()).unwrap_or(u32::MAX)).to_le_bytes())?;
|
||||
self.file.write_all(checksum.as_bytes())?;
|
||||
self.file.write_all(&payload)?;
|
||||
|
||||
self.write_offset += (RECORD_HEADER_SIZE + payload.len()) as u64;
|
||||
self.entries.push(segment.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bootstrap(&mut self) -> Result<()> {
|
||||
self.ensure_header()?;
|
||||
let (entries, offset) = self.scan_entries()?;
|
||||
self.entries = entries;
|
||||
self.write_offset = offset;
|
||||
self.file.seek(SeekFrom::Start(self.write_offset))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_header(&mut self) -> Result<()> {
|
||||
let len = self.file.metadata()?.len();
|
||||
if len < FILE_HEADER_SIZE as u64 {
|
||||
self.file.set_len(0)?;
|
||||
self.file.seek(SeekFrom::Start(0))?;
|
||||
self.file.write_all(&FILE_MAGIC)?;
|
||||
self.file.write_all(&FILE_VERSION.to_le_bytes())?;
|
||||
self.file.sync_data()?;
|
||||
self.write_offset = FILE_HEADER_SIZE as u64;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut magic = [0u8; FILE_MAGIC.len()];
|
||||
self.file.seek(SeekFrom::Start(0))?;
|
||||
self.file.read_exact(&mut magic)?;
|
||||
if magic != FILE_MAGIC {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "manifest wal magic mismatch".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut version_bytes = [0u8; 4];
|
||||
self.file.read_exact(&mut version_bytes)?;
|
||||
let version = u32::from_le_bytes(version_bytes);
|
||||
if version != FILE_VERSION {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "manifest wal version mismatch".into(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn scan_entries(&mut self) -> Result<(Vec<IndexSegmentRef>, u64)> {
|
||||
let mut entries = Vec::new();
|
||||
let mut offset = FILE_HEADER_SIZE as u64;
|
||||
let file_len = self.file.metadata()?.len();
|
||||
|
||||
while offset < file_len {
|
||||
if file_len - offset < RECORD_HEADER_SIZE as u64 {
|
||||
self.truncate_at(offset)?;
|
||||
break;
|
||||
}
|
||||
|
||||
self.file.seek(SeekFrom::Start(offset))?;
|
||||
let mut header = [0u8; RECORD_HEADER_SIZE];
|
||||
if let Err(err) = self.file.read_exact(&mut header) {
|
||||
if err.kind() == ErrorKind::UnexpectedEof {
|
||||
self.truncate_at(offset)?;
|
||||
break;
|
||||
}
|
||||
return Err(err.into());
|
||||
}
|
||||
offset += RECORD_HEADER_SIZE as u64;
|
||||
|
||||
let payload_len = u32::from_le_bytes(header[..4].try_into().unwrap()) as u64;
|
||||
if payload_len == 0 {
|
||||
return Err(MemvidError::ManifestWalCorrupted {
|
||||
offset: offset - RECORD_HEADER_SIZE as u64,
|
||||
reason: "record length is zero",
|
||||
});
|
||||
}
|
||||
if payload_len as usize > MAX_RECORD_BYTES {
|
||||
return Err(MemvidError::ManifestWalCorrupted {
|
||||
offset: offset - RECORD_HEADER_SIZE as u64,
|
||||
reason: "record length exceeds limit",
|
||||
});
|
||||
}
|
||||
if offset + payload_len > file_len {
|
||||
offset -= RECORD_HEADER_SIZE as u64;
|
||||
self.truncate_at(offset)?;
|
||||
break;
|
||||
}
|
||||
|
||||
// Safe: validated payload_len <= MAX_RECORD_BYTES (4MB) on line 184
|
||||
let mut payload = vec![0u8; payload_len as usize];
|
||||
if let Err(err) = self.file.read_exact(&mut payload) {
|
||||
if err.kind() == ErrorKind::UnexpectedEof {
|
||||
offset -= RECORD_HEADER_SIZE as u64;
|
||||
self.truncate_at(offset)?;
|
||||
break;
|
||||
}
|
||||
return Err(err.into());
|
||||
}
|
||||
offset += payload_len;
|
||||
|
||||
let digest = hash(&payload);
|
||||
if digest.as_bytes() != &header[4..] {
|
||||
return Err(MemvidError::ChecksumMismatch {
|
||||
context: "manifest_wal",
|
||||
});
|
||||
}
|
||||
|
||||
let (segment, consumed) = decode_from_slice(&payload, wal_config())?;
|
||||
if consumed != payload.len() {
|
||||
return Err(MemvidError::ManifestWalCorrupted {
|
||||
offset: offset - payload_len,
|
||||
reason: "payload contains trailing bytes",
|
||||
});
|
||||
}
|
||||
entries.push(segment);
|
||||
}
|
||||
|
||||
Ok((entries, offset))
|
||||
}
|
||||
|
||||
fn truncate_at(&mut self, offset: u64) -> Result<()> {
|
||||
self.file.set_len(offset)?;
|
||||
self.file.sync_data()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{SegmentCommon, SegmentKind, SegmentSpan, SegmentStats};
|
||||
use tempfile::tempdir;
|
||||
|
||||
fn sample_segment(id: u64) -> IndexSegmentRef {
|
||||
let mut common = SegmentCommon::new(id, id * 10, 128, [id as u8; 32]);
|
||||
common.span = Some(SegmentSpan {
|
||||
frame_start: id * 100,
|
||||
frame_end: id * 100 + 10,
|
||||
..SegmentSpan::default()
|
||||
});
|
||||
IndexSegmentRef {
|
||||
kind: SegmentKind::Vector,
|
||||
common,
|
||||
stats: SegmentStats {
|
||||
doc_count: 1,
|
||||
vector_count: 10,
|
||||
time_entries: 0,
|
||||
bytes_uncompressed: 2048,
|
||||
build_micros: 42,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn append_and_replay_roundtrip() -> Result<()> {
|
||||
let dir = tempdir()?;
|
||||
let path = dir.path().join("wal.mv2");
|
||||
{
|
||||
let mut wal = ManifestWal::open(&path)?;
|
||||
wal.append_segments(&[sample_segment(1), sample_segment(2)])?;
|
||||
wal.flush()?;
|
||||
}
|
||||
let wal = ManifestWal::open(&path)?;
|
||||
let entries = wal.replay()?;
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert_eq!(entries[0].common.segment_id, 1);
|
||||
assert_eq!(entries[1].common.segment_id, 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncates_partial_record() -> Result<()> {
|
||||
let dir = tempdir()?;
|
||||
let path = dir.path().join("wal.mv2");
|
||||
{
|
||||
let mut wal = ManifestWal::open(&path)?;
|
||||
wal.append_segments(&[sample_segment(7)])?;
|
||||
wal.flush()?;
|
||||
}
|
||||
|
||||
// Simulate crash mid-record by chopping a few bytes.
|
||||
let file = OpenOptions::new().read(true).write(true).open(&path)?;
|
||||
let len = file.metadata()?.len();
|
||||
file.set_len(len.saturating_sub(5))?;
|
||||
|
||||
let wal = ManifestWal::open(&path)?;
|
||||
let entries = wal.replay()?;
|
||||
assert!(entries.is_empty(), "partial record should be dropped");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//! Low-level IO primitives for interacting with `.mv2` files.
|
||||
|
||||
pub mod header;
|
||||
#[cfg(feature = "parallel_segments")]
|
||||
pub mod manifest_wal;
|
||||
#[cfg(feature = "temporal_track")]
|
||||
pub mod temporal_index;
|
||||
pub mod time_index;
|
||||
pub mod wal;
|
||||
|
||||
pub use wal::{EmbeddedWal, WalRecord, WalStats};
|
||||
@@ -0,0 +1,556 @@
|
||||
//! Binary encode/decode helpers for the temporal mentions track (feature gated).
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::ops::Range;
|
||||
|
||||
use blake3::Hasher;
|
||||
|
||||
use crate::constants::{TEMPORAL_TRACK_MAGIC, TEMPORAL_TRACK_VERSION};
|
||||
use crate::error::{MemvidError, Result};
|
||||
use crate::types::{
|
||||
AnchorSource, TemporalAnchor, TemporalMention, TemporalMentionFlags, TemporalMentionKind,
|
||||
TemporalTrack,
|
||||
};
|
||||
|
||||
const HEADER_SIZE: usize = 56;
|
||||
const CHECKSUM_OFFSET: usize = 24;
|
||||
const MENTION_RECORD_SIZE: usize = 32; // padded to 32 bytes for alignment
|
||||
const ANCHOR_RECORD_SIZE: usize = 24;
|
||||
const MAX_TEMPORAL_TRACK_BYTES: u64 = 1 << 34; // 16 GiB safety ceiling
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct RawMention {
|
||||
ts_utc: i64,
|
||||
frame_id: u64,
|
||||
byte_start: u32,
|
||||
byte_len: u32,
|
||||
kind: u8,
|
||||
confidence: u16,
|
||||
tz_hint_minutes: i16,
|
||||
flags: u8,
|
||||
reserved: u16,
|
||||
}
|
||||
|
||||
impl RawMention {
|
||||
fn from_high_level(mention: &TemporalMention) -> Self {
|
||||
Self {
|
||||
ts_utc: mention.ts_utc,
|
||||
frame_id: mention.frame_id,
|
||||
byte_start: mention.byte_start,
|
||||
byte_len: mention.byte_len,
|
||||
kind: mention.kind.to_u8(),
|
||||
confidence: mention.confidence,
|
||||
tz_hint_minutes: mention.tz_hint_minutes,
|
||||
flags: mention.flags.0,
|
||||
reserved: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn encode(self) -> [u8; MENTION_RECORD_SIZE] {
|
||||
let mut out = [0u8; MENTION_RECORD_SIZE];
|
||||
out[0..8].copy_from_slice(&self.ts_utc.to_le_bytes());
|
||||
out[8..16].copy_from_slice(&self.frame_id.to_le_bytes());
|
||||
out[16..20].copy_from_slice(&self.byte_start.to_le_bytes());
|
||||
out[20..24].copy_from_slice(&self.byte_len.to_le_bytes());
|
||||
out[24] = self.kind;
|
||||
out[25..27].copy_from_slice(&self.confidence.to_le_bytes());
|
||||
out[27..29].copy_from_slice(&self.tz_hint_minutes.to_le_bytes());
|
||||
out[29] = self.flags;
|
||||
out[30..32].copy_from_slice(&self.reserved.to_le_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8]) -> Result<Self> {
|
||||
let ts_utc = i64::from_le_bytes(bytes[0..8].try_into().unwrap());
|
||||
let frame_id = u64::from_le_bytes(bytes[8..16].try_into().unwrap());
|
||||
let byte_start = u32::from_le_bytes(bytes[16..20].try_into().unwrap());
|
||||
let byte_len = u32::from_le_bytes(bytes[20..24].try_into().unwrap());
|
||||
let kind = bytes[24];
|
||||
let confidence = u16::from_le_bytes(bytes[25..27].try_into().unwrap());
|
||||
let tz_hint_minutes = i16::from_le_bytes(bytes[27..29].try_into().unwrap());
|
||||
let flags = bytes[29];
|
||||
let reserved = u16::from_le_bytes(bytes[30..32].try_into().unwrap());
|
||||
Ok(Self {
|
||||
ts_utc,
|
||||
frame_id,
|
||||
byte_start,
|
||||
byte_len,
|
||||
kind,
|
||||
confidence,
|
||||
tz_hint_minutes,
|
||||
flags,
|
||||
reserved,
|
||||
})
|
||||
}
|
||||
|
||||
fn into_high_level(self) -> Result<TemporalMention> {
|
||||
let kind =
|
||||
TemporalMentionKind::from_u8(self.kind).ok_or(MemvidError::InvalidTemporalTrack {
|
||||
reason: "unknown mention kind".into(),
|
||||
})?;
|
||||
Ok(TemporalMention::new(
|
||||
self.ts_utc,
|
||||
self.frame_id,
|
||||
self.byte_start,
|
||||
self.byte_len,
|
||||
kind,
|
||||
self.confidence,
|
||||
self.tz_hint_minutes,
|
||||
TemporalMentionFlags(self.flags),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct RawAnchor {
|
||||
frame_id: u64,
|
||||
anchor_ts: i64,
|
||||
source: u8,
|
||||
reserved: [u8; 7],
|
||||
}
|
||||
|
||||
impl RawAnchor {
|
||||
fn from_high_level(anchor: &TemporalAnchor) -> Self {
|
||||
Self {
|
||||
frame_id: anchor.frame_id,
|
||||
anchor_ts: anchor.anchor_ts,
|
||||
source: anchor.source as u8,
|
||||
reserved: [0u8; 7],
|
||||
}
|
||||
}
|
||||
|
||||
fn encode(self) -> [u8; ANCHOR_RECORD_SIZE] {
|
||||
let mut out = [0u8; ANCHOR_RECORD_SIZE];
|
||||
out[0..8].copy_from_slice(&self.frame_id.to_le_bytes());
|
||||
out[8..16].copy_from_slice(&self.anchor_ts.to_le_bytes());
|
||||
out[16] = self.source;
|
||||
out[17..24].copy_from_slice(&self.reserved);
|
||||
out
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8]) -> Result<Self> {
|
||||
let frame_id = u64::from_le_bytes(bytes[0..8].try_into().unwrap());
|
||||
let anchor_ts = i64::from_le_bytes(bytes[8..16].try_into().unwrap());
|
||||
let source = bytes[16];
|
||||
let mut reserved = [0u8; 7];
|
||||
reserved.copy_from_slice(&bytes[17..24]);
|
||||
Ok(Self {
|
||||
frame_id,
|
||||
anchor_ts,
|
||||
source,
|
||||
reserved,
|
||||
})
|
||||
}
|
||||
|
||||
fn into_high_level(self) -> Result<TemporalAnchor> {
|
||||
let source = match self.source {
|
||||
0 => AnchorSource::Explicit,
|
||||
1 => AnchorSource::FrameTimestamp,
|
||||
2 => AnchorSource::Metadata,
|
||||
3 => AnchorSource::IngestionClock,
|
||||
_ => {
|
||||
return Err(MemvidError::InvalidTemporalTrack {
|
||||
reason: "unknown anchor source".into(),
|
||||
});
|
||||
}
|
||||
};
|
||||
Ok(TemporalAnchor::new(self.frame_id, self.anchor_ts, source))
|
||||
}
|
||||
}
|
||||
|
||||
fn write_header<W: Write + Seek>(
|
||||
writer: &mut W,
|
||||
entry_count: u64,
|
||||
anchor_count: u64,
|
||||
flags: u16,
|
||||
) -> Result<(u64, [u8; HEADER_SIZE])> {
|
||||
let offset = writer.seek(SeekFrom::Current(0))?;
|
||||
let mut header = [0u8; HEADER_SIZE];
|
||||
header[0..4].copy_from_slice(&TEMPORAL_TRACK_MAGIC);
|
||||
header[4..6].copy_from_slice(&TEMPORAL_TRACK_VERSION.to_le_bytes());
|
||||
header[6..8].copy_from_slice(&flags.to_le_bytes());
|
||||
header[8..16].copy_from_slice(&entry_count.to_le_bytes());
|
||||
header[16..24].copy_from_slice(&anchor_count.to_le_bytes());
|
||||
// checksum stays zero until finalisation
|
||||
writer.write_all(&header)?;
|
||||
Ok((offset, header))
|
||||
}
|
||||
|
||||
fn patch_checksum<W: Write + Seek>(
|
||||
writer: &mut W,
|
||||
header_offset: u64,
|
||||
checksum: &[u8; 32],
|
||||
restore_pos: u64,
|
||||
) -> Result<()> {
|
||||
writer.seek(SeekFrom::Start(header_offset + CHECKSUM_OFFSET as u64))?;
|
||||
writer.write_all(checksum)?;
|
||||
writer.seek(SeekFrom::Start(restore_pos))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Serialises the provided mentions + anchors into the temporal track region.
|
||||
pub fn append_track<W: Write + Seek>(
|
||||
writer: &mut W,
|
||||
mentions: &mut [TemporalMention],
|
||||
anchors: &mut [TemporalAnchor],
|
||||
flags: u32,
|
||||
) -> Result<(u64, u64, [u8; 32])> {
|
||||
#[cfg(test)]
|
||||
println!(
|
||||
"append_track: mentions={}, anchors={}, flags={}",
|
||||
mentions.len(),
|
||||
anchors.len(),
|
||||
flags
|
||||
);
|
||||
mentions.sort_by(mention_cmp);
|
||||
anchors.sort_by_key(|anchor| anchor.frame_id);
|
||||
validate_mentions_sorted(mentions)?;
|
||||
validate_anchors_sorted(anchors)?;
|
||||
|
||||
let entry_count = mentions.len() as u64;
|
||||
let anchor_count = anchors.len() as u64;
|
||||
let header_flags = flags as u16;
|
||||
|
||||
let (header_offset, mut header) =
|
||||
write_header(writer, entry_count, anchor_count, header_flags)?;
|
||||
header[CHECKSUM_OFFSET..CHECKSUM_OFFSET + 32].fill(0);
|
||||
|
||||
let mut hasher = Hasher::new();
|
||||
hasher.update(&header);
|
||||
|
||||
for mention in mentions.iter().map(RawMention::from_high_level) {
|
||||
let encoded = mention.encode();
|
||||
writer.write_all(&encoded)?;
|
||||
hasher.update(&encoded);
|
||||
}
|
||||
|
||||
for anchor in anchors.iter().map(RawAnchor::from_high_level) {
|
||||
let encoded = anchor.encode();
|
||||
writer.write_all(&encoded)?;
|
||||
hasher.update(&encoded);
|
||||
}
|
||||
|
||||
let checksum = *hasher.finalize().as_bytes();
|
||||
let end = writer.seek(SeekFrom::Current(0))?;
|
||||
patch_checksum(writer, header_offset, &checksum, end)?;
|
||||
|
||||
Ok((header_offset, end - header_offset, checksum))
|
||||
}
|
||||
|
||||
/// Reads and validates a temporal track from disk.
|
||||
pub fn read_track<R: Read + Seek>(
|
||||
reader: &mut R,
|
||||
offset: u64,
|
||||
length: u64,
|
||||
) -> Result<TemporalTrack> {
|
||||
if length > MAX_TEMPORAL_TRACK_BYTES {
|
||||
return Err(MemvidError::InvalidTemporalTrack {
|
||||
reason: "length exceeds supported limit".into(),
|
||||
});
|
||||
}
|
||||
|
||||
if length < HEADER_SIZE as u64 {
|
||||
return Err(MemvidError::InvalidTemporalTrack {
|
||||
reason: "length shorter than header".into(),
|
||||
});
|
||||
}
|
||||
|
||||
reader.seek(SeekFrom::Start(offset))?;
|
||||
let mut header = [0u8; HEADER_SIZE];
|
||||
reader.read_exact(&mut header)?;
|
||||
|
||||
if header[0..4] != TEMPORAL_TRACK_MAGIC {
|
||||
return Err(MemvidError::InvalidTemporalTrack {
|
||||
reason: "magic mismatch".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let version = u16::from_le_bytes(header[4..6].try_into().unwrap());
|
||||
if version > TEMPORAL_TRACK_VERSION {
|
||||
return Err(MemvidError::InvalidTemporalTrack {
|
||||
reason: "unsupported track version".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let flags = u16::from_le_bytes(header[6..8].try_into().unwrap()) as u32;
|
||||
let entry_count = u64::from_le_bytes(header[8..16].try_into().unwrap());
|
||||
let anchor_count = u64::from_le_bytes(header[16..24].try_into().unwrap());
|
||||
let checksum: [u8; 32] = header[CHECKSUM_OFFSET..CHECKSUM_OFFSET + 32]
|
||||
.try_into()
|
||||
.unwrap();
|
||||
|
||||
let expected_entries_bytes = entry_count.checked_mul(MENTION_RECORD_SIZE as u64).ok_or(
|
||||
MemvidError::InvalidTemporalTrack {
|
||||
reason: "entry count overflow".into(),
|
||||
},
|
||||
)?;
|
||||
let expected_anchor_bytes = anchor_count.checked_mul(ANCHOR_RECORD_SIZE as u64).ok_or(
|
||||
MemvidError::InvalidTemporalTrack {
|
||||
reason: "anchor count overflow".into(),
|
||||
},
|
||||
)?;
|
||||
|
||||
let total_expected = HEADER_SIZE as u64 + expected_entries_bytes + expected_anchor_bytes;
|
||||
if total_expected != length {
|
||||
return Err(MemvidError::InvalidTemporalTrack {
|
||||
reason: "length does not match declared counts".into(),
|
||||
});
|
||||
}
|
||||
|
||||
// Safe: length validated against MAX_TEMPORAL_TRACK_BYTES (16 GiB) on line 247
|
||||
// and HEADER_SIZE is constant, so result fits in usize
|
||||
let mut body = vec![0u8; (length - HEADER_SIZE as u64) as usize];
|
||||
reader.read_exact(&mut body)?;
|
||||
|
||||
let mut header_for_hash = header;
|
||||
header_for_hash[CHECKSUM_OFFSET..CHECKSUM_OFFSET + 32].fill(0);
|
||||
let mut hasher = Hasher::new();
|
||||
hasher.update(&header_for_hash);
|
||||
hasher.update(&body);
|
||||
let computed = *hasher.finalize().as_bytes();
|
||||
if computed != checksum {
|
||||
return Err(MemvidError::InvalidTemporalTrack {
|
||||
reason: "checksum mismatch".into(),
|
||||
});
|
||||
}
|
||||
|
||||
// Safe: counts validated by checked_mul and total_expected == length check above
|
||||
let mut mentions = Vec::with_capacity(entry_count as usize);
|
||||
let mut anchors = Vec::with_capacity(anchor_count as usize);
|
||||
|
||||
// Safe: validated by checked_mul overflow check on line 283
|
||||
let mentions_bytes = expected_entries_bytes as usize;
|
||||
for chunk in body[..mentions_bytes].chunks_exact(MENTION_RECORD_SIZE) {
|
||||
let raw = RawMention::decode(chunk)?;
|
||||
mentions.push(raw.into_high_level()?);
|
||||
}
|
||||
|
||||
for chunk in body[mentions_bytes..].chunks_exact(ANCHOR_RECORD_SIZE) {
|
||||
let raw = RawAnchor::decode(chunk)?;
|
||||
anchors.push(raw.into_high_level()?);
|
||||
}
|
||||
|
||||
validate_mentions_sorted(&mentions)?;
|
||||
validate_anchors_sorted(&anchors)?;
|
||||
|
||||
Ok(TemporalTrack {
|
||||
mentions,
|
||||
anchors,
|
||||
flags,
|
||||
})
|
||||
}
|
||||
|
||||
/// Computes the deterministic checksum used for manifest verification.
|
||||
pub fn calculate_checksum(
|
||||
mentions: &[TemporalMention],
|
||||
anchors: &[TemporalAnchor],
|
||||
flags: u32,
|
||||
) -> [u8; 32] {
|
||||
let mut sorted_mentions = mentions.to_vec();
|
||||
sorted_mentions.sort_by(mention_cmp);
|
||||
let mut sorted_anchors = anchors.to_vec();
|
||||
sorted_anchors.sort_by_key(|anchor| anchor.frame_id);
|
||||
|
||||
let mut header = [0u8; HEADER_SIZE];
|
||||
header[0..4].copy_from_slice(&TEMPORAL_TRACK_MAGIC);
|
||||
header[4..6].copy_from_slice(&TEMPORAL_TRACK_VERSION.to_le_bytes());
|
||||
header[6..8].copy_from_slice(&(flags as u16).to_le_bytes());
|
||||
header[8..16].copy_from_slice(&(sorted_mentions.len() as u64).to_le_bytes());
|
||||
header[16..24].copy_from_slice(&(sorted_anchors.len() as u64).to_le_bytes());
|
||||
|
||||
let mut hasher = Hasher::new();
|
||||
hasher.update(&header);
|
||||
|
||||
for mention in sorted_mentions.iter().map(RawMention::from_high_level) {
|
||||
hasher.update(&mention.encode());
|
||||
}
|
||||
|
||||
for anchor in sorted_anchors.iter().map(RawAnchor::from_high_level) {
|
||||
hasher.update(&anchor.encode());
|
||||
}
|
||||
|
||||
*hasher.finalize().as_bytes()
|
||||
}
|
||||
|
||||
/// Returns mention indices within `[start_utc, end_utc]` (inclusive).
|
||||
#[must_use]
|
||||
pub fn window(mentions: &[TemporalMention], start_utc: i64, end_utc: i64) -> Range<usize> {
|
||||
if mentions.is_empty() || start_utc > end_utc {
|
||||
return 0..0;
|
||||
}
|
||||
|
||||
let lower = mentions.partition_point(|mention| mention.ts_utc < start_utc);
|
||||
let upper = mentions.partition_point(|mention| mention.ts_utc <= end_utc);
|
||||
lower..upper
|
||||
}
|
||||
|
||||
fn mention_cmp(a: &TemporalMention, b: &TemporalMention) -> Ordering {
|
||||
a.ts_utc
|
||||
.cmp(&b.ts_utc)
|
||||
.then_with(|| a.frame_id.cmp(&b.frame_id))
|
||||
.then_with(|| a.byte_start.cmp(&b.byte_start))
|
||||
}
|
||||
|
||||
fn validate_mentions_sorted(mentions: &[TemporalMention]) -> Result<()> {
|
||||
if mentions
|
||||
.windows(2)
|
||||
.any(|pair| mention_cmp(&pair[0], &pair[1]) == Ordering::Greater)
|
||||
{
|
||||
return Err(MemvidError::InvalidTemporalTrack {
|
||||
reason: "mentions not sorted".into(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_anchors_sorted(anchors: &[TemporalAnchor]) -> Result<()> {
|
||||
if anchors
|
||||
.windows(2)
|
||||
.any(|pair| pair[0].frame_id >= pair[1].frame_id)
|
||||
{
|
||||
return Err(MemvidError::InvalidTemporalTrack {
|
||||
reason: "anchors not strictly increasing".into(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempfile;
|
||||
|
||||
#[test]
|
||||
fn append_and_read_roundtrip() {
|
||||
let mut mentions = vec![
|
||||
TemporalMention::new(
|
||||
10,
|
||||
1,
|
||||
0,
|
||||
5,
|
||||
TemporalMentionKind::Date,
|
||||
900,
|
||||
0,
|
||||
TemporalMentionFlags::empty(),
|
||||
),
|
||||
TemporalMention::new(
|
||||
20,
|
||||
1,
|
||||
6,
|
||||
4,
|
||||
TemporalMentionKind::DateTime,
|
||||
800,
|
||||
60,
|
||||
TemporalMentionFlags::empty(),
|
||||
),
|
||||
TemporalMention::new(
|
||||
20,
|
||||
2,
|
||||
0,
|
||||
3,
|
||||
TemporalMentionKind::RangeStart,
|
||||
750,
|
||||
0,
|
||||
TemporalMentionFlags(TemporalMentionFlags::HAS_RANGE),
|
||||
),
|
||||
];
|
||||
let mut anchors = vec![
|
||||
TemporalAnchor::new(1, 100, AnchorSource::FrameTimestamp),
|
||||
TemporalAnchor::new(3, 200, AnchorSource::Metadata),
|
||||
];
|
||||
|
||||
let mut file = tempfile().expect("temp");
|
||||
let (offset, length, checksum) =
|
||||
append_track(&mut file, &mut mentions, &mut anchors, 0).expect("append track");
|
||||
|
||||
let track = read_track(&mut file, offset, length).expect("read track");
|
||||
assert_eq!(track.mentions.len(), 3);
|
||||
assert_eq!(track.anchors.len(), 2);
|
||||
assert_eq!(track.flags, 0);
|
||||
|
||||
let expected_checksum = calculate_checksum(&track.mentions, &track.anchors, track.flags);
|
||||
assert_eq!(checksum, expected_checksum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_filters_correctly() {
|
||||
let mentions = vec![
|
||||
TemporalMention::new(
|
||||
10,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
TemporalMentionKind::Date,
|
||||
1000,
|
||||
0,
|
||||
TemporalMentionFlags::empty(),
|
||||
),
|
||||
TemporalMention::new(
|
||||
12,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
TemporalMentionKind::Date,
|
||||
1000,
|
||||
0,
|
||||
TemporalMentionFlags::empty(),
|
||||
),
|
||||
TemporalMention::new(
|
||||
15,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
TemporalMentionKind::Date,
|
||||
1000,
|
||||
0,
|
||||
TemporalMentionFlags::empty(),
|
||||
),
|
||||
];
|
||||
assert_eq!(window(&mentions, 9, 11), 0..1);
|
||||
assert_eq!(window(&mentions, 10, 15), 0..3);
|
||||
assert_eq!(window(&mentions, 16, 20), 3..3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reject_unsorted_mentions() {
|
||||
let mentions = vec![
|
||||
TemporalMention::new(
|
||||
15,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
TemporalMentionKind::Date,
|
||||
1000,
|
||||
0,
|
||||
TemporalMentionFlags::empty(),
|
||||
),
|
||||
TemporalMention::new(
|
||||
10,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
TemporalMentionKind::Date,
|
||||
1000,
|
||||
0,
|
||||
TemporalMentionFlags::empty(),
|
||||
),
|
||||
];
|
||||
assert!(matches!(
|
||||
validate_mentions_sorted(&mentions),
|
||||
Err(MemvidError::InvalidTemporalTrack { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reject_unsorted_anchors() {
|
||||
let anchors = vec![
|
||||
TemporalAnchor::new(2, 100, AnchorSource::FrameTimestamp),
|
||||
TemporalAnchor::new(2, 110, AnchorSource::Metadata),
|
||||
];
|
||||
assert!(matches!(
|
||||
validate_anchors_sorted(&anchors),
|
||||
Err(MemvidError::InvalidTemporalTrack { .. })
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
|
||||
use blake3::Hasher;
|
||||
|
||||
use crate::{
|
||||
constants::TIME_INDEX_MAGIC,
|
||||
error::{MemvidError, Result},
|
||||
};
|
||||
|
||||
/// Raw entry used to build the time index track.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TimeIndexEntry {
|
||||
pub timestamp: i64,
|
||||
pub frame_id: u64,
|
||||
}
|
||||
|
||||
impl TimeIndexEntry {
|
||||
#[must_use]
|
||||
pub fn new(timestamp: i64, frame_id: u64) -> Self {
|
||||
Self {
|
||||
timestamp,
|
||||
frame_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends entries to the time index track, returning `(offset, length, checksum)`.
|
||||
/// Entries are sorted by `(timestamp, frame_id)` prior to writing.
|
||||
pub fn append_track<W: Write + Seek>(
|
||||
writer: &mut W,
|
||||
entries: &mut [TimeIndexEntry],
|
||||
) -> Result<(u64, u64, [u8; 32])> {
|
||||
entries.sort_by_key(|entry| (entry.timestamp, entry.frame_id));
|
||||
|
||||
let offset = writer.stream_position()?;
|
||||
let mut hasher = Hasher::new();
|
||||
|
||||
writer.write_all(&TIME_INDEX_MAGIC)?;
|
||||
hasher.update(&TIME_INDEX_MAGIC);
|
||||
|
||||
let count = entries.len() as u64;
|
||||
let count_bytes = count.to_le_bytes();
|
||||
writer.write_all(&count_bytes)?;
|
||||
hasher.update(&count_bytes);
|
||||
|
||||
for entry in entries.iter() {
|
||||
let ts_bytes = entry.timestamp.to_le_bytes();
|
||||
let id_bytes = entry.frame_id.to_le_bytes();
|
||||
writer.write_all(&ts_bytes)?;
|
||||
writer.write_all(&id_bytes)?;
|
||||
hasher.update(&ts_bytes);
|
||||
hasher.update(&id_bytes);
|
||||
}
|
||||
|
||||
let end = writer.stream_position()?;
|
||||
let length = end - offset;
|
||||
Ok((offset, length, *hasher.finalize().as_bytes()))
|
||||
}
|
||||
|
||||
/// Reads the time index entries located at `(offset, length)` and validates ordering.
|
||||
pub fn read_track<R: Read + Seek>(
|
||||
reader: &mut R,
|
||||
offset: u64,
|
||||
length: u64,
|
||||
) -> Result<Vec<TimeIndexEntry>> {
|
||||
reader.seek(SeekFrom::Start(offset))?;
|
||||
|
||||
let mut magic = [0u8; 4];
|
||||
reader.read_exact(&mut magic)?;
|
||||
if magic != TIME_INDEX_MAGIC {
|
||||
return Err(MemvidError::InvalidTimeIndex {
|
||||
reason: "magic mismatch".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut count_buf = [0u8; 8];
|
||||
reader.read_exact(&mut count_buf)?;
|
||||
let count = u64::from_le_bytes(count_buf);
|
||||
|
||||
let header_len = 4u64 + 8;
|
||||
if length < header_len {
|
||||
return Err(MemvidError::InvalidTimeIndex {
|
||||
reason: "length shorter than header".into(),
|
||||
});
|
||||
}
|
||||
let payload_bytes = length - header_len;
|
||||
let expected_payload = count
|
||||
.checked_mul((std::mem::size_of::<i64>() + std::mem::size_of::<u64>()) as u64)
|
||||
.ok_or(MemvidError::InvalidTimeIndex {
|
||||
reason: "entry count overflow".into(),
|
||||
})?;
|
||||
if payload_bytes != expected_payload {
|
||||
return Err(MemvidError::InvalidTimeIndex {
|
||||
reason: "length does not match declared count".into(),
|
||||
});
|
||||
}
|
||||
|
||||
// Safe: count validated by checked_mul and payload_bytes comparison above
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut entries = Vec::with_capacity(count as usize);
|
||||
let mut prev: Option<TimeIndexEntry> = None;
|
||||
for _ in 0..count {
|
||||
let mut ts_buf = [0u8; 8];
|
||||
reader.read_exact(&mut ts_buf)?;
|
||||
let timestamp = i64::from_le_bytes(ts_buf);
|
||||
|
||||
let mut id_buf = [0u8; 8];
|
||||
reader.read_exact(&mut id_buf)?;
|
||||
let frame_id = u64::from_le_bytes(id_buf);
|
||||
|
||||
let entry = TimeIndexEntry {
|
||||
timestamp,
|
||||
frame_id,
|
||||
};
|
||||
if let Some(prev_entry) = prev {
|
||||
if entry.timestamp < prev_entry.timestamp
|
||||
|| (entry.timestamp == prev_entry.timestamp && entry.frame_id < prev_entry.frame_id)
|
||||
{
|
||||
return Err(MemvidError::InvalidTimeIndex {
|
||||
reason: "entries not sorted".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
prev = Some(entry);
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Calculates the checksum for the provided entries in canonical order.
|
||||
#[must_use]
|
||||
pub fn calculate_checksum(entries: &[TimeIndexEntry]) -> [u8; 32] {
|
||||
let mut sorted = entries.to_vec();
|
||||
sorted.sort_by_key(|entry| (entry.timestamp, entry.frame_id));
|
||||
|
||||
let mut hasher = Hasher::new();
|
||||
hasher.update(&TIME_INDEX_MAGIC);
|
||||
hasher.update(&(sorted.len() as u64).to_le_bytes());
|
||||
for entry in &sorted {
|
||||
hasher.update(&entry.timestamp.to_le_bytes());
|
||||
hasher.update(&entry.frame_id.to_le_bytes());
|
||||
}
|
||||
*hasher.finalize().as_bytes()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use tempfile::tempfile;
|
||||
|
||||
#[test]
|
||||
fn append_and_read_roundtrip() {
|
||||
let mut file = tempfile().expect("temp file");
|
||||
let mut entries = vec![
|
||||
TimeIndexEntry::new(30, 2),
|
||||
TimeIndexEntry::new(10, 0),
|
||||
TimeIndexEntry::new(20, 1),
|
||||
];
|
||||
|
||||
let (offset, length, checksum) =
|
||||
append_track(&mut file, &mut entries).expect("append track");
|
||||
assert_eq!(entries[0].timestamp, 10); // sorted in place
|
||||
let read_entries = read_track(&mut file, offset, length).expect("read track");
|
||||
assert_eq!(read_entries.len(), 3);
|
||||
assert!(
|
||||
read_entries
|
||||
.windows(2)
|
||||
.all(|w| w[0].timestamp <= w[1].timestamp)
|
||||
);
|
||||
|
||||
let expected_checksum = calculate_checksum(&read_entries);
|
||||
assert_eq!(checksum, expected_checksum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_rejects_unsorted_entries() {
|
||||
let mut file = tempfile().expect("temp file");
|
||||
// Craft an invalid track where entries descend.
|
||||
file.write_all(&TIME_INDEX_MAGIC).unwrap();
|
||||
file.write_all(&(2u64).to_le_bytes()).unwrap();
|
||||
file.write_all(&50i64.to_le_bytes()).unwrap();
|
||||
file.write_all(&5u64.to_le_bytes()).unwrap();
|
||||
file.write_all(&40i64.to_le_bytes()).unwrap();
|
||||
file.write_all(&4u64.to_le_bytes()).unwrap();
|
||||
|
||||
let length = file.seek(SeekFrom::End(0)).unwrap();
|
||||
file.seek(SeekFrom::Start(0)).unwrap();
|
||||
let err = read_track(&mut file, 0, length).expect_err("unsorted entries must fail");
|
||||
matches!(err, MemvidError::InvalidTimeIndex { .. });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calculate_checksum_is_deterministic() {
|
||||
let entries = vec![
|
||||
TimeIndexEntry::new(5, 10),
|
||||
TimeIndexEntry::new(1, 2),
|
||||
TimeIndexEntry::new(5, 9),
|
||||
];
|
||||
let checksum_a = calculate_checksum(&entries);
|
||||
let checksum_b = calculate_checksum(&entries);
|
||||
assert_eq!(checksum_a, checksum_b);
|
||||
}
|
||||
}
|
||||
+517
@@ -0,0 +1,517 @@
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
|
||||
use crate::{
|
||||
constants::{WAL_CHECKPOINT_PERIOD, WAL_CHECKPOINT_THRESHOLD},
|
||||
error::{MemvidError, Result},
|
||||
types::Header,
|
||||
};
|
||||
|
||||
// Each WAL record header: [seq: u64][len: u32][reserved: 4 bytes][checksum: 32 bytes]
|
||||
const ENTRY_HEADER_SIZE: usize = 48;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct WalStats {
|
||||
pub region_size: u64,
|
||||
pub pending_bytes: u64,
|
||||
pub appends_since_checkpoint: u64,
|
||||
pub sequence: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct WalRecord {
|
||||
pub sequence: u64,
|
||||
pub payload: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct EmbeddedWal {
|
||||
file: File,
|
||||
region_offset: u64,
|
||||
region_size: u64,
|
||||
write_head: u64,
|
||||
checkpoint_head: u64,
|
||||
pending_bytes: u64,
|
||||
sequence: u64,
|
||||
checkpoint_sequence: u64,
|
||||
appends_since_checkpoint: u64,
|
||||
read_only: bool,
|
||||
skip_sync: bool,
|
||||
}
|
||||
|
||||
impl EmbeddedWal {
|
||||
pub fn open(file: &File, header: &Header) -> Result<Self> {
|
||||
Self::open_internal(file, header, false)
|
||||
}
|
||||
|
||||
pub fn open_read_only(file: &File, header: &Header) -> Result<Self> {
|
||||
Self::open_internal(file, header, true)
|
||||
}
|
||||
|
||||
fn open_internal(file: &File, header: &Header, read_only: bool) -> Result<Self> {
|
||||
if header.wal_size == 0 {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "wal_size must be non-zero".into(),
|
||||
});
|
||||
}
|
||||
let mut clone = file.try_clone()?;
|
||||
let region_offset = header.wal_offset;
|
||||
let region_size = header.wal_size;
|
||||
let checkpoint_sequence = header.wal_sequence;
|
||||
|
||||
let (entries, next_head) = Self::scan_records(&mut clone, region_offset, region_size)?;
|
||||
|
||||
let pending_bytes = entries
|
||||
.iter()
|
||||
.filter(|entry| entry.sequence > checkpoint_sequence)
|
||||
.map(|entry| entry.total_size)
|
||||
.sum();
|
||||
let sequence = entries
|
||||
.last()
|
||||
.map_or(checkpoint_sequence, |entry| entry.sequence);
|
||||
|
||||
let mut wal = Self {
|
||||
file: clone,
|
||||
region_offset,
|
||||
region_size,
|
||||
write_head: next_head % region_size,
|
||||
checkpoint_head: header.wal_checkpoint_pos % region_size,
|
||||
pending_bytes,
|
||||
sequence,
|
||||
checkpoint_sequence,
|
||||
appends_since_checkpoint: 0,
|
||||
read_only,
|
||||
skip_sync: false,
|
||||
};
|
||||
|
||||
if !wal.read_only {
|
||||
wal.initialise_sentinel()?;
|
||||
}
|
||||
Ok(wal)
|
||||
}
|
||||
|
||||
fn assert_writable(&self) -> Result<()> {
|
||||
if self.read_only {
|
||||
return Err(MemvidError::Lock(
|
||||
"wal is read-only; reopen memory with write access".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn append_entry(&mut self, payload: &[u8]) -> Result<u64> {
|
||||
self.assert_writable()?;
|
||||
let payload_len = payload.len();
|
||||
if payload_len > u32::MAX as usize {
|
||||
return Err(MemvidError::CheckpointFailed {
|
||||
reason: "WAL payload too large".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let entry_size = ENTRY_HEADER_SIZE as u64 + payload_len as u64;
|
||||
if entry_size > self.region_size {
|
||||
return Err(MemvidError::CheckpointFailed {
|
||||
reason: "embedded WAL region too small for entry".into(),
|
||||
});
|
||||
}
|
||||
if self.pending_bytes + entry_size > self.region_size {
|
||||
return Err(MemvidError::CheckpointFailed {
|
||||
reason: "embedded WAL region full".into(),
|
||||
});
|
||||
}
|
||||
|
||||
// Check if we need to wrap around
|
||||
let wrapping = self.write_head + entry_size > self.region_size;
|
||||
if wrapping {
|
||||
// If wrapping would overwrite uncommitted data, return "WAL full" error
|
||||
// instead of silently overwriting. This triggers WAL growth.
|
||||
// The checkpoint_head marks where committed data starts - if we wrap and would
|
||||
// write over any pending (uncommitted) data, we must grow the WAL instead.
|
||||
if self.pending_bytes > 0 {
|
||||
return Err(MemvidError::CheckpointFailed {
|
||||
reason: "embedded WAL region full".into(),
|
||||
});
|
||||
}
|
||||
self.write_head = 0;
|
||||
}
|
||||
|
||||
let next_sequence = self.sequence + 1;
|
||||
tracing::debug!(
|
||||
wal.write_head = self.write_head,
|
||||
wal.sequence = next_sequence,
|
||||
wal.payload_len = payload_len,
|
||||
"wal append entry"
|
||||
);
|
||||
self.write_record(self.write_head, next_sequence, payload)?;
|
||||
|
||||
self.write_head = (self.write_head + entry_size) % self.region_size;
|
||||
self.pending_bytes += entry_size;
|
||||
self.sequence = self.sequence.wrapping_add(1);
|
||||
self.appends_since_checkpoint = self.appends_since_checkpoint.saturating_add(1);
|
||||
|
||||
self.maybe_write_sentinel()?;
|
||||
|
||||
Ok(self.sequence)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn should_checkpoint(&self) -> bool {
|
||||
if self.read_only || self.region_size == 0 {
|
||||
return false;
|
||||
}
|
||||
let occupancy = self.pending_bytes as f64 / self.region_size as f64;
|
||||
occupancy >= WAL_CHECKPOINT_THRESHOLD
|
||||
|| self.appends_since_checkpoint >= WAL_CHECKPOINT_PERIOD
|
||||
}
|
||||
|
||||
pub fn record_checkpoint(&mut self, header: &mut Header) -> Result<()> {
|
||||
self.assert_writable()?;
|
||||
self.checkpoint_head = self.write_head;
|
||||
self.pending_bytes = 0;
|
||||
self.appends_since_checkpoint = 0;
|
||||
self.checkpoint_sequence = self.sequence;
|
||||
header.wal_checkpoint_pos = self.checkpoint_head;
|
||||
header.wal_sequence = self.checkpoint_sequence;
|
||||
self.maybe_write_sentinel()
|
||||
}
|
||||
|
||||
pub fn pending_records(&mut self) -> Result<Vec<WalRecord>> {
|
||||
self.records_after(self.checkpoint_sequence)
|
||||
}
|
||||
|
||||
pub fn records_after(&mut self, sequence: u64) -> Result<Vec<WalRecord>> {
|
||||
let (entries, next_head) =
|
||||
Self::scan_records(&mut self.file, self.region_offset, self.region_size)?;
|
||||
|
||||
self.sequence = entries.last().map_or(self.sequence, |entry| entry.sequence);
|
||||
self.pending_bytes = entries
|
||||
.iter()
|
||||
.filter(|entry| entry.sequence > self.checkpoint_sequence)
|
||||
.map(|entry| entry.total_size)
|
||||
.sum();
|
||||
self.write_head = next_head % self.region_size;
|
||||
if !self.read_only {
|
||||
self.initialise_sentinel()?;
|
||||
}
|
||||
|
||||
Ok(entries
|
||||
.into_iter()
|
||||
.filter(|entry| entry.sequence > sequence)
|
||||
.map(|entry| WalRecord {
|
||||
sequence: entry.sequence,
|
||||
payload: entry.payload,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn stats(&self) -> WalStats {
|
||||
WalStats {
|
||||
region_size: self.region_size,
|
||||
pending_bytes: self.pending_bytes,
|
||||
appends_since_checkpoint: self.appends_since_checkpoint,
|
||||
sequence: self.sequence,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn region_offset(&self) -> u64 {
|
||||
self.region_offset
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn file(&self) -> &File {
|
||||
&self.file
|
||||
}
|
||||
|
||||
/// Enable or disable per-entry fsync.
|
||||
///
|
||||
/// When `skip` is `true`, `write_record()` will not call `sync_all()` after
|
||||
/// each WAL append. The caller **must** call [`flush()`](Self::flush) after
|
||||
/// the batch to ensure durability.
|
||||
pub fn set_skip_sync(&mut self, skip: bool) {
|
||||
self.skip_sync = skip;
|
||||
}
|
||||
|
||||
/// Force an `fsync` on the underlying WAL file.
|
||||
///
|
||||
/// Call this after a batch of appends performed with `skip_sync = true`
|
||||
/// to ensure all data is durable on disk.
|
||||
pub fn flush(&mut self) -> Result<()> {
|
||||
self.file.sync_all().map_err(Into::into)
|
||||
}
|
||||
|
||||
fn initialise_sentinel(&mut self) -> Result<()> {
|
||||
self.maybe_write_sentinel()
|
||||
}
|
||||
|
||||
fn write_record(&mut self, position: u64, sequence: u64, payload: &[u8]) -> Result<()> {
|
||||
self.assert_writable()?;
|
||||
let digest = blake3::hash(payload);
|
||||
let mut header = [0u8; ENTRY_HEADER_SIZE];
|
||||
header[..8].copy_from_slice(&sequence.to_le_bytes());
|
||||
header[8..12]
|
||||
.copy_from_slice(&(u32::try_from(payload.len()).unwrap_or(u32::MAX)).to_le_bytes());
|
||||
header[16..48].copy_from_slice(digest.as_bytes());
|
||||
|
||||
// Atomic write: combine header and payload into single buffer
|
||||
// This prevents corruption if the file is closed mid-write
|
||||
let mut combined = Vec::with_capacity(ENTRY_HEADER_SIZE + payload.len());
|
||||
combined.extend_from_slice(&header);
|
||||
combined.extend_from_slice(payload);
|
||||
|
||||
self.seek_and_write(position, &combined)?;
|
||||
if tracing::enabled!(tracing::Level::DEBUG) {
|
||||
if let Err(err) = self.debug_verify_header(position, sequence, payload.len()) {
|
||||
tracing::warn!(error = %err, "wal header verify failed");
|
||||
}
|
||||
}
|
||||
|
||||
// Force fsync to ensure data is durable before returning
|
||||
// Critical for preventing corruption during rapid file operations
|
||||
// In batch mode (skip_sync=true), fsync is deferred to flush() for performance
|
||||
if !self.skip_sync {
|
||||
self.file.sync_all()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_zero_header(&mut self, position: u64) -> Result<u64> {
|
||||
self.assert_writable()?;
|
||||
if self.region_size == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let mut pos = position % self.region_size;
|
||||
let remaining = self.region_size - pos;
|
||||
if remaining < ENTRY_HEADER_SIZE as u64 {
|
||||
if remaining > 0 {
|
||||
// Safe: remaining < ENTRY_HEADER_SIZE (48) so always fits in usize
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let zero_tail = vec![0u8; remaining as usize];
|
||||
self.seek_and_write(pos, &zero_tail)?;
|
||||
}
|
||||
pos = 0;
|
||||
}
|
||||
let zero = [0u8; ENTRY_HEADER_SIZE];
|
||||
self.seek_and_write(pos, &zero)?;
|
||||
Ok(pos)
|
||||
}
|
||||
|
||||
fn seek_and_write(&mut self, position: u64, bytes: &[u8]) -> Result<()> {
|
||||
self.assert_writable()?;
|
||||
let pos = position % self.region_size;
|
||||
let absolute = self.region_offset + pos;
|
||||
self.file.seek(SeekFrom::Start(absolute))?;
|
||||
self.file.write_all(bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn maybe_write_sentinel(&mut self) -> Result<()> {
|
||||
if self.read_only || self.region_size == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
if self.pending_bytes >= self.region_size {
|
||||
return Ok(());
|
||||
}
|
||||
// Sentinel marks end of valid entries - always keep write_head in sync
|
||||
let next = self.write_zero_header(self.write_head)?;
|
||||
self.write_head = next;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn scan_records(file: &mut File, offset: u64, size: u64) -> Result<(Vec<ScannedRecord>, u64)> {
|
||||
let mut records = Vec::new();
|
||||
let mut cursor = 0u64;
|
||||
while cursor + ENTRY_HEADER_SIZE as u64 <= size {
|
||||
file.seek(SeekFrom::Start(offset + cursor))?;
|
||||
let mut header = [0u8; ENTRY_HEADER_SIZE];
|
||||
file.read_exact(&mut header)?;
|
||||
|
||||
let sequence = u64::from_le_bytes(header[..8].try_into().map_err(|_| {
|
||||
MemvidError::WalCorruption {
|
||||
offset: cursor,
|
||||
reason: "invalid wal sequence header".into(),
|
||||
}
|
||||
})?);
|
||||
let length = u64::from(u32::from_le_bytes(header[8..12].try_into().map_err(
|
||||
|_| MemvidError::WalCorruption {
|
||||
offset: cursor,
|
||||
reason: "invalid wal length header".into(),
|
||||
},
|
||||
)?));
|
||||
let checksum = &header[16..48];
|
||||
|
||||
if sequence == 0 && length == 0 {
|
||||
break;
|
||||
}
|
||||
if length == 0 || cursor + ENTRY_HEADER_SIZE as u64 + length > size {
|
||||
tracing::error!(
|
||||
wal.scan_offset = cursor,
|
||||
wal.sequence = sequence,
|
||||
wal.length = length,
|
||||
wal.region_size = size,
|
||||
"wal record length invalid"
|
||||
);
|
||||
return Err(MemvidError::WalCorruption {
|
||||
offset: cursor,
|
||||
reason: "wal record length invalid".into(),
|
||||
});
|
||||
}
|
||||
|
||||
// Safe: length comes from u32::from_le_bytes above, so max is u32::MAX
|
||||
// which fits in usize on all supported platforms (32-bit and 64-bit)
|
||||
let length_usize = usize::try_from(length).map_err(|_| MemvidError::WalCorruption {
|
||||
offset: cursor,
|
||||
reason: "wal record length too large for platform".into(),
|
||||
})?;
|
||||
let mut payload = vec![0u8; length_usize];
|
||||
file.read_exact(&mut payload)?;
|
||||
let expected = blake3::hash(&payload);
|
||||
if expected.as_bytes() != checksum {
|
||||
return Err(MemvidError::WalCorruption {
|
||||
offset: cursor,
|
||||
reason: "wal record checksum mismatch".into(),
|
||||
});
|
||||
}
|
||||
|
||||
records.push(ScannedRecord {
|
||||
sequence,
|
||||
payload,
|
||||
total_size: ENTRY_HEADER_SIZE as u64 + length,
|
||||
});
|
||||
|
||||
cursor += ENTRY_HEADER_SIZE as u64 + length;
|
||||
}
|
||||
|
||||
Ok((records, cursor))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ScannedRecord {
|
||||
sequence: u64,
|
||||
payload: Vec<u8>,
|
||||
total_size: u64,
|
||||
}
|
||||
|
||||
impl EmbeddedWal {
|
||||
fn debug_verify_header(
|
||||
&mut self,
|
||||
position: u64,
|
||||
expected_sequence: u64,
|
||||
expected_len: usize,
|
||||
) -> Result<()> {
|
||||
if self.region_size == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let pos = position % self.region_size;
|
||||
let absolute = self.region_offset + pos;
|
||||
let mut buf = [0u8; ENTRY_HEADER_SIZE];
|
||||
self.file.seek(SeekFrom::Start(absolute))?;
|
||||
self.file.read_exact(&mut buf)?;
|
||||
|
||||
// Safe byte extraction - return early if malformed (debug function)
|
||||
let seq = buf
|
||||
.get(..8)
|
||||
.and_then(|s| <[u8; 8]>::try_from(s).ok())
|
||||
.map_or(0, u64::from_le_bytes);
|
||||
let len = buf
|
||||
.get(8..12)
|
||||
.and_then(|s| <[u8; 4]>::try_from(s).ok())
|
||||
.map_or(0, u32::from_le_bytes);
|
||||
|
||||
tracing::debug!(
|
||||
wal.verify_position = pos,
|
||||
wal.verify_sequence = seq,
|
||||
wal.expected_sequence = expected_sequence,
|
||||
wal.verify_length = len,
|
||||
wal.expected_length = expected_len,
|
||||
"wal header verify"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::constants::WAL_OFFSET;
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use tempfile::tempfile;
|
||||
|
||||
fn header_for(size: u64) -> Header {
|
||||
Header {
|
||||
magic: *b"MV2\0",
|
||||
version: 0x0201,
|
||||
footer_offset: 0,
|
||||
wal_offset: WAL_OFFSET,
|
||||
wal_size: size,
|
||||
wal_checkpoint_pos: 0,
|
||||
wal_sequence: 0,
|
||||
toc_checksum: [0u8; 32],
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_wal(size: u64) -> (File, Header) {
|
||||
let file = tempfile().expect("temp file");
|
||||
file.set_len(WAL_OFFSET + size).expect("set_len");
|
||||
let header = header_for(size);
|
||||
(file, header)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn append_and_recover() {
|
||||
let (file, header) = prepare_wal(1024);
|
||||
let mut wal = EmbeddedWal::open(&file, &header).expect("open wal");
|
||||
|
||||
wal.append_entry(b"first").expect("append first");
|
||||
wal.append_entry(b"second").expect("append second");
|
||||
|
||||
let records = wal.records_after(0).expect("records");
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0].payload, b"first");
|
||||
assert_eq!(records[0].sequence, 1);
|
||||
assert_eq!(records[1].payload, b"second");
|
||||
assert_eq!(records[1].sequence, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrap_and_checkpoint() {
|
||||
let size = (ENTRY_HEADER_SIZE as u64 * 2) + 64;
|
||||
let (file, mut header) = prepare_wal(size);
|
||||
let mut wal = EmbeddedWal::open(&file, &header).expect("open wal");
|
||||
|
||||
wal.append_entry(&[0xAA; 32]).expect("append a");
|
||||
wal.append_entry(&[0xBB; 32]).expect("append b");
|
||||
wal.record_checkpoint(&mut header).expect("checkpoint");
|
||||
|
||||
assert!(wal.pending_records().expect("pending").is_empty());
|
||||
|
||||
wal.append_entry(&[0xCC; 32]).expect("append c");
|
||||
let records = wal.pending_records().expect("after append");
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(records[0].payload, vec![0xCC; 32]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupted_record_reports_offset() {
|
||||
let (mut file, header) = prepare_wal(64);
|
||||
// Write a record header that claims an impossible length so scan_records trips.
|
||||
file.seek(SeekFrom::Start(header.wal_offset)).expect("seek");
|
||||
let mut record = [0u8; ENTRY_HEADER_SIZE];
|
||||
record[..8].copy_from_slice(&1u64.to_le_bytes()); // sequence
|
||||
record[8..12].copy_from_slice(&(u32::MAX).to_le_bytes()); // absurd length
|
||||
file.write_all(&record).expect("write corrupt header");
|
||||
file.sync_all().expect("sync");
|
||||
|
||||
let err = EmbeddedWal::open(&file, &header).expect_err("open should fail");
|
||||
match err {
|
||||
MemvidError::WalCorruption { offset, reason } => {
|
||||
assert_eq!(offset, 0);
|
||||
assert!(reason.contains("length"), "reason should mention length");
|
||||
}
|
||||
other => panic!("unexpected error: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
+828
@@ -0,0 +1,828 @@
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::{BTreeMap, HashMap},
|
||||
};
|
||||
|
||||
use blake3::hash;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{MemvidError, Result, types::FrameId};
|
||||
|
||||
// Bincode configuration reused for deterministic layout.
|
||||
fn lex_config() -> impl bincode::config::Config {
|
||||
bincode::config::standard()
|
||||
.with_fixed_int_encoding()
|
||||
.with_little_endian()
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
const LEX_DECODE_LIMIT: usize = crate::MAX_INDEX_BYTES as usize;
|
||||
const LEX_SECTION_SOFT_CHARS: usize = 900;
|
||||
const LEX_SECTION_HARD_CHARS: usize = 1400;
|
||||
const LEX_SECTION_MAX_COUNT: usize = 2048;
|
||||
|
||||
/// Intermediate builder that collects documents prior to serialisation.
|
||||
#[derive(Default)]
|
||||
pub struct LexIndexBuilder {
|
||||
documents: Vec<LexDocument>,
|
||||
}
|
||||
|
||||
impl LexIndexBuilder {
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn add_document(
|
||||
&mut self,
|
||||
frame_id: FrameId,
|
||||
uri: &str,
|
||||
title: Option<&str>,
|
||||
content: &str,
|
||||
tags: &HashMap<String, String>,
|
||||
) {
|
||||
let tokens = tokenize(content);
|
||||
// Convert HashMap to BTreeMap for deterministic serialization
|
||||
let tags: BTreeMap<_, _> = tags.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
|
||||
let mut sections = chunk_sections(content);
|
||||
|
||||
let (content_owned, content_lower) = if content.is_empty() {
|
||||
(String::new(), String::new())
|
||||
} else if sections.is_empty() {
|
||||
let owned = content.to_string();
|
||||
let lower = owned.to_ascii_lowercase();
|
||||
sections.push(LexSection {
|
||||
offset: 0,
|
||||
content: owned.clone(),
|
||||
content_lower: lower.clone(),
|
||||
});
|
||||
(owned, lower)
|
||||
} else {
|
||||
(String::new(), String::new())
|
||||
};
|
||||
self.documents.push(LexDocument {
|
||||
frame_id,
|
||||
tokens,
|
||||
tags,
|
||||
content: content_owned,
|
||||
content_lower,
|
||||
uri: Some(uri.to_string()),
|
||||
title: title.map(ToString::to_string),
|
||||
sections,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn finish(mut self) -> Result<LexIndexArtifact> {
|
||||
for document in &mut self.documents {
|
||||
document.ensure_sections();
|
||||
}
|
||||
let bytes = bincode::serde::encode_to_vec(&self.documents, lex_config())?;
|
||||
let checksum = *hash(&bytes).as_bytes();
|
||||
Ok(LexIndexArtifact {
|
||||
bytes,
|
||||
doc_count: self.documents.len() as u64,
|
||||
checksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialized lexical index artifact ready to be embedded in the `.mv2` file.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LexIndexArtifact {
|
||||
pub bytes: Vec<u8>,
|
||||
pub doc_count: u64,
|
||||
pub checksum: [u8; 32],
|
||||
}
|
||||
|
||||
/// Read-only lexical index decoded from persisted bytes.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LexIndex {
|
||||
documents: Vec<LexDocument>,
|
||||
}
|
||||
|
||||
impl LexIndex {
|
||||
pub fn decode(bytes: &[u8]) -> Result<Self> {
|
||||
let new_config = bincode::config::standard()
|
||||
.with_fixed_int_encoding()
|
||||
.with_little_endian()
|
||||
.with_limit::<LEX_DECODE_LIMIT>();
|
||||
if let Ok((documents, read)) =
|
||||
bincode::serde::decode_from_slice::<Vec<LexDocument>, _>(bytes, new_config)
|
||||
{
|
||||
if read == bytes.len() {
|
||||
return Ok(Self::from_documents(documents));
|
||||
}
|
||||
}
|
||||
|
||||
let legacy_fixed = bincode::config::standard()
|
||||
.with_fixed_int_encoding()
|
||||
.with_little_endian()
|
||||
.with_limit::<LEX_DECODE_LIMIT>();
|
||||
if let Ok((legacy_docs, read)) =
|
||||
bincode::serde::decode_from_slice::<Vec<LegacyLexDocument>, _>(bytes, legacy_fixed)
|
||||
{
|
||||
if read == bytes.len() {
|
||||
let documents = legacy_docs.into_iter().map(legacy_to_current).collect();
|
||||
return Ok(Self::from_documents(documents));
|
||||
}
|
||||
}
|
||||
|
||||
let legacy_config = bincode::config::standard()
|
||||
.with_little_endian()
|
||||
.with_limit::<LEX_DECODE_LIMIT>();
|
||||
if let Ok((legacy_docs, read)) =
|
||||
bincode::serde::decode_from_slice::<Vec<LegacyLexDocument>, _>(bytes, legacy_config)
|
||||
{
|
||||
if read == bytes.len() {
|
||||
let documents = legacy_docs.into_iter().map(legacy_to_current).collect();
|
||||
return Ok(Self::from_documents(documents));
|
||||
}
|
||||
}
|
||||
|
||||
Err(MemvidError::InvalidToc {
|
||||
reason: "unsupported lex index encoding".into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn from_documents(mut documents: Vec<LexDocument>) -> Self {
|
||||
for document in &mut documents {
|
||||
document.ensure_sections();
|
||||
}
|
||||
Self { documents }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn search(&self, query: &str, limit: usize) -> Vec<LexSearchHit> {
|
||||
let mut query_tokens = tokenize(query);
|
||||
query_tokens.retain(|token| !token.is_empty());
|
||||
if query_tokens.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut matches = self.compute_matches(&query_tokens, None, None);
|
||||
matches.truncate(limit);
|
||||
matches
|
||||
.into_iter()
|
||||
.map(|m| {
|
||||
let snippets = build_snippets(&m.content, &m.occurrences, 160, 3);
|
||||
LexSearchHit {
|
||||
frame_id: m.frame_id,
|
||||
score: m.score,
|
||||
match_count: m.occurrences.len(),
|
||||
snippets,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn documents_mut(&mut self) -> &mut [LexDocument] {
|
||||
&mut self.documents
|
||||
}
|
||||
|
||||
pub(crate) fn remove_document(&mut self, frame_id: FrameId) {
|
||||
self.documents.retain(|doc| doc.frame_id != frame_id);
|
||||
}
|
||||
|
||||
pub(crate) fn compute_matches(
|
||||
&self,
|
||||
query_tokens: &[String],
|
||||
uri_filter: Option<&str>,
|
||||
scope_filter: Option<&str>,
|
||||
) -> Vec<LexMatch> {
|
||||
if query_tokens.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut hits = Vec::new();
|
||||
let phrase = query_tokens.join(" ");
|
||||
for document in &self.documents {
|
||||
if let Some(uri) = uri_filter {
|
||||
if !uri_matches(document.uri.as_deref(), uri) {
|
||||
continue;
|
||||
}
|
||||
} else if let Some(scope) = scope_filter {
|
||||
match document.uri.as_deref() {
|
||||
Some(candidate) if candidate.starts_with(scope) => {}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
|
||||
if document.sections.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
for section in &document.sections {
|
||||
let haystack = section.content_lower.as_str();
|
||||
if haystack.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut occurrences: Vec<(usize, usize)> = Vec::new();
|
||||
|
||||
if query_tokens.len() == 1 {
|
||||
let needle = &query_tokens[0];
|
||||
if needle.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut start = 0usize;
|
||||
while let Some(idx) = haystack[start..].find(needle) {
|
||||
let local_start = start + idx;
|
||||
let local_end = local_start + needle.len();
|
||||
occurrences.push((local_start, local_end));
|
||||
start = local_end;
|
||||
}
|
||||
} else {
|
||||
let mut all_occurrences = Vec::new();
|
||||
let mut all_present = true;
|
||||
for needle in query_tokens {
|
||||
if needle.is_empty() {
|
||||
all_present = false;
|
||||
break;
|
||||
}
|
||||
let mut start = 0usize;
|
||||
let mut found_for_token = false;
|
||||
while let Some(idx) = haystack[start..].find(needle) {
|
||||
found_for_token = true;
|
||||
let local_start = start + idx;
|
||||
let local_end = local_start + needle.len();
|
||||
all_occurrences.push((local_start, local_end));
|
||||
start = local_end;
|
||||
}
|
||||
if !found_for_token {
|
||||
all_present = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !all_present {
|
||||
continue;
|
||||
}
|
||||
occurrences = all_occurrences;
|
||||
}
|
||||
|
||||
if occurrences.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
occurrences.sort_by_key(|(start, _)| *start);
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let mut score = occurrences.len() as f32;
|
||||
if !phrase.is_empty() && section.content_lower.contains(&phrase) {
|
||||
score += 1000.0;
|
||||
}
|
||||
hits.push(LexMatch {
|
||||
frame_id: document.frame_id,
|
||||
score,
|
||||
occurrences,
|
||||
content: section.content.clone(),
|
||||
uri: document.uri.clone(),
|
||||
title: document.title.clone(),
|
||||
chunk_offset: section.offset,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
hits.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal));
|
||||
|
||||
// Deduplicate by frame_id, keeping the highest-scoring match for each frame.
|
||||
// This prevents the same document from appearing multiple times when it has
|
||||
// multiple sections that match the query.
|
||||
let mut seen_frames: std::collections::HashSet<FrameId> = std::collections::HashSet::new();
|
||||
let mut deduped = Vec::with_capacity(hits.len());
|
||||
for hit in hits {
|
||||
if seen_frames.insert(hit.frame_id) {
|
||||
deduped.push(hit);
|
||||
}
|
||||
}
|
||||
deduped
|
||||
}
|
||||
}
|
||||
|
||||
fn uri_matches(candidate: Option<&str>, expected: &str) -> bool {
|
||||
let Some(uri) = candidate else {
|
||||
return false;
|
||||
};
|
||||
if expected.contains('#') {
|
||||
uri.eq_ignore_ascii_case(expected)
|
||||
} else {
|
||||
let expected_lower = expected.to_ascii_lowercase();
|
||||
let candidate_lower = uri.to_ascii_lowercase();
|
||||
candidate_lower.starts_with(&expected_lower)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub(crate) struct LexDocument {
|
||||
pub(crate) frame_id: FrameId,
|
||||
tokens: Vec<String>,
|
||||
tags: BTreeMap<String, String>,
|
||||
#[serde(default)]
|
||||
content: String,
|
||||
#[serde(default)]
|
||||
pub(crate) content_lower: String,
|
||||
#[serde(default)]
|
||||
pub(crate) uri: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) title: Option<String>,
|
||||
#[serde(default)]
|
||||
sections: Vec<LexSection>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub(crate) struct LexSection {
|
||||
pub(crate) offset: usize,
|
||||
#[serde(default)]
|
||||
pub(crate) content: String,
|
||||
#[serde(default)]
|
||||
pub(crate) content_lower: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct LegacyLexDocument {
|
||||
frame_id: FrameId,
|
||||
tokens: Vec<String>,
|
||||
tags: BTreeMap<String, String>,
|
||||
#[serde(default)]
|
||||
content: Option<String>,
|
||||
#[serde(default)]
|
||||
uri: Option<String>,
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
}
|
||||
|
||||
impl LexDocument {
|
||||
fn ensure_sections(&mut self) {
|
||||
if !self.sections.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
if self.content.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
if self.content_lower.is_empty() {
|
||||
self.content_lower = self.content.to_ascii_lowercase();
|
||||
}
|
||||
|
||||
self.sections.push(LexSection {
|
||||
offset: 0,
|
||||
content: self.content.clone(),
|
||||
content_lower: self.content_lower.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn legacy_to_current(legacy: LegacyLexDocument) -> LexDocument {
|
||||
let content = legacy.content.unwrap_or_default();
|
||||
let content_lower = content.to_ascii_lowercase();
|
||||
let sections = if content.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![LexSection {
|
||||
offset: 0,
|
||||
content: content.clone(),
|
||||
content_lower: content_lower.clone(),
|
||||
}]
|
||||
};
|
||||
LexDocument {
|
||||
frame_id: legacy.frame_id,
|
||||
tokens: legacy.tokens,
|
||||
tags: legacy.tags,
|
||||
content,
|
||||
content_lower,
|
||||
uri: legacy.uri,
|
||||
title: legacy.title,
|
||||
sections,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LexSearchHit {
|
||||
pub frame_id: FrameId,
|
||||
pub score: f32,
|
||||
pub match_count: usize,
|
||||
pub snippets: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct LexMatch {
|
||||
pub frame_id: FrameId,
|
||||
pub score: f32,
|
||||
pub occurrences: Vec<(usize, usize)>,
|
||||
pub content: String,
|
||||
pub uri: Option<String>,
|
||||
pub title: Option<String>,
|
||||
pub chunk_offset: usize,
|
||||
}
|
||||
|
||||
fn tokenize(input: &str) -> Vec<String> {
|
||||
input
|
||||
.split(|c: char| !is_token_char(c))
|
||||
.filter_map(|token| {
|
||||
if token.chars().any(char::is_alphanumeric) {
|
||||
Some(token.to_lowercase())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn is_token_char(ch: char) -> bool {
|
||||
ch.is_alphanumeric() || matches!(ch, '&' | '@' | '+' | '/' | '_')
|
||||
}
|
||||
|
||||
fn build_snippets(
|
||||
content: &str,
|
||||
occurrences: &[(usize, usize)],
|
||||
window: usize,
|
||||
max_snippets: usize,
|
||||
) -> Vec<String> {
|
||||
compute_snippet_slices(content, occurrences, window, max_snippets)
|
||||
.into_iter()
|
||||
.map(|(start, end)| content[start..end].replace('\n', " "))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn chunk_sections(content: &str) -> Vec<LexSection> {
|
||||
if content.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
if content.len() <= LEX_SECTION_HARD_CHARS {
|
||||
return vec![LexSection {
|
||||
offset: 0,
|
||||
content: content.to_string(),
|
||||
content_lower: content.to_ascii_lowercase(),
|
||||
}];
|
||||
}
|
||||
|
||||
let mut sections: Vec<LexSection> = Vec::new();
|
||||
let mut chunk_start = 0usize;
|
||||
let mut last_soft_break = None;
|
||||
let mut iter = content.char_indices().peekable();
|
||||
|
||||
while let Some((idx, ch)) = iter.next() {
|
||||
let char_end = idx + ch.len_utf8();
|
||||
let current_len = char_end.saturating_sub(chunk_start);
|
||||
let next_char = iter.peek().map(|(_, next)| *next);
|
||||
|
||||
if is_soft_boundary(ch, next_char) {
|
||||
last_soft_break = Some(char_end);
|
||||
if current_len < LEX_SECTION_SOFT_CHARS {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if current_len < LEX_SECTION_HARD_CHARS {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut split_at = last_soft_break.unwrap_or(char_end);
|
||||
if split_at <= chunk_start {
|
||||
split_at = char_end;
|
||||
}
|
||||
|
||||
push_section(&mut sections, content, chunk_start, split_at);
|
||||
chunk_start = split_at;
|
||||
last_soft_break = None;
|
||||
|
||||
if sections.len() >= LEX_SECTION_MAX_COUNT {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if chunk_start < content.len() {
|
||||
if sections.len() >= LEX_SECTION_MAX_COUNT {
|
||||
if let Some(last) = sections.last_mut() {
|
||||
let slice = &content[last.offset..];
|
||||
last.content = slice.to_string();
|
||||
last.content_lower = slice.to_ascii_lowercase();
|
||||
}
|
||||
} else {
|
||||
push_section(&mut sections, content, chunk_start, content.len());
|
||||
}
|
||||
}
|
||||
|
||||
if sections.is_empty() {
|
||||
sections.push(LexSection {
|
||||
offset: 0,
|
||||
content: content.to_string(),
|
||||
content_lower: content.to_ascii_lowercase(),
|
||||
});
|
||||
}
|
||||
|
||||
sections
|
||||
}
|
||||
|
||||
fn push_section(sections: &mut Vec<LexSection>, content: &str, start: usize, end: usize) {
|
||||
if end <= start {
|
||||
return;
|
||||
}
|
||||
|
||||
let slice = &content[start..end];
|
||||
sections.push(LexSection {
|
||||
offset: start,
|
||||
content: slice.to_string(),
|
||||
content_lower: slice.to_ascii_lowercase(),
|
||||
});
|
||||
}
|
||||
|
||||
fn is_soft_boundary(ch: char, next: Option<char>) -> bool {
|
||||
match ch {
|
||||
'.' | '!' | '?' => next.is_none_or(char::is_whitespace),
|
||||
'\n' => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn compute_snippet_slices(
|
||||
content: &str,
|
||||
occurrences: &[(usize, usize)],
|
||||
window: usize,
|
||||
max_snippets: usize,
|
||||
) -> Vec<(usize, usize)> {
|
||||
if content.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
if occurrences.is_empty() {
|
||||
let end = advance_boundary(content, 0, window);
|
||||
return vec![(0, end)];
|
||||
}
|
||||
|
||||
let mut merged: Vec<(usize, usize)> = Vec::new();
|
||||
for &(start, end) in occurrences {
|
||||
let mut snippet_start = start.saturating_sub(window / 2);
|
||||
let mut snippet_end = (end + window / 2).min(content.len());
|
||||
|
||||
if let Some(adj) = sentence_start_before(content, snippet_start) {
|
||||
snippet_start = adj;
|
||||
}
|
||||
if let Some(adj) = sentence_end_after(content, snippet_end) {
|
||||
snippet_end = adj;
|
||||
}
|
||||
|
||||
snippet_start = prev_char_boundary(content, snippet_start);
|
||||
snippet_end = next_char_boundary(content, snippet_end);
|
||||
|
||||
if snippet_end <= snippet_start {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(last) = merged.last_mut() {
|
||||
if snippet_start <= last.1 + 20 {
|
||||
last.1 = last.1.max(snippet_end);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
merged.push((
|
||||
snippet_start.min(content.len()),
|
||||
snippet_end.min(content.len()),
|
||||
));
|
||||
if merged.len() >= max_snippets {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if merged.is_empty() {
|
||||
let end = advance_boundary(content, 0, window);
|
||||
merged.push((0, end));
|
||||
}
|
||||
|
||||
merged
|
||||
}
|
||||
|
||||
fn sentence_start_before(content: &str, idx: usize) -> Option<usize> {
|
||||
if idx == 0 {
|
||||
return Some(0);
|
||||
}
|
||||
let mut idx = idx.min(content.len());
|
||||
idx = prev_char_boundary(content, idx);
|
||||
let mut candidate = None;
|
||||
for (pos, ch) in content[..idx].char_indices() {
|
||||
if matches!(ch, '.' | '!' | '?' | '\n') {
|
||||
candidate = Some(pos + ch.len_utf8());
|
||||
}
|
||||
}
|
||||
candidate.map(|pos| {
|
||||
let mut pos = next_char_boundary(content, pos);
|
||||
while pos < content.len() && content.as_bytes()[pos].is_ascii_whitespace() {
|
||||
pos += 1;
|
||||
}
|
||||
prev_char_boundary(content, pos)
|
||||
})
|
||||
}
|
||||
|
||||
fn sentence_end_after(content: &str, idx: usize) -> Option<usize> {
|
||||
if idx >= content.len() {
|
||||
return Some(content.len());
|
||||
}
|
||||
let mut idx = idx;
|
||||
idx = prev_char_boundary(content, idx);
|
||||
for (offset, ch) in content[idx..].char_indices() {
|
||||
let global = idx + offset;
|
||||
if matches!(ch, '.' | '!' | '?') {
|
||||
return Some(next_char_boundary(content, global + ch.len_utf8()));
|
||||
}
|
||||
if ch == '\n' {
|
||||
return Some(global);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn prev_char_boundary(content: &str, mut idx: usize) -> usize {
|
||||
if idx > content.len() {
|
||||
idx = content.len();
|
||||
}
|
||||
while idx > 0 && !content.is_char_boundary(idx) {
|
||||
idx -= 1;
|
||||
}
|
||||
idx
|
||||
}
|
||||
|
||||
fn next_char_boundary(content: &str, mut idx: usize) -> usize {
|
||||
if idx > content.len() {
|
||||
idx = content.len();
|
||||
}
|
||||
while idx < content.len() && !content.is_char_boundary(idx) {
|
||||
idx += 1;
|
||||
}
|
||||
idx
|
||||
}
|
||||
|
||||
fn advance_boundary(content: &str, start: usize, mut window: usize) -> usize {
|
||||
if start >= content.len() {
|
||||
return content.len();
|
||||
}
|
||||
let mut last = content.len();
|
||||
for (offset, _) in content[start..].char_indices() {
|
||||
if window == 0 {
|
||||
return start + offset;
|
||||
}
|
||||
last = start + offset;
|
||||
window -= 1;
|
||||
}
|
||||
content.len().max(last)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn builder_produces_artifact() {
|
||||
let mut builder = LexIndexBuilder::new();
|
||||
let mut tags = HashMap::new();
|
||||
tags.insert("source".into(), "test".into());
|
||||
builder.add_document(0, "mv2://docs/one", Some("Doc One"), "hello world", &tags);
|
||||
builder.add_document(
|
||||
1,
|
||||
"mv2://docs/two",
|
||||
Some("Doc Two"),
|
||||
"rust systems",
|
||||
&HashMap::new(),
|
||||
);
|
||||
|
||||
let artifact = builder.finish().expect("finish");
|
||||
assert_eq!(artifact.doc_count, 2);
|
||||
assert!(!artifact.bytes.is_empty());
|
||||
|
||||
let index = LexIndex::decode(&artifact.bytes).expect("decode");
|
||||
let hits = index.search("rust", 10);
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(hits[0].frame_id, 1);
|
||||
assert!(hits[0].match_count >= 1);
|
||||
assert!(!hits[0].snippets.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenizer_lowercases_and_filters() {
|
||||
let tokens = tokenize("Hello, Rust-lang!");
|
||||
assert_eq!(tokens, vec!["hello", "rust", "lang"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenizer_retains_connector_characters() {
|
||||
let tokens = tokenize("N&M EXPRESS LLC @ 2024");
|
||||
assert_eq!(tokens, vec!["n&m", "express", "llc", "2024"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_matches_deduplicates_by_frame_id() {
|
||||
// Create a document with content long enough to be split into multiple sections.
|
||||
// The section soft limit is 900 chars, hard limit is 1400 chars.
|
||||
// We'll create content > 2000 chars with the search term appearing in each section.
|
||||
let mut builder = LexIndexBuilder::new();
|
||||
|
||||
// Build content with "quantum" appearing in multiple sections
|
||||
let section1 = "Quantum computing represents a revolutionary approach to computation. \
|
||||
The fundamental principles of quantum mechanics enable quantum computers to process \
|
||||
information in ways classical computers cannot. Quantum bits or qubits can exist in \
|
||||
superposition states, allowing quantum algorithms to explore multiple solutions \
|
||||
simultaneously. This quantum parallelism offers exponential speedups for certain \
|
||||
computational problems. Researchers continue to advance quantum hardware and software. \
|
||||
The field of quantum computing is rapidly evolving with new breakthroughs. \
|
||||
Major tech companies invest heavily in quantum research and development. \
|
||||
Quantum error correction remains a significant challenge for practical quantum computers.";
|
||||
|
||||
let section2 = "Applications of quantum computing span many domains including cryptography, \
|
||||
drug discovery, and optimization problems. Quantum cryptography promises unbreakable \
|
||||
encryption through quantum key distribution protocols. In the pharmaceutical industry, \
|
||||
quantum simulations could revolutionize how we discover new medicines. Quantum \
|
||||
algorithms like Shor's algorithm threaten current encryption standards. Financial \
|
||||
institutions explore quantum computing for portfolio optimization. The quantum \
|
||||
advantage may soon be demonstrated for practical real-world applications. Quantum \
|
||||
machine learning combines quantum computing with artificial intelligence techniques. \
|
||||
The future of quantum computing holds immense promise for scientific discovery.";
|
||||
|
||||
let full_content = format!("{} {}", section1, section2);
|
||||
assert!(
|
||||
full_content.len() > 1400,
|
||||
"Content should be long enough to create multiple sections"
|
||||
);
|
||||
|
||||
builder.add_document(
|
||||
42, // frame_id
|
||||
"mv2://docs/quantum",
|
||||
Some("Quantum Computing Overview"),
|
||||
&full_content,
|
||||
&HashMap::new(),
|
||||
);
|
||||
|
||||
let artifact = builder.finish().expect("finish should succeed");
|
||||
let index = LexIndex::decode(&artifact.bytes).expect("decode should succeed");
|
||||
|
||||
// Search for "quantum" which appears many times across both sections
|
||||
let query_tokens = tokenize("quantum");
|
||||
let matches = index.compute_matches(&query_tokens, None, None);
|
||||
|
||||
// Verify: no duplicate frame_ids in results
|
||||
let frame_ids: Vec<_> = matches.iter().map(|m| m.frame_id).collect();
|
||||
let unique_frame_ids: std::collections::HashSet<_> = frame_ids.iter().copied().collect();
|
||||
|
||||
assert_eq!(
|
||||
frame_ids.len(),
|
||||
unique_frame_ids.len(),
|
||||
"Results should not contain duplicate frame_ids. Found: {:?}",
|
||||
frame_ids
|
||||
);
|
||||
|
||||
// Should have exactly one result for frame_id 42
|
||||
assert_eq!(matches.len(), 1, "Should have exactly one match");
|
||||
assert_eq!(matches[0].frame_id, 42, "Match should be for frame_id 42");
|
||||
assert!(matches[0].score > 0.0, "Match should have a positive score");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_matches_keeps_highest_score_per_frame() {
|
||||
// Test that when multiple sections match, we keep the highest-scoring one
|
||||
let mut builder = LexIndexBuilder::new();
|
||||
|
||||
// Create content where "target" appears more times in the second section
|
||||
let section1 = "This is the first section with one target mention. \
|
||||
It contains various other words to pad the content and make it long enough \
|
||||
to be split into multiple sections by the chunking algorithm. We need quite \
|
||||
a bit of text here to ensure the sections are created properly. The content \
|
||||
continues with more filler text about various topics. Keep writing to reach \
|
||||
the section boundary. More text follows to ensure we cross the soft limit. \
|
||||
This should be enough to trigger section creation at the boundary point.";
|
||||
|
||||
let section2 = "The second section has target target target multiple times. \
|
||||
Target appears here repeatedly: target target target target. This section \
|
||||
should score higher because it has more occurrences of the search term target. \
|
||||
We mention target again to boost the score further. Target target target. \
|
||||
The abundance of target keywords makes this section rank higher in relevance.";
|
||||
|
||||
let full_content = format!("{} {}", section1, section2);
|
||||
|
||||
builder.add_document(
|
||||
99,
|
||||
"mv2://docs/multi-section",
|
||||
Some("Multi-Section Document"),
|
||||
&full_content,
|
||||
&HashMap::new(),
|
||||
);
|
||||
|
||||
let artifact = builder.finish().expect("finish");
|
||||
let index = LexIndex::decode(&artifact.bytes).expect("decode");
|
||||
|
||||
let query_tokens = tokenize("target");
|
||||
let matches = index.compute_matches(&query_tokens, None, None);
|
||||
|
||||
// Should have exactly one result (deduplicated)
|
||||
assert_eq!(
|
||||
matches.len(),
|
||||
1,
|
||||
"Should have exactly one deduplicated match"
|
||||
);
|
||||
|
||||
// The match should have the higher score (from section2 with more "target" occurrences)
|
||||
// Section1 has 1 occurrence, Section2 has ~10+ occurrences
|
||||
assert!(
|
||||
matches[0].score >= 5.0,
|
||||
"Should keep the highest-scoring match, score was: {}",
|
||||
matches[0].score
|
||||
);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user