chore: import upstream snapshot with attribution
CI / Test (macos-latest, stable) (push) Has been cancelled
CI / Test (ubuntu-latest, stable) (push) Has been cancelled
CI / Test (windows-latest, stable) (push) Has been cancelled
CI / Lint (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:45:24 +08:00
commit 70cb81e982
218 changed files with 410378 additions and 0 deletions
+179
View File
@@ -0,0 +1,179 @@
//! Basic usage example demonstrating create, put, find, and timeline operations.
//!
//! Run with: cargo run --example basic_usage
use std::path::PathBuf;
use tempfile::tempdir;
use memvid_core::{Memvid, PutOptions, Result, SearchRequest, TimelineQuery};
fn main() -> Result<()> {
// Create a temporary directory for our example
let dir = tempdir().expect("failed to create temp dir");
let path: PathBuf = dir.path().join("example.mv2");
println!("=== Memvid Core Basic Usage Example ===\n");
// ========================================
// 1. CREATE a new memory file
// ========================================
println!("1. Creating memory file at {:?}", path);
let mut mem = Memvid::create(&path)?;
println!(" Memory created successfully!\n");
// ========================================
// 2. PUT documents into the memory
// ========================================
println!("2. Adding documents to memory...");
// Simple put with just bytes
let seq1 = mem.put_bytes(b"Hello, Memvid! This is a simple text document.")?;
println!(" Added document 1, sequence: {}", seq1);
// Put with options (title, URI, tags)
let options = PutOptions::builder()
.title("Getting Started Guide")
.uri("mv2://docs/getting-started.md")
.tag("category", "documentation")
.tag("version", "2.0")
.build();
let seq2 = mem.put_bytes_with_options(
b"This guide covers the basics of using Memvid for AI memory storage.",
options,
)?;
println!(" Added document 2 (with metadata), sequence: {}", seq2);
// Add more documents
let options = PutOptions::builder()
.title("API Reference")
.uri("mv2://docs/api-reference.md")
.tag("category", "documentation")
.build();
mem.put_bytes_with_options(
b"The Memvid API provides methods for create, put, find, and timeline operations.",
options,
)?;
let options = PutOptions::builder()
.title("FAQ")
.uri("mv2://docs/faq.md")
.tag("category", "support")
.build();
mem.put_bytes_with_options(
b"Frequently asked questions about Memvid memory files and search.",
options,
)?;
// Commit changes to persist them
mem.commit()?;
println!(" Committed all changes\n");
// ========================================
// 3. STATS - Check memory statistics
// ========================================
println!("3. Memory statistics:");
let stats = mem.stats()?;
println!(" Frame count: {}", stats.frame_count);
println!(" Has lexical index: {}", stats.has_lex_index);
println!(" Has vector index: {}", stats.has_vec_index);
println!(" Has time index: {}", stats.has_time_index);
println!();
// ========================================
// 4. FIND - Search for documents
// ========================================
println!("4. Searching for documents...");
// Search for "memvid"
let request = SearchRequest {
query: "memvid".to_string(),
top_k: 10,
snippet_chars: 200,
uri: None,
scope: None,
cursor: None,
#[cfg(feature = "temporal_track")]
temporal: None,
as_of_frame: None,
as_of_ts: None,
no_sketch: false,
acl_context: None,
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
};
let response = mem.search(request)?;
println!(" Query: 'memvid'");
println!(" Total hits: {}", response.total_hits);
println!(" Elapsed: {}ms", response.elapsed_ms);
for hit in &response.hits {
let title = hit.title.as_deref().unwrap_or("Untitled");
let score = hit.score.unwrap_or(0.0);
println!(" - [{}] {} (score: {:.3})", hit.frame_id, title, score);
println!(
" Snippet: {}...",
&hit.text.chars().take(60).collect::<String>()
);
}
println!();
// Search within a scope
let request = SearchRequest {
query: "documentation".to_string(),
top_k: 10,
snippet_chars: 100,
uri: None,
scope: Some("mv2://docs/".to_string()),
cursor: None,
#[cfg(feature = "temporal_track")]
temporal: None,
as_of_frame: None,
as_of_ts: None,
no_sketch: false,
acl_context: None,
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
};
let response = mem.search(request)?;
println!(" Query: 'documentation' (scope: mv2://docs/)");
println!(" Total hits: {}", response.total_hits);
println!();
// ========================================
// 5. TIMELINE - Browse documents chronologically
// ========================================
println!("5. Timeline (chronological view):");
let timeline = mem.timeline(TimelineQuery::default())?;
for entry in &timeline {
let uri = entry.uri.as_deref().unwrap_or("(no uri)");
println!(
" [{}] {} - {}",
entry.frame_id,
uri,
entry.preview.chars().take(40).collect::<String>()
);
}
println!();
// ========================================
// 6. REOPEN - Close and reopen the memory
// ========================================
println!("6. Closing and reopening memory...");
drop(mem);
let reopened = Memvid::open(&path)?;
let stats = reopened.stats()?;
println!(" Reopened successfully!");
println!(" Frame count after reopen: {}", stats.frame_count);
println!();
// ========================================
// 7. VERIFY - Check file integrity
// ========================================
println!("7. Verifying file integrity...");
drop(reopened); // Close the memory before verifying
let report = Memvid::verify(&path, false)?;
println!(" Verification status: {:?}", report.overall_status);
println!();
println!("=== Example completed successfully! ===");
Ok(())
}
+260
View File
@@ -0,0 +1,260 @@
//! CLIP Visual Search Example
//!
//! Demonstrates using CLIP embeddings to search PDF pages and images
//! using natural language queries.
//!
//! Run with:
//! ```bash
//! cargo run --example clip_visual_search --features clip,pdfium -- /path/to/pdf
//! ```
//!
//! Prerequisites:
//! 1. Download the MobileCLIP-S2 ONNX models:
//! ```bash
//! mkdir -p ~/.local/share/memvid/models
//! curl -L 'https://huggingface.co/Xenova/mobileclip_s2/resolve/main/onnx/vision_model_int8.onnx' \
//! -o ~/.local/share/memvid/models/mobileclip-s2_vision.onnx
//! curl -L 'https://huggingface.co/Xenova/mobileclip_s2/resolve/main/onnx/text_model_int8.onnx' \
//! -o ~/.local/share/memvid/models/mobileclip-s2_text.onnx
//! ```
//!
//! 2. For PDF page rendering, install pdfium:
//! - macOS: `brew install nicbarker/pdfium/pdfium-mac-arm64` or `pdfium-mac-x64`
//! - Linux: Download from https://github.com/nicbarker/pdfium-builds/releases
fn main() -> memvid_core::Result<()> {
#[cfg(not(feature = "clip"))]
{
eprintln!("This example requires the 'clip' feature.");
eprintln!("Run with: cargo run --example clip_visual_search --features clip");
Ok(())
}
#[cfg(feature = "clip")]
{
use memvid_core::clip::{ClipConfig, ClipIndex, ClipIndexBuilder, ClipModel};
use std::env;
use std::path::PathBuf;
use tempfile::tempdir;
println!("=== CLIP Visual Search Example ===\n");
// Get PDF path from args or use default
let args: Vec<String> = env::args().collect();
let pdf_path = if args.len() > 1 {
PathBuf::from(&args[1])
} else {
// Default to the SP Global Impact Report if no path provided
PathBuf::from(
"/Users/olow/Desktop/memvid-org/brickfield/sp-global-impact-report-2024.pdf",
)
};
if !pdf_path.exists() {
eprintln!("PDF not found: {}", pdf_path.display());
eprintln!(
"Usage: cargo run --example clip_visual_search --features clip,pdfium -- /path/to/pdf"
);
return Ok(());
}
println!("PDF: {}", pdf_path.display());
// Initialize CLIP model
println!("\n1. Loading CLIP model...");
let config = ClipConfig::default();
println!(" Model: {}", config.model_name);
println!(" Models dir: {}", config.models_dir.display());
let clip = match ClipModel::new(config) {
Ok(model) => {
println!(" Model initialized (lazy loading)");
model
}
Err(e) => {
eprintln!(" Failed to initialize CLIP: {}", e);
eprintln!("\n Make sure to download the models first:");
eprintln!(" mkdir -p ~/.local/share/memvid/models");
eprintln!(
" curl -L 'https://huggingface.co/Xenova/mobileclip_s2/resolve/main/onnx/vision_model_int8.onnx' \\"
);
eprintln!(" -o ~/.local/share/memvid/models/mobileclip-s2_vision.onnx");
return Ok(());
}
};
// For this demo, we'll create synthetic embeddings since PDF rendering
// requires pdfium which may not be available
println!("\n2. Building CLIP index with sample embeddings...");
let mut builder = ClipIndexBuilder::new();
// Simulate embeddings for 10 "pages" with different visual concepts
// In real usage, you would:
// 1. Render each PDF page to an image
// 2. Pass the image to clip.encode_image(&image)
// 3. Store the embedding with the page's frame_id
let sample_concepts: Vec<(&str, Vec<f32>)> = vec![
// These would be real embeddings from actual images in production
(
"charts and graphs",
random_embedding(clip.dims() as usize, 1),
),
(
"sustainability report cover",
random_embedding(clip.dims() as usize, 2),
),
(
"ESG metrics table",
random_embedding(clip.dims() as usize, 3),
),
(
"environmental impact diagram",
random_embedding(clip.dims() as usize, 4),
),
(
"carbon emissions chart",
random_embedding(clip.dims() as usize, 5),
),
(
"renewable energy infographic",
random_embedding(clip.dims() as usize, 6),
),
(
"corporate governance structure",
random_embedding(clip.dims() as usize, 7),
),
(
"diversity statistics",
random_embedding(clip.dims() as usize, 8),
),
(
"supply chain map",
random_embedding(clip.dims() as usize, 9),
),
(
"financial highlights",
random_embedding(clip.dims() as usize, 10),
),
];
for (i, (concept, embedding)) in sample_concepts.iter().enumerate() {
builder.add_document(i as u64, Some(i as u32), embedding.clone());
println!(" Added page {} ({})", i + 1, concept);
}
let artifact = builder.finish()?;
println!(
"\n Index built: {} vectors, {} dimensions",
artifact.vector_count, artifact.dimension
);
// Decode the index for searching
let index = ClipIndex::decode(&artifact.bytes)?;
// Demonstrate text-to-image search
println!("\n3. Searching with natural language queries...\n");
// Try encoding a text query
println!(" Encoding query: 'sustainability charts'");
match clip.encode_text("sustainability charts") {
Ok(query_embedding) => {
println!(" Query embedding: {} dimensions", query_embedding.len());
let hits = index.search(&query_embedding, 3);
println!("\n Top 3 matches:");
for (rank, hit) in hits.iter().enumerate() {
let concept = sample_concepts
.get(hit.frame_id as usize)
.map(|(c, _)| *c)
.unwrap_or("unknown");
println!(
" {}. Page {} ({}) - distance: {:.4}",
rank + 1,
hit.frame_id + 1,
concept,
hit.distance
);
}
}
Err(e) => {
eprintln!(" Failed to encode text (model not loaded): {}", e);
eprintln!(" Make sure the text model ONNX file is downloaded.");
}
}
// Demo with Memvid integration
println!("\n4. Creating Memvid memory with CLIP support...");
let dir = tempdir().expect("failed to create temp dir");
let path = dir.path().join("clip_demo.mv2");
let mut mem = memvid_core::Memvid::create(&path)?;
// Enable CLIP index
mem.enable_clip()?;
println!(" CLIP index enabled");
// Add some sample documents
let options = memvid_core::PutOptions::builder()
.title("SP Global Impact Report 2024 - Page 1")
.uri("mv2://reports/sp-global/page-1")
.build();
mem.put_bytes_with_options(
b"This page contains sustainability charts and ESG metrics.",
options,
)?;
mem.commit()?;
println!(" Added sample document");
let stats = mem.stats()?;
println!("\n Memory stats:");
println!(" - Frames: {}", stats.frame_count);
println!(" - Has CLIP index: {}", stats.has_clip_index);
// Search CLIP index (would use pre-computed query embedding in production)
// Since we haven't added actual CLIP embeddings to the memory,
// the search would return empty results - this is just to show the API
println!("\n=== Example completed successfully! ===");
println!("\nTo use CLIP in production:");
println!("1. During ingestion: Generate CLIP embeddings for images/PDF pages");
println!("2. Store embeddings in the CLIP index alongside text content");
println!("3. At query time: Encode the text query with clip.encode_text()");
println!("4. Search the CLIP index with mem.search_clip(&query_embedding, limit)");
Ok(())
}
}
/// Generate a pseudo-random embedding for demo purposes
/// In production, use clip.encode_image() on actual images
#[cfg(feature = "clip")]
fn random_embedding(dims: usize, seed: u64) -> Vec<f32> {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
let mut embedding = Vec::with_capacity(dims);
for i in 0..dims {
(seed, i).hash(&mut hasher);
let hash = hasher.finish();
// Generate pseudo-random float between -1 and 1
let val = ((hash as f32) / (u64::MAX as f32)) * 2.0 - 1.0;
embedding.push(val);
hasher = DefaultHasher::new();
}
// L2 normalize
let norm: f32 = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 1e-10 {
for v in &mut embedding {
*v /= norm;
}
}
embedding
}
+162
View File
@@ -0,0 +1,162 @@
//! Generate visual performance comparison report
//!
//! Run with: cargo run --example generate_performance_report --features lex
use memvid_core::{Memvid, PutOptions, SearchRequest};
use std::time::Instant;
fn main() -> memvid_core::Result<()> {
println!("=== Search Precision Performance Report ===\n");
// Create test corpus
println!("Setting up test corpus (1000 documents)...");
let temp_file = "/tmp/perf_test.mv2";
let _ = std::fs::remove_file(temp_file);
let mut mem = Memvid::create(temp_file)?;
// Add 1000 documents with controlled distribution
for i in 0..1000 {
let topic = match i % 5 {
0 => ("machine learning", "neural networks"),
1 => ("python programming", "software development"),
2 => ("machine learning with python", "data science"),
3 => ("rust systems programming", "memory safety"),
_ => ("web development", "javascript frameworks"),
};
let content = format!(
"Document {} about {}. This article covers {} in depth.",
i, topic.0, topic.1
);
mem.put_bytes_with_options(
content.as_bytes(),
PutOptions::builder()
.title(format!("Doc {} - {}", i, topic.0))
.build(),
)?;
if (i + 1) % 100 == 0 {
mem.commit()?;
}
}
mem.commit()?;
println!("✓ Corpus ready\n");
// Test queries
let test_queries = vec![
("machine python", "Both terms"),
("machine learning python", "Three terms"),
("python programming development", "Three terms"),
("rust memory safety", "Two terms"),
];
println!("┌─────────────────────────────────────────────────────────────────────┐");
println!("│ QUERY PERFORMANCE METRICS │");
println!("├─────────────────────────────────────────────────────────────────────┤");
for (query, desc) in &test_queries {
// Warm up
for _ in 0..10 {
let _ = mem.search(SearchRequest {
query: query.to_string(),
top_k: 100,
snippet_chars: 200,
uri: None,
scope: None,
cursor: None,
#[cfg(feature = "temporal_track")]
temporal: None,
as_of_frame: None,
as_of_ts: None,
no_sketch: false,
acl_context: None,
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
})?;
}
// Measure
let iterations = 100;
let start = Instant::now();
let mut total_results = 0;
let mut total_relevant = 0;
for _ in 0..iterations {
let results = mem.search(SearchRequest {
query: query.to_string(),
top_k: 100,
snippet_chars: 200,
uri: None,
scope: None,
cursor: None,
#[cfg(feature = "temporal_track")]
temporal: None,
as_of_frame: None,
as_of_ts: None,
no_sketch: false,
acl_context: None,
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
})?;
let terms: Vec<&str> = query.split_whitespace().collect();
let relevant = results
.hits
.iter()
.filter(|hit| {
let text = hit.text.to_lowercase();
terms.iter().all(|term| text.contains(term))
})
.count();
total_results += results.hits.len();
total_relevant += relevant;
}
let elapsed = start.elapsed();
let avg_latency_us = elapsed.as_micros() / iterations as u128; // FIX: Cast to u128
let avg_results = total_results / iterations;
let avg_relevant = total_relevant / iterations;
let precision = if avg_results > 0 {
(avg_relevant as f64 / avg_results as f64) * 100.0
} else {
0.0
};
println!("");
println!("│ Query: \"{}\" ({})", query, desc);
println!("│ Latency: {:.2}ms", avg_latency_us as f64 / 1000.0);
println!("│ Results: {} docs", avg_results);
println!("│ Relevant: {} docs", avg_relevant);
println!("│ Precision: {:.1}%", precision);
println!("│ Memory: ~{} KB", (avg_results * 3).max(1));
}
println!("└─────────────────────────────────────────────────────────────────────┘");
// Comparison with hypothetical OR behavior
println!("\n┌─────────────────────────────────────────────────────────────────────┐");
println!("│ COMPARISON: AND vs OR (Estimated) │");
println!("├─────────────────────────────────────────────────────────────────────┤");
println!("");
println!("│ Query: \"machine python\"");
println!("");
println!("│ WITH AND (Current): │ WITH OR (Previous):");
println!("│ • Results: ~5-8 docs │ • Results: ~80-120 docs");
println!("│ • Precision: 100% │ • Precision: ~6-8%");
println!("│ • Memory: ~20 KB │ • Memory: ~300 KB");
println!("│ • Processing: 6-10ms │ • Processing: 96-144ms");
println!("");
println!("│ IMPROVEMENT: ");
println!("│ ✓ 15x better precision (6% → 100%) ");
println!("│ ✓ 93% less memory (300KB → 20KB) ");
println!("│ ✓ 93% faster processing (120ms → 8ms) ");
println!("│ ✓ No query latency regression (~1.2ms) ");
println!("");
println!("└─────────────────────────────────────────────────────────────────────┘");
std::fs::remove_file(temp_file)?;
Ok(())
}
+163
View File
@@ -0,0 +1,163 @@
//! Example demonstrating OpenAI API embedding usage.
//!
//! This example shows how to:
//! - Create an OpenAI embedder with default configuration
//! - Generate embeddings using the API
//! - Compute cosine similarity between embeddings
//! - Use different models (small, large, ada)
//!
//! ## Prerequisites
//!
//! Set your OpenAI API key:
//! ```bash
//! export OPENAI_API_KEY="sk-..."
//! ```
//!
//! ## Run
//!
//! ```bash
//! cargo run --example openai_embedding --features api_embed
//! ```
use memvid_core::Result;
#[cfg(feature = "api_embed")]
use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
#[cfg(feature = "api_embed")]
use memvid_core::types::embedding::EmbeddingProvider;
/// Compute cosine similarity between two vectors
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
assert_eq!(a.len(), b.len(), "Vectors must have same length");
let dot_product: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm_a > 0.0 && norm_b > 0.0 {
dot_product / (norm_a * norm_b)
} else {
0.0
}
}
#[cfg(feature = "api_embed")]
fn main() -> Result<()> {
println!("=== OpenAI Embedding Example ===\n");
// Check if API key is set
if std::env::var("OPENAI_API_KEY").is_err() {
eprintln!("Error: OPENAI_API_KEY environment variable not set.");
eprintln!("Please set it with: export OPENAI_API_KEY=\"sk-...\"");
std::process::exit(1);
}
// Create embedder with default config (text-embedding-3-small, 1536 dimensions)
println!("Creating OpenAI embedder (text-embedding-3-small)...");
let config = OpenAIConfig::default();
let embedder = OpenAIEmbedder::new(config)?;
println!("Model: {}", embedder.model());
println!("Kind: {}", embedder.kind());
println!("Dimensions: {}", embedder.dimension());
println!("Ready: {}\n", embedder.is_ready());
// Example 1: Single text embedding
println!("--- Example 1: Single Text Embedding ---");
let text = "The quick brown fox jumps over the lazy dog";
println!("Embedding text: \"{}\"", text);
let embedding = embedder.embed_text(text)?;
println!("Generated embedding of dimension {}\n", embedding.len());
// Example 2: Semantic similarity
println!("--- Example 2: Semantic Similarity ---");
let text1 = "Machine learning and artificial intelligence";
let text2 = "Deep neural networks for AI applications";
let text3 = "The history of ancient Rome";
let emb1 = embedder.embed_text(text1)?;
let emb2 = embedder.embed_text(text2)?;
let emb3 = embedder.embed_text(text3)?;
println!("Text 1: \"{}\"", text1);
println!("Text 2: \"{}\"", text2);
println!("Text 3: \"{}\"", text3);
println!();
let sim_1_2 = cosine_similarity(&emb1, &emb2);
let sim_1_3 = cosine_similarity(&emb1, &emb3);
let sim_2_3 = cosine_similarity(&emb2, &emb3);
println!("Similarity (1 ↔ 2): {:.4}", sim_1_2);
println!("Similarity (1 ↔ 3): {:.4}", sim_1_3);
println!("Similarity (2 ↔ 3): {:.4}", sim_2_3);
if sim_1_2 > sim_1_3 {
println!("\n✓ Related texts (1 & 2) have higher similarity than unrelated (1 & 3)!");
}
println!();
// Example 3: Batch processing
println!("--- Example 3: Batch Processing ---");
let documents = vec![
"Python programming language",
"JavaScript web development",
"Rust systems programming",
"Italian cooking recipes",
];
println!("Processing {} documents in batch...", documents.len());
let batch_embeddings = embedder.embed_batch(&documents)?;
println!(
"✓ Generated {} embeddings of dimension {}\n",
batch_embeddings.len(),
batch_embeddings.first().map(|e| e.len()).unwrap_or(0)
);
// Find most similar pair
let mut max_sim = 0.0;
let mut max_pair = (0, 0);
for i in 0..batch_embeddings.len() {
for j in (i + 1)..batch_embeddings.len() {
let sim = cosine_similarity(&batch_embeddings[i], &batch_embeddings[j]);
if sim > max_sim {
max_sim = sim;
max_pair = (i, j);
}
}
}
println!("Most similar pair (similarity: {:.4}):", max_sim);
println!(" [{}] \"{}\"", max_pair.0, documents[max_pair.0]);
println!(" [{}] \"{}\"\n", max_pair.1, documents[max_pair.1]);
// Example 4: Available models
println!("--- Example 4: Available Models ---");
println!("OpenAI embedding models:");
println!(" - text-embedding-3-small (1536d) - Default, fastest, cheapest");
println!(" - text-embedding-3-large (3072d) - Highest quality");
println!(" - text-embedding-ada-002 (1536d) - Legacy model");
println!();
println!("To use a different model:");
println!(" let config = OpenAIConfig::large();");
println!(" let embedder = OpenAIEmbedder::new(config)?;");
println!();
println!("=== Example Complete ===");
println!("\nKey takeaways:");
println!("✓ API embeddings require OPENAI_API_KEY environment variable");
println!("✓ text-embedding-3-small is fast and cost-effective");
println!("✓ Batch processing reduces API calls for multiple texts");
println!("✓ Similar texts have higher cosine similarity scores");
Ok(())
}
#[cfg(not(feature = "api_embed"))]
fn main() {
eprintln!("This example requires the 'api_embed' feature.");
eprintln!("Run with: cargo run --example openai_embedding --features api_embed");
std::process::exit(1);
}
+150
View File
@@ -0,0 +1,150 @@
//! PDF ingestion example demonstrating how to ingest and search PDF documents.
//!
//! This example demonstrates PDF text extraction, chunking, and semantic search.
//!
//! Run with:
//! ```bash
//! cargo run --example pdf_ingestion -- /path/to/pdf
//! ```
use std::env;
use std::path::PathBuf;
use tempfile::tempdir;
use memvid_core::{Memvid, PutOptions, Result, SearchRequest};
fn main() -> Result<()> {
// Get PDF path from args
let args: Vec<String> = env::args().collect();
if args.len() < 2 {
eprintln!("Usage: cargo run --example pdf_ingestion -- /path/to/pdf");
eprintln!("\nExample:");
eprintln!(" cargo run --example pdf_ingestion -- examples/1706.03762v7.pdf");
return Ok(());
}
let pdf_path = PathBuf::from(&args[1]);
if !pdf_path.exists() {
eprintln!("ERROR: PDF file not found at {:?}", pdf_path);
eprintln!("Usage: cargo run --example pdf_ingestion -- /path/to/pdf");
return Ok(());
}
// Create a temporary directory for our memory file
let dir = tempdir().expect("failed to create temp dir");
let mv2_path: PathBuf = dir.path().join("paper.mv2");
println!("=== Memvid PDF Ingestion Example ===\n");
// ========================================
// 1. CREATE a new memory file
// ========================================
println!("1. Creating memory file...");
let mut mem = Memvid::create(&mv2_path)?;
println!(" Memory created at {:?}\n", mv2_path);
// ========================================
// 2. INGEST the PDF file
// ========================================
println!("2. Ingesting PDF: {:?}", pdf_path);
// Read the PDF file
let pdf_bytes = std::fs::read(&pdf_path)?;
println!(" PDF size: {} bytes", pdf_bytes.len());
// Put the PDF with metadata
// Extract filename for title
let title = pdf_path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("PDF Document")
.to_string();
let options = PutOptions::builder()
.title(&title)
.uri(format!(
"mv2://pdfs/{}",
pdf_path.file_name().unwrap_or_default().to_string_lossy()
))
.build();
let frame_id = mem.put_bytes_with_options(&pdf_bytes, options)?;
println!(" Ingested as frame: {}", frame_id);
// Commit changes
mem.commit()?;
println!(" Committed successfully!\n");
// ========================================
// 3. CHECK memory statistics
// ========================================
println!("3. Memory statistics:");
let stats = mem.stats()?;
println!(" Frame count: {}", stats.frame_count);
println!(" Has lexical index: {}", stats.has_lex_index);
println!();
// ========================================
// 4. SEARCH the ingested PDF
// ========================================
println!("4. Searching the paper...\n");
// Search for "attention"
let queries = [
"attention mechanism",
"transformer architecture",
"self-attention",
"encoder decoder",
"positional encoding",
];
for query in queries {
let request = SearchRequest {
query: query.to_string(),
top_k: 3,
snippet_chars: 150,
uri: None,
scope: None,
cursor: None,
#[cfg(feature = "temporal_track")]
temporal: None,
as_of_frame: None,
as_of_ts: None,
no_sketch: false,
acl_context: None,
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
};
let response = mem.search(request)?;
println!(" Query: '{}'", query);
println!(
" Hits: {} ({}ms)",
response.total_hits, response.elapsed_ms
);
for (i, hit) in response.hits.iter().take(2).enumerate() {
let snippet = hit
.text
.chars()
.take(100)
.collect::<String>()
.replace('\n', " ");
println!(" {}. {}...", i + 1, snippet);
}
println!();
}
// ========================================
// 5. VERIFY file integrity
// ========================================
println!("5. Verifying file integrity...");
drop(mem);
let report = Memvid::verify(&mv2_path, false)?;
println!(" Status: {:?}", report.overall_status);
println!();
println!("=== PDF ingestion example completed! ===");
Ok(())
}
+101
View File
@@ -0,0 +1,101 @@
//! Benchmark comparing SIMD vs Scalar L2 distance calculations.
//!
//! Run with: `cargo run --example simd_benchmark --features simd --release`
use std::hint::black_box;
use std::time::Instant;
fn main() {
let num_vectors = 10_000;
let dims = 384; // Standard embedding dimension
let num_iterations = 100;
println!("SIMD vs Scalar L2 Distance Benchmark");
println!("=====================================");
println!("Vectors: {}", num_vectors);
println!("Dimensions: {}", dims);
println!("Iterations: {}", num_iterations);
println!();
// Generate random vectors
let query: Vec<f32> = (0..dims).map(|i| (i as f32 * 0.001) % 1.0).collect();
let vectors: Vec<Vec<f32>> = (0..num_vectors)
.map(|v| (0..dims).map(|i| ((v + i) as f32 * 0.0017) % 1.0).collect())
.collect();
// Benchmark SIMD version
let simd_start = Instant::now();
let mut simd_sum = 0.0f32;
for _ in 0..num_iterations {
for vec in &vectors {
simd_sum += black_box(l2_distance_simd(black_box(&query), black_box(vec)));
}
}
let simd_elapsed = simd_start.elapsed();
black_box(simd_sum);
// Benchmark Scalar version
let scalar_start = Instant::now();
let mut scalar_sum = 0.0f32;
for _ in 0..num_iterations {
for vec in &vectors {
scalar_sum += black_box(l2_distance_scalar(black_box(&query), black_box(vec)));
}
}
let scalar_elapsed = scalar_start.elapsed();
black_box(scalar_sum);
// Results
let total_ops = num_vectors * num_iterations;
let simd_per_op_ns = simd_elapsed.as_nanos() as f64 / total_ops as f64;
let scalar_per_op_ns = scalar_elapsed.as_nanos() as f64 / total_ops as f64;
let speedup = scalar_elapsed.as_nanos() as f64 / simd_elapsed.as_nanos() as f64;
println!("Results:");
println!("--------");
println!(
"SIMD: {:>8.2}ms total, {:>6.1}ns per distance",
simd_elapsed.as_secs_f64() * 1000.0,
simd_per_op_ns
);
println!(
"Scalar: {:>8.2}ms total, {:>6.1}ns per distance",
scalar_elapsed.as_secs_f64() * 1000.0,
scalar_per_op_ns
);
println!();
println!("Speedup: {:.2}x", speedup);
// Verify correctness
let simd_result = l2_distance_simd(&query, &vectors[0]);
let scalar_result = l2_distance_scalar(&query, &vectors[0]);
let diff = (simd_result - scalar_result).abs();
println!();
println!("Correctness check:");
println!(" SIMD result: {:.8}", simd_result);
println!(" Scalar result: {:.8}", scalar_result);
println!(" Difference: {:.2e} (should be < 1e-5)", diff);
assert!(diff < 1e-4, "Results differ too much!");
println!(" ✓ Results match!");
}
/// SIMD L2 distance using the wide crate
#[cfg(feature = "simd")]
fn l2_distance_simd(a: &[f32], b: &[f32]) -> f32 {
memvid_core::simd::l2_distance_simd(a, b)
}
#[cfg(not(feature = "simd"))]
fn l2_distance_simd(a: &[f32], b: &[f32]) -> f32 {
l2_distance_scalar(a, b)
}
/// Scalar L2 distance (the OLD implementation)
#[inline(never)]
fn l2_distance_scalar(a: &[f32], b: &[f32]) -> f32 {
a.iter()
.zip(b.iter())
.map(|(x, y)| (x - y).powi(2))
.sum::<f32>()
.sqrt()
}
+132
View File
@@ -0,0 +1,132 @@
//! Test to demonstrate the implicit OR bug in query parsing
//!
//! Run with: cargo run --example test_implicit_or_bug --features lex
use memvid_core::{Memvid, PutOptions, SearchRequest};
fn main() -> memvid_core::Result<()> {
let test_file = "/tmp/test_implicit.mv2";
// Clean up old file if exists
let _ = std::fs::remove_file(test_file);
println!("Creating test .mv2 file...");
let mut mem = Memvid::create(test_file)?;
// Add documents
println!("Adding test documents...");
mem.put_bytes_with_options(
b"I love machine learning",
PutOptions::builder().title("Doc 1").build(),
)?;
mem.put_bytes_with_options(
b"I love Python programming",
PutOptions::builder().title("Doc 2").build(),
)?;
mem.put_bytes_with_options(
b"Machine learning with Python is awesome",
PutOptions::builder().title("Doc 3").build(),
)?;
mem.commit()?;
println!();
println!("=== TESTING IMPLICIT OPERATOR BEHAVIOR ===");
println!("Query: 'machine python'");
println!();
println!("Expected behavior (AND): Should return 1 result");
println!(" - Only Doc 3 has BOTH 'machine' AND 'python'");
println!();
println!("Current behavior (OR): Returns 3 results");
println!(" - Any doc with 'machine' OR 'python'");
println!();
// Search with implicit operator
let results = mem.search(SearchRequest {
query: "machine python".to_string(),
top_k: 10,
snippet_chars: 200,
uri: None,
scope: None,
cursor: None,
#[cfg(feature = "temporal_track")]
temporal: None,
as_of_frame: None,
as_of_ts: None,
no_sketch: false,
acl_context: None,
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
})?;
println!("ACTUAL RESULTS: {} documents found", results.hits.len());
println!();
for (i, hit) in results.hits.iter().enumerate() {
let title = hit
.title
.as_ref()
.unwrap_or(&"Untitled".to_string())
.clone();
let text_lower = hit.text.to_lowercase();
let has_machine = text_lower.contains("machine");
let has_python = text_lower.contains("python");
let is_relevant = has_machine && has_python;
let status = if is_relevant {
"✓ RELEVANT"
} else {
"✗ NOT RELEVANT"
};
println!(
" {}. {} [machine={}, python={}] {}",
i + 1,
title,
has_machine,
has_python,
status
);
}
println!();
// Calculate precision
let relevant_count = results
.hits
.iter()
.filter(|hit| {
let text = hit.text.to_lowercase();
text.contains("machine") && text.contains("python")
})
.count();
let precision = if results.hits.is_empty() {
0.0
} else {
(relevant_count as f64 / results.hits.len() as f64) * 100.0
};
println!(
"PRECISION: {:.1}% ({}/{} results are relevant)",
precision,
relevant_count,
results.hits.len()
);
println!();
// Verdict
if results.hits.len() == 1 && relevant_count == 1 {
println!("✓ CORRECT: Query uses implicit AND");
println!(" Perfect precision - returns only relevant docs");
} else if results.hits.len() > 1 {
println!("✗ BUG DETECTED: Query uses implicit OR");
println!(" Low precision - returns docs with EITHER term");
println!();
} else {
println!("? NO RESULTS: Check if lex feature is enabled");
}
// Cleanup
std::fs::remove_file(test_file)?;
Ok(())
}
+81
View File
@@ -0,0 +1,81 @@
//! Whisper transcription example demonstrating audio transcription.
//!
//! Run with:
//! ```bash
//! cargo run --example test_whisper --features whisper -- /path/to/audio
//! ```
#[cfg(feature = "whisper")]
use memvid_core::{WhisperConfig, WhisperTranscriber};
#[cfg(feature = "whisper")]
use std::env;
#[cfg(feature = "whisper")]
use std::path::PathBuf;
#[cfg(feature = "whisper")]
use std::time::Instant;
#[cfg(not(feature = "whisper"))]
fn main() {
eprintln!(
"This example requires the `whisper` feature.\n\
Re-run with:\n\
cargo run --example test_whisper --features whisper -- /path/to/audio"
);
}
#[cfg(feature = "whisper")]
fn main() {
// Get audio path from args
let args: Vec<String> = env::args().collect();
if args.len() < 2 {
eprintln!("Usage: cargo run --example test_whisper --features whisper -- /path/to/audio");
eprintln!("\nExample:");
eprintln!(
" cargo run --example test_whisper --features whisper -- examples/call_sale.mp3"
);
return;
}
let audio_path = PathBuf::from(&args[1]);
if !audio_path.exists() {
eprintln!("ERROR: Audio file not found at {:?}", audio_path);
eprintln!("Usage: cargo run --example test_whisper --features whisper -- /path/to/audio");
return;
}
println!("=== Whisper Transcription Example ===\n");
println!("Creating Whisper transcriber...");
let start = Instant::now();
let config = WhisperConfig::default();
println!("Model dir: {:?}", config.models_dir);
println!("Model name: {}", config.model_name);
let mut transcriber = WhisperTranscriber::new(&config).expect("Failed to create transcriber");
println!("Transcriber created in {:?}", start.elapsed());
println!("\nTranscribing audio file: {}", audio_path.display());
let start = Instant::now();
match transcriber.transcribe_file(&audio_path) {
Ok(result) => {
println!("Transcription completed in {:?}", start.elapsed());
println!("\n=== Transcription Result ===");
println!("Duration: {:.2} seconds", result.duration_secs);
println!("Language: {}", result.language);
println!("\nText:\n{}", result.text);
if !result.segments.is_empty() {
println!("\n=== Segments ===");
for seg in &result.segments {
println!("[{:.2}s - {:.2}s] {}", seg.start, seg.end, seg.text);
}
}
}
Err(e) => {
eprintln!("Transcription failed: {}", e);
}
}
println!("\n=== Transcription example completed! ===");
}
+139
View File
@@ -0,0 +1,139 @@
//! Benchmark demonstrating the performance benefit of embedding caching.
//!
//! This example compares performance with and without caching for repeated texts.
//!
//! ## Prerequisites
//!
//! Download the BGE-small model:
//!
//! ```bash
//! mkdir -p ~/.cache/memvid/text-models
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
//! ```
//!
//! ## Run
//!
//! ```bash
//! cargo run --example text_embed_cache_bench --features vec --release
//! ```
use memvid_core::Result;
#[cfg(feature = "vec")]
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
use std::time::Instant;
fn main() -> Result<()> {
println!("╔════════════════════════════════════════════════════════════╗");
println!("║ Embedding Cache Benchmark ║");
println!("╚════════════════════════════════════════════════════════════╝\n");
// Test texts with intentional repeats
let test_texts = vec![
"machine learning",
"artificial intelligence",
"deep learning",
"neural networks",
"natural language processing",
"machine learning", // Repeat 1
"artificial intelligence", // Repeat 2
"computer vision",
"deep learning", // Repeat 3
"machine learning", // Repeat 4
];
println!(
"Test data: {} texts ({} unique, {} repeats)\n",
test_texts.len(),
7, // unique
3
); // repeats
// ---------- Test with Cache Enabled ----------
println!("──────────────────────────────────────────────────────────");
println!("Test 1: WITH Cache (enabled by default)");
println!("──────────────────────────────────────────────────────────");
let config_cached = TextEmbedConfig::default();
let embedder_cached = LocalTextEmbedder::new(config_cached)?;
println!("Processing texts...");
let start = Instant::now();
for (i, text) in test_texts.iter().enumerate() {
let _ = embedder_cached.encode_text(text)?;
if i < test_texts.len() - 1 {
print!(".");
}
}
println!("!");
let cached_time = start.elapsed();
if let Some(stats) = embedder_cached.cache_stats() {
println!("\n📊 Cache Statistics:");
println!(" Hits: {}", stats.hits);
println!(" Misses: {}", stats.misses);
println!(" Size: {}/{}", stats.size, stats.capacity);
println!(" Hit Rate: {:.1}%", stats.hit_rate() * 100.0);
}
println!("\n⏱️ Total Time: {:?}", cached_time);
// ---------- Test with Cache Disabled ----------
println!("\n──────────────────────────────────────────────────────────");
println!("Test 2: WITHOUT Cache (force disabled)");
println!("──────────────────────────────────────────────────────────");
let config_uncached = TextEmbedConfig {
enable_cache: false,
..Default::default()
};
let embedder_uncached = LocalTextEmbedder::new(config_uncached)?;
println!("Processing texts...");
let start = Instant::now();
for (i, text) in test_texts.iter().enumerate() {
let _ = embedder_uncached.encode_text(text)?;
if i < test_texts.len() - 1 {
print!(".");
}
}
println!("!");
let uncached_time = start.elapsed();
println!("\n⏱️ Total Time: {:?}", uncached_time);
// ---------- Results ----------
println!("\n╔════════════════════════════════════════════════════════════╗");
println!("║ Results ║");
println!("╚════════════════════════════════════════════════════════════╝\n");
let speedup = uncached_time.as_secs_f64() / cached_time.as_secs_f64();
println!("┌────────────────┬──────────────┬──────────────┐");
println!("│ Configuration │ Time │ Speedup │");
println!("├────────────────┼──────────────┼──────────────┤");
println!(
"│ With Cache │ {:>10.3}s │ {:>9.2}x │",
cached_time.as_secs_f64(),
speedup
);
println!(
"│ Without Cache │ {:>10.3}s │ baseline │",
uncached_time.as_secs_f64()
);
println!("└────────────────┴──────────────┴──────────────┘\n");
if speedup > 1.0 {
println!("✅ Cache provides {:.1}% speedup!", (speedup - 1.0) * 100.0);
} else {
println!("⚠️ No speedup observed (may indicate all cache misses)");
}
println!("\n💡 Note: Speedup increases with more repeated texts.");
println!(" Real-world applications with 50-90% repeated queries");
println!(" can see 2-10x improvements!\n");
Ok(())
}
+178
View File
@@ -0,0 +1,178 @@
//! Example demonstrating local text embedding usage.
//!
//! This example shows how to:
//! - Create a local text embedder with default configuration (BGE-small)
//! - Generate embeddings for sample texts
//! - Compute cosine similarity between embeddings
//! - Batch process multiple texts
//! - Use different models (BGE-base, Nomic, GTE-large)
//!
//! ## Prerequisites
//!
//! Before running this example, download the BGE-small model:
//!
//! ```bash
//! mkdir -p ~/.cache/memvid/text-models
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
//! ```
//!
//! ## Run
//!
//! ```bash
//! cargo run --example text_embedding --features vec
//! ```
use memvid_core::Result;
#[cfg(feature = "vec")]
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
#[cfg(feature = "vec")]
use memvid_core::types::embedding::EmbeddingProvider;
/// Compute cosine similarity between two vectors
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
assert_eq!(a.len(), b.len(), "Vectors must have same length");
let dot_product: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm_a > 0.0 && norm_b > 0.0 {
dot_product / (norm_a * norm_b)
} else {
0.0
}
}
fn main() -> Result<()> {
println!("=== Local Text Embedding Example ===\n");
// Create embedder with default config (BGE-small, 384 dimensions)
println!("Creating local text embedder (BGE-small-en-v1.5)...");
let config = TextEmbedConfig::default();
let embedder = LocalTextEmbedder::new(config)?;
println!("Model: {}", embedder.model());
println!("Kind: {}", embedder.kind());
println!("Dimensions: {}", embedder.dimension());
println!("Ready: {}\n", embedder.is_ready());
// Example 1: Generate embeddings for sample texts
println!("--- Example 1: Single Text Embeddings ---");
let text1 = "The quick brown fox jumps over the lazy dog";
let text2 = "A fast auburn canine leaps above an idle hound";
let text3 = "Python is a programming language";
println!("Generating embeddings...");
let emb1 = embedder.embed_text(text1)?;
let emb2 = embedder.embed_text(text2)?;
let emb3 = embedder.embed_text(text3)?;
println!("✓ Generated {} embeddings of dimension {}\n", 3, emb1.len());
// Example 2: Compute semantic similarity
println!("--- Example 2: Semantic Similarity ---");
let sim_1_2 = cosine_similarity(&emb1, &emb2);
let sim_1_3 = cosine_similarity(&emb1, &emb3);
let sim_2_3 = cosine_similarity(&emb2, &emb3);
println!("Text 1: \"{}\"", text1);
println!("Text 2: \"{}\"", text2);
println!("Text 3: \"{}\"", text3);
println!();
println!("Similarity (1 ↔ 2): {:.4}", sim_1_2);
println!("Similarity (1 ↔ 3): {:.4}", sim_1_3);
println!("Similarity (2 ↔ 3): {:.4}", sim_2_3);
println!();
if sim_1_2 > sim_1_3 {
println!("✓ As expected, similar texts (1 & 2) have higher similarity!");
}
println!();
// Example 3: Batch processing
println!("--- Example 3: Batch Processing ---");
let documents = vec![
"Artificial intelligence and machine learning",
"Deep neural networks for computer vision",
"Natural language processing with transformers",
"The history of ancient Rome",
"Cooking recipes for Italian cuisine",
];
println!("Processing {} documents in batch...", documents.len());
let batch_embeddings = embedder.embed_batch(&documents)?;
println!("✓ Generated {} embeddings\n", batch_embeddings.len());
// Find most similar pair
println!("Finding most similar document pair...");
let mut max_sim = 0.0;
let mut max_pair = (0, 0);
for i in 0..batch_embeddings.len() {
for j in (i + 1)..batch_embeddings.len() {
let sim = cosine_similarity(&batch_embeddings[i], &batch_embeddings[j]);
if sim > max_sim {
max_sim = sim;
max_pair = (i, j);
}
}
}
println!("Most similar pair (similarity: {:.4}):", max_sim);
println!(" [{}] \"{}\"", max_pair.0, documents[max_pair.0]);
println!(" [{}] \"{}\"\n", max_pair.1, documents[max_pair.1]);
// Example 4: Search query use case
println!("--- Example 4: Search Query ---");
let query = "machine learning algorithms";
let query_emb = embedder.embed_text(query)?;
println!("Query: \"{}\"", query);
println!("\nRanked results:");
let mut scores: Vec<(usize, f32)> = batch_embeddings
.iter()
.enumerate()
.map(|(i, emb)| (i, cosine_similarity(&query_emb, emb)))
.collect();
scores.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
for (rank, (idx, score)) in scores.iter().take(3).enumerate() {
println!(" {}. [{:.4}] \"{}\"", rank + 1, score, documents[*idx]);
}
println!();
// Example 5: Model unloading (memory management)
println!("--- Example 5: Memory Management ---");
println!("Model loaded: {}", embedder.is_loaded());
embedder.unload()?;
println!("After unload: {}", embedder.is_loaded());
println!("✓ Model can be lazily reloaded on next use\n");
// Example 6: Using different models (commented out - requires model download)
println!("--- Example 6: Different Models ---");
println!("Available models:");
println!(" - bge-small-en-v1.5 (384d) - Default, fast");
println!(" - bge-base-en-v1.5 (768d) - Better quality");
println!(" - nomic-embed-text-v1.5 (768d) - Versatile");
println!(" - gte-large (1024d) - Highest quality");
println!();
println!("To use a different model:");
println!(" let config = TextEmbedConfig::bge_base();");
println!(" let embedder = LocalTextEmbedder::new(config)?;");
println!();
println!("=== Example Complete ===");
println!("\nKey takeaways:");
println!("✓ Local embeddings run entirely offline (no API calls)");
println!("✓ Models are lazy-loaded on first use");
println!("✓ Embeddings are L2-normalized for cosine similarity");
println!("✓ Batch processing is efficient for multiple texts");
println!("✓ Similar texts have higher cosine similarity scores");
Ok(())
}