chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,179 @@
|
||||
//! Basic usage example demonstrating create, put, find, and timeline operations.
|
||||
//!
|
||||
//! Run with: cargo run --example basic_usage
|
||||
|
||||
use std::path::PathBuf;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use memvid_core::{Memvid, PutOptions, Result, SearchRequest, TimelineQuery};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// Create a temporary directory for our example
|
||||
let dir = tempdir().expect("failed to create temp dir");
|
||||
let path: PathBuf = dir.path().join("example.mv2");
|
||||
|
||||
println!("=== Memvid Core Basic Usage Example ===\n");
|
||||
|
||||
// ========================================
|
||||
// 1. CREATE a new memory file
|
||||
// ========================================
|
||||
println!("1. Creating memory file at {:?}", path);
|
||||
let mut mem = Memvid::create(&path)?;
|
||||
println!(" Memory created successfully!\n");
|
||||
|
||||
// ========================================
|
||||
// 2. PUT documents into the memory
|
||||
// ========================================
|
||||
println!("2. Adding documents to memory...");
|
||||
|
||||
// Simple put with just bytes
|
||||
let seq1 = mem.put_bytes(b"Hello, Memvid! This is a simple text document.")?;
|
||||
println!(" Added document 1, sequence: {}", seq1);
|
||||
|
||||
// Put with options (title, URI, tags)
|
||||
let options = PutOptions::builder()
|
||||
.title("Getting Started Guide")
|
||||
.uri("mv2://docs/getting-started.md")
|
||||
.tag("category", "documentation")
|
||||
.tag("version", "2.0")
|
||||
.build();
|
||||
let seq2 = mem.put_bytes_with_options(
|
||||
b"This guide covers the basics of using Memvid for AI memory storage.",
|
||||
options,
|
||||
)?;
|
||||
println!(" Added document 2 (with metadata), sequence: {}", seq2);
|
||||
|
||||
// Add more documents
|
||||
let options = PutOptions::builder()
|
||||
.title("API Reference")
|
||||
.uri("mv2://docs/api-reference.md")
|
||||
.tag("category", "documentation")
|
||||
.build();
|
||||
mem.put_bytes_with_options(
|
||||
b"The Memvid API provides methods for create, put, find, and timeline operations.",
|
||||
options,
|
||||
)?;
|
||||
|
||||
let options = PutOptions::builder()
|
||||
.title("FAQ")
|
||||
.uri("mv2://docs/faq.md")
|
||||
.tag("category", "support")
|
||||
.build();
|
||||
mem.put_bytes_with_options(
|
||||
b"Frequently asked questions about Memvid memory files and search.",
|
||||
options,
|
||||
)?;
|
||||
|
||||
// Commit changes to persist them
|
||||
mem.commit()?;
|
||||
println!(" Committed all changes\n");
|
||||
|
||||
// ========================================
|
||||
// 3. STATS - Check memory statistics
|
||||
// ========================================
|
||||
println!("3. Memory statistics:");
|
||||
let stats = mem.stats()?;
|
||||
println!(" Frame count: {}", stats.frame_count);
|
||||
println!(" Has lexical index: {}", stats.has_lex_index);
|
||||
println!(" Has vector index: {}", stats.has_vec_index);
|
||||
println!(" Has time index: {}", stats.has_time_index);
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 4. FIND - Search for documents
|
||||
// ========================================
|
||||
println!("4. Searching for documents...");
|
||||
|
||||
// Search for "memvid"
|
||||
let request = SearchRequest {
|
||||
query: "memvid".to_string(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
};
|
||||
let response = mem.search(request)?;
|
||||
println!(" Query: 'memvid'");
|
||||
println!(" Total hits: {}", response.total_hits);
|
||||
println!(" Elapsed: {}ms", response.elapsed_ms);
|
||||
for hit in &response.hits {
|
||||
let title = hit.title.as_deref().unwrap_or("Untitled");
|
||||
let score = hit.score.unwrap_or(0.0);
|
||||
println!(" - [{}] {} (score: {:.3})", hit.frame_id, title, score);
|
||||
println!(
|
||||
" Snippet: {}...",
|
||||
&hit.text.chars().take(60).collect::<String>()
|
||||
);
|
||||
}
|
||||
println!();
|
||||
|
||||
// Search within a scope
|
||||
let request = SearchRequest {
|
||||
query: "documentation".to_string(),
|
||||
top_k: 10,
|
||||
snippet_chars: 100,
|
||||
uri: None,
|
||||
scope: Some("mv2://docs/".to_string()),
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
};
|
||||
let response = mem.search(request)?;
|
||||
println!(" Query: 'documentation' (scope: mv2://docs/)");
|
||||
println!(" Total hits: {}", response.total_hits);
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 5. TIMELINE - Browse documents chronologically
|
||||
// ========================================
|
||||
println!("5. Timeline (chronological view):");
|
||||
let timeline = mem.timeline(TimelineQuery::default())?;
|
||||
for entry in &timeline {
|
||||
let uri = entry.uri.as_deref().unwrap_or("(no uri)");
|
||||
println!(
|
||||
" [{}] {} - {}",
|
||||
entry.frame_id,
|
||||
uri,
|
||||
entry.preview.chars().take(40).collect::<String>()
|
||||
);
|
||||
}
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 6. REOPEN - Close and reopen the memory
|
||||
// ========================================
|
||||
println!("6. Closing and reopening memory...");
|
||||
drop(mem);
|
||||
|
||||
let reopened = Memvid::open(&path)?;
|
||||
let stats = reopened.stats()?;
|
||||
println!(" Reopened successfully!");
|
||||
println!(" Frame count after reopen: {}", stats.frame_count);
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 7. VERIFY - Check file integrity
|
||||
// ========================================
|
||||
println!("7. Verifying file integrity...");
|
||||
drop(reopened); // Close the memory before verifying
|
||||
let report = Memvid::verify(&path, false)?;
|
||||
println!(" Verification status: {:?}", report.overall_status);
|
||||
println!();
|
||||
|
||||
println!("=== Example completed successfully! ===");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
//! CLIP Visual Search Example
|
||||
//!
|
||||
//! Demonstrates using CLIP embeddings to search PDF pages and images
|
||||
//! using natural language queries.
|
||||
//!
|
||||
//! Run with:
|
||||
//! ```bash
|
||||
//! cargo run --example clip_visual_search --features clip,pdfium -- /path/to/pdf
|
||||
//! ```
|
||||
//!
|
||||
//! Prerequisites:
|
||||
//! 1. Download the MobileCLIP-S2 ONNX models:
|
||||
//! ```bash
|
||||
//! mkdir -p ~/.local/share/memvid/models
|
||||
//! curl -L 'https://huggingface.co/Xenova/mobileclip_s2/resolve/main/onnx/vision_model_int8.onnx' \
|
||||
//! -o ~/.local/share/memvid/models/mobileclip-s2_vision.onnx
|
||||
//! curl -L 'https://huggingface.co/Xenova/mobileclip_s2/resolve/main/onnx/text_model_int8.onnx' \
|
||||
//! -o ~/.local/share/memvid/models/mobileclip-s2_text.onnx
|
||||
//! ```
|
||||
//!
|
||||
//! 2. For PDF page rendering, install pdfium:
|
||||
//! - macOS: `brew install nicbarker/pdfium/pdfium-mac-arm64` or `pdfium-mac-x64`
|
||||
//! - Linux: Download from https://github.com/nicbarker/pdfium-builds/releases
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
#[cfg(not(feature = "clip"))]
|
||||
{
|
||||
eprintln!("This example requires the 'clip' feature.");
|
||||
eprintln!("Run with: cargo run --example clip_visual_search --features clip");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "clip")]
|
||||
{
|
||||
use memvid_core::clip::{ClipConfig, ClipIndex, ClipIndexBuilder, ClipModel};
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::tempdir;
|
||||
|
||||
println!("=== CLIP Visual Search Example ===\n");
|
||||
|
||||
// Get PDF path from args or use default
|
||||
let args: Vec<String> = env::args().collect();
|
||||
let pdf_path = if args.len() > 1 {
|
||||
PathBuf::from(&args[1])
|
||||
} else {
|
||||
// Default to the SP Global Impact Report if no path provided
|
||||
PathBuf::from(
|
||||
"/Users/olow/Desktop/memvid-org/brickfield/sp-global-impact-report-2024.pdf",
|
||||
)
|
||||
};
|
||||
|
||||
if !pdf_path.exists() {
|
||||
eprintln!("PDF not found: {}", pdf_path.display());
|
||||
eprintln!(
|
||||
"Usage: cargo run --example clip_visual_search --features clip,pdfium -- /path/to/pdf"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
println!("PDF: {}", pdf_path.display());
|
||||
|
||||
// Initialize CLIP model
|
||||
println!("\n1. Loading CLIP model...");
|
||||
let config = ClipConfig::default();
|
||||
println!(" Model: {}", config.model_name);
|
||||
println!(" Models dir: {}", config.models_dir.display());
|
||||
|
||||
let clip = match ClipModel::new(config) {
|
||||
Ok(model) => {
|
||||
println!(" Model initialized (lazy loading)");
|
||||
model
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" Failed to initialize CLIP: {}", e);
|
||||
eprintln!("\n Make sure to download the models first:");
|
||||
eprintln!(" mkdir -p ~/.local/share/memvid/models");
|
||||
eprintln!(
|
||||
" curl -L 'https://huggingface.co/Xenova/mobileclip_s2/resolve/main/onnx/vision_model_int8.onnx' \\"
|
||||
);
|
||||
eprintln!(" -o ~/.local/share/memvid/models/mobileclip-s2_vision.onnx");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
// For this demo, we'll create synthetic embeddings since PDF rendering
|
||||
// requires pdfium which may not be available
|
||||
println!("\n2. Building CLIP index with sample embeddings...");
|
||||
|
||||
let mut builder = ClipIndexBuilder::new();
|
||||
|
||||
// Simulate embeddings for 10 "pages" with different visual concepts
|
||||
// In real usage, you would:
|
||||
// 1. Render each PDF page to an image
|
||||
// 2. Pass the image to clip.encode_image(&image)
|
||||
// 3. Store the embedding with the page's frame_id
|
||||
|
||||
let sample_concepts: Vec<(&str, Vec<f32>)> = vec![
|
||||
// These would be real embeddings from actual images in production
|
||||
(
|
||||
"charts and graphs",
|
||||
random_embedding(clip.dims() as usize, 1),
|
||||
),
|
||||
(
|
||||
"sustainability report cover",
|
||||
random_embedding(clip.dims() as usize, 2),
|
||||
),
|
||||
(
|
||||
"ESG metrics table",
|
||||
random_embedding(clip.dims() as usize, 3),
|
||||
),
|
||||
(
|
||||
"environmental impact diagram",
|
||||
random_embedding(clip.dims() as usize, 4),
|
||||
),
|
||||
(
|
||||
"carbon emissions chart",
|
||||
random_embedding(clip.dims() as usize, 5),
|
||||
),
|
||||
(
|
||||
"renewable energy infographic",
|
||||
random_embedding(clip.dims() as usize, 6),
|
||||
),
|
||||
(
|
||||
"corporate governance structure",
|
||||
random_embedding(clip.dims() as usize, 7),
|
||||
),
|
||||
(
|
||||
"diversity statistics",
|
||||
random_embedding(clip.dims() as usize, 8),
|
||||
),
|
||||
(
|
||||
"supply chain map",
|
||||
random_embedding(clip.dims() as usize, 9),
|
||||
),
|
||||
(
|
||||
"financial highlights",
|
||||
random_embedding(clip.dims() as usize, 10),
|
||||
),
|
||||
];
|
||||
|
||||
for (i, (concept, embedding)) in sample_concepts.iter().enumerate() {
|
||||
builder.add_document(i as u64, Some(i as u32), embedding.clone());
|
||||
println!(" Added page {} ({})", i + 1, concept);
|
||||
}
|
||||
|
||||
let artifact = builder.finish()?;
|
||||
println!(
|
||||
"\n Index built: {} vectors, {} dimensions",
|
||||
artifact.vector_count, artifact.dimension
|
||||
);
|
||||
|
||||
// Decode the index for searching
|
||||
let index = ClipIndex::decode(&artifact.bytes)?;
|
||||
|
||||
// Demonstrate text-to-image search
|
||||
println!("\n3. Searching with natural language queries...\n");
|
||||
|
||||
// Try encoding a text query
|
||||
println!(" Encoding query: 'sustainability charts'");
|
||||
match clip.encode_text("sustainability charts") {
|
||||
Ok(query_embedding) => {
|
||||
println!(" Query embedding: {} dimensions", query_embedding.len());
|
||||
|
||||
let hits = index.search(&query_embedding, 3);
|
||||
println!("\n Top 3 matches:");
|
||||
for (rank, hit) in hits.iter().enumerate() {
|
||||
let concept = sample_concepts
|
||||
.get(hit.frame_id as usize)
|
||||
.map(|(c, _)| *c)
|
||||
.unwrap_or("unknown");
|
||||
println!(
|
||||
" {}. Page {} ({}) - distance: {:.4}",
|
||||
rank + 1,
|
||||
hit.frame_id + 1,
|
||||
concept,
|
||||
hit.distance
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" Failed to encode text (model not loaded): {}", e);
|
||||
eprintln!(" Make sure the text model ONNX file is downloaded.");
|
||||
}
|
||||
}
|
||||
|
||||
// Demo with Memvid integration
|
||||
println!("\n4. Creating Memvid memory with CLIP support...");
|
||||
|
||||
let dir = tempdir().expect("failed to create temp dir");
|
||||
let path = dir.path().join("clip_demo.mv2");
|
||||
|
||||
let mut mem = memvid_core::Memvid::create(&path)?;
|
||||
|
||||
// Enable CLIP index
|
||||
mem.enable_clip()?;
|
||||
println!(" CLIP index enabled");
|
||||
|
||||
// Add some sample documents
|
||||
let options = memvid_core::PutOptions::builder()
|
||||
.title("SP Global Impact Report 2024 - Page 1")
|
||||
.uri("mv2://reports/sp-global/page-1")
|
||||
.build();
|
||||
mem.put_bytes_with_options(
|
||||
b"This page contains sustainability charts and ESG metrics.",
|
||||
options,
|
||||
)?;
|
||||
|
||||
mem.commit()?;
|
||||
println!(" Added sample document");
|
||||
|
||||
let stats = mem.stats()?;
|
||||
println!("\n Memory stats:");
|
||||
println!(" - Frames: {}", stats.frame_count);
|
||||
println!(" - Has CLIP index: {}", stats.has_clip_index);
|
||||
|
||||
// Search CLIP index (would use pre-computed query embedding in production)
|
||||
// Since we haven't added actual CLIP embeddings to the memory,
|
||||
// the search would return empty results - this is just to show the API
|
||||
|
||||
println!("\n=== Example completed successfully! ===");
|
||||
println!("\nTo use CLIP in production:");
|
||||
println!("1. During ingestion: Generate CLIP embeddings for images/PDF pages");
|
||||
println!("2. Store embeddings in the CLIP index alongside text content");
|
||||
println!("3. At query time: Encode the text query with clip.encode_text()");
|
||||
println!("4. Search the CLIP index with mem.search_clip(&query_embedding, limit)");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a pseudo-random embedding for demo purposes
|
||||
/// In production, use clip.encode_image() on actual images
|
||||
#[cfg(feature = "clip")]
|
||||
fn random_embedding(dims: usize, seed: u64) -> Vec<f32> {
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
let mut embedding = Vec::with_capacity(dims);
|
||||
|
||||
for i in 0..dims {
|
||||
(seed, i).hash(&mut hasher);
|
||||
let hash = hasher.finish();
|
||||
// Generate pseudo-random float between -1 and 1
|
||||
let val = ((hash as f32) / (u64::MAX as f32)) * 2.0 - 1.0;
|
||||
embedding.push(val);
|
||||
hasher = DefaultHasher::new();
|
||||
}
|
||||
|
||||
// L2 normalize
|
||||
let norm: f32 = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
if norm > 1e-10 {
|
||||
for v in &mut embedding {
|
||||
*v /= norm;
|
||||
}
|
||||
}
|
||||
|
||||
embedding
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
//! Generate visual performance comparison report
|
||||
//!
|
||||
//! Run with: cargo run --example generate_performance_report --features lex
|
||||
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
println!("=== Search Precision Performance Report ===\n");
|
||||
|
||||
// Create test corpus
|
||||
println!("Setting up test corpus (1000 documents)...");
|
||||
let temp_file = "/tmp/perf_test.mv2";
|
||||
let _ = std::fs::remove_file(temp_file);
|
||||
|
||||
let mut mem = Memvid::create(temp_file)?;
|
||||
|
||||
// Add 1000 documents with controlled distribution
|
||||
for i in 0..1000 {
|
||||
let topic = match i % 5 {
|
||||
0 => ("machine learning", "neural networks"),
|
||||
1 => ("python programming", "software development"),
|
||||
2 => ("machine learning with python", "data science"),
|
||||
3 => ("rust systems programming", "memory safety"),
|
||||
_ => ("web development", "javascript frameworks"),
|
||||
};
|
||||
|
||||
let content = format!(
|
||||
"Document {} about {}. This article covers {} in depth.",
|
||||
i, topic.0, topic.1
|
||||
);
|
||||
|
||||
mem.put_bytes_with_options(
|
||||
content.as_bytes(),
|
||||
PutOptions::builder()
|
||||
.title(format!("Doc {} - {}", i, topic.0))
|
||||
.build(),
|
||||
)?;
|
||||
|
||||
if (i + 1) % 100 == 0 {
|
||||
mem.commit()?;
|
||||
}
|
||||
}
|
||||
mem.commit()?;
|
||||
println!("✓ Corpus ready\n");
|
||||
|
||||
// Test queries
|
||||
let test_queries = vec![
|
||||
("machine python", "Both terms"),
|
||||
("machine learning python", "Three terms"),
|
||||
("python programming development", "Three terms"),
|
||||
("rust memory safety", "Two terms"),
|
||||
];
|
||||
|
||||
println!("┌─────────────────────────────────────────────────────────────────────┐");
|
||||
println!("│ QUERY PERFORMANCE METRICS │");
|
||||
println!("├─────────────────────────────────────────────────────────────────────┤");
|
||||
|
||||
for (query, desc) in &test_queries {
|
||||
// Warm up
|
||||
for _ in 0..10 {
|
||||
let _ = mem.search(SearchRequest {
|
||||
query: query.to_string(),
|
||||
top_k: 100,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})?;
|
||||
}
|
||||
|
||||
// Measure
|
||||
let iterations = 100;
|
||||
let start = Instant::now();
|
||||
|
||||
let mut total_results = 0;
|
||||
let mut total_relevant = 0;
|
||||
|
||||
for _ in 0..iterations {
|
||||
let results = mem.search(SearchRequest {
|
||||
query: query.to_string(),
|
||||
top_k: 100,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})?;
|
||||
|
||||
let terms: Vec<&str> = query.split_whitespace().collect();
|
||||
let relevant = results
|
||||
.hits
|
||||
.iter()
|
||||
.filter(|hit| {
|
||||
let text = hit.text.to_lowercase();
|
||||
terms.iter().all(|term| text.contains(term))
|
||||
})
|
||||
.count();
|
||||
|
||||
total_results += results.hits.len();
|
||||
total_relevant += relevant;
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
let avg_latency_us = elapsed.as_micros() / iterations as u128; // FIX: Cast to u128
|
||||
let avg_results = total_results / iterations;
|
||||
let avg_relevant = total_relevant / iterations;
|
||||
let precision = if avg_results > 0 {
|
||||
(avg_relevant as f64 / avg_results as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
println!("│");
|
||||
println!("│ Query: \"{}\" ({})", query, desc);
|
||||
println!("│ Latency: {:.2}ms", avg_latency_us as f64 / 1000.0);
|
||||
println!("│ Results: {} docs", avg_results);
|
||||
println!("│ Relevant: {} docs", avg_relevant);
|
||||
println!("│ Precision: {:.1}%", precision);
|
||||
println!("│ Memory: ~{} KB", (avg_results * 3).max(1));
|
||||
}
|
||||
|
||||
println!("└─────────────────────────────────────────────────────────────────────┘");
|
||||
|
||||
// Comparison with hypothetical OR behavior
|
||||
println!("\n┌─────────────────────────────────────────────────────────────────────┐");
|
||||
println!("│ COMPARISON: AND vs OR (Estimated) │");
|
||||
println!("├─────────────────────────────────────────────────────────────────────┤");
|
||||
println!("│");
|
||||
println!("│ Query: \"machine python\"");
|
||||
println!("│");
|
||||
println!("│ WITH AND (Current): │ WITH OR (Previous):");
|
||||
println!("│ • Results: ~5-8 docs │ • Results: ~80-120 docs");
|
||||
println!("│ • Precision: 100% │ • Precision: ~6-8%");
|
||||
println!("│ • Memory: ~20 KB │ • Memory: ~300 KB");
|
||||
println!("│ • Processing: 6-10ms │ • Processing: 96-144ms");
|
||||
println!("│");
|
||||
println!("│ IMPROVEMENT: ");
|
||||
println!("│ ✓ 15x better precision (6% → 100%) ");
|
||||
println!("│ ✓ 93% less memory (300KB → 20KB) ");
|
||||
println!("│ ✓ 93% faster processing (120ms → 8ms) ");
|
||||
println!("│ ✓ No query latency regression (~1.2ms) ");
|
||||
println!("│");
|
||||
println!("└─────────────────────────────────────────────────────────────────────┘");
|
||||
|
||||
std::fs::remove_file(temp_file)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
//! Example demonstrating OpenAI API embedding usage.
|
||||
//!
|
||||
//! This example shows how to:
|
||||
//! - Create an OpenAI embedder with default configuration
|
||||
//! - Generate embeddings using the API
|
||||
//! - Compute cosine similarity between embeddings
|
||||
//! - Use different models (small, large, ada)
|
||||
//!
|
||||
//! ## Prerequisites
|
||||
//!
|
||||
//! Set your OpenAI API key:
|
||||
//! ```bash
|
||||
//! export OPENAI_API_KEY="sk-..."
|
||||
//! ```
|
||||
//!
|
||||
//! ## Run
|
||||
//!
|
||||
//! ```bash
|
||||
//! cargo run --example openai_embedding --features api_embed
|
||||
//! ```
|
||||
|
||||
use memvid_core::Result;
|
||||
|
||||
#[cfg(feature = "api_embed")]
|
||||
use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
|
||||
#[cfg(feature = "api_embed")]
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
/// Compute cosine similarity between two vectors
|
||||
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
assert_eq!(a.len(), b.len(), "Vectors must have same length");
|
||||
|
||||
let dot_product: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
|
||||
let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
|
||||
if norm_a > 0.0 && norm_b > 0.0 {
|
||||
dot_product / (norm_a * norm_b)
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "api_embed")]
|
||||
fn main() -> Result<()> {
|
||||
println!("=== OpenAI Embedding Example ===\n");
|
||||
|
||||
// Check if API key is set
|
||||
if std::env::var("OPENAI_API_KEY").is_err() {
|
||||
eprintln!("Error: OPENAI_API_KEY environment variable not set.");
|
||||
eprintln!("Please set it with: export OPENAI_API_KEY=\"sk-...\"");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// Create embedder with default config (text-embedding-3-small, 1536 dimensions)
|
||||
println!("Creating OpenAI embedder (text-embedding-3-small)...");
|
||||
let config = OpenAIConfig::default();
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
|
||||
println!("Model: {}", embedder.model());
|
||||
println!("Kind: {}", embedder.kind());
|
||||
println!("Dimensions: {}", embedder.dimension());
|
||||
println!("Ready: {}\n", embedder.is_ready());
|
||||
|
||||
// Example 1: Single text embedding
|
||||
println!("--- Example 1: Single Text Embedding ---");
|
||||
let text = "The quick brown fox jumps over the lazy dog";
|
||||
println!("Embedding text: \"{}\"", text);
|
||||
|
||||
let embedding = embedder.embed_text(text)?;
|
||||
println!("Generated embedding of dimension {}\n", embedding.len());
|
||||
|
||||
// Example 2: Semantic similarity
|
||||
println!("--- Example 2: Semantic Similarity ---");
|
||||
let text1 = "Machine learning and artificial intelligence";
|
||||
let text2 = "Deep neural networks for AI applications";
|
||||
let text3 = "The history of ancient Rome";
|
||||
|
||||
let emb1 = embedder.embed_text(text1)?;
|
||||
let emb2 = embedder.embed_text(text2)?;
|
||||
let emb3 = embedder.embed_text(text3)?;
|
||||
|
||||
println!("Text 1: \"{}\"", text1);
|
||||
println!("Text 2: \"{}\"", text2);
|
||||
println!("Text 3: \"{}\"", text3);
|
||||
println!();
|
||||
|
||||
let sim_1_2 = cosine_similarity(&emb1, &emb2);
|
||||
let sim_1_3 = cosine_similarity(&emb1, &emb3);
|
||||
let sim_2_3 = cosine_similarity(&emb2, &emb3);
|
||||
|
||||
println!("Similarity (1 ↔ 2): {:.4}", sim_1_2);
|
||||
println!("Similarity (1 ↔ 3): {:.4}", sim_1_3);
|
||||
println!("Similarity (2 ↔ 3): {:.4}", sim_2_3);
|
||||
|
||||
if sim_1_2 > sim_1_3 {
|
||||
println!("\n✓ Related texts (1 & 2) have higher similarity than unrelated (1 & 3)!");
|
||||
}
|
||||
println!();
|
||||
|
||||
// Example 3: Batch processing
|
||||
println!("--- Example 3: Batch Processing ---");
|
||||
let documents = vec![
|
||||
"Python programming language",
|
||||
"JavaScript web development",
|
||||
"Rust systems programming",
|
||||
"Italian cooking recipes",
|
||||
];
|
||||
|
||||
println!("Processing {} documents in batch...", documents.len());
|
||||
let batch_embeddings = embedder.embed_batch(&documents)?;
|
||||
println!(
|
||||
"✓ Generated {} embeddings of dimension {}\n",
|
||||
batch_embeddings.len(),
|
||||
batch_embeddings.first().map(|e| e.len()).unwrap_or(0)
|
||||
);
|
||||
|
||||
// Find most similar pair
|
||||
let mut max_sim = 0.0;
|
||||
let mut max_pair = (0, 0);
|
||||
|
||||
for i in 0..batch_embeddings.len() {
|
||||
for j in (i + 1)..batch_embeddings.len() {
|
||||
let sim = cosine_similarity(&batch_embeddings[i], &batch_embeddings[j]);
|
||||
if sim > max_sim {
|
||||
max_sim = sim;
|
||||
max_pair = (i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("Most similar pair (similarity: {:.4}):", max_sim);
|
||||
println!(" [{}] \"{}\"", max_pair.0, documents[max_pair.0]);
|
||||
println!(" [{}] \"{}\"\n", max_pair.1, documents[max_pair.1]);
|
||||
|
||||
// Example 4: Available models
|
||||
println!("--- Example 4: Available Models ---");
|
||||
println!("OpenAI embedding models:");
|
||||
println!(" - text-embedding-3-small (1536d) - Default, fastest, cheapest");
|
||||
println!(" - text-embedding-3-large (3072d) - Highest quality");
|
||||
println!(" - text-embedding-ada-002 (1536d) - Legacy model");
|
||||
println!();
|
||||
println!("To use a different model:");
|
||||
println!(" let config = OpenAIConfig::large();");
|
||||
println!(" let embedder = OpenAIEmbedder::new(config)?;");
|
||||
println!();
|
||||
|
||||
println!("=== Example Complete ===");
|
||||
println!("\nKey takeaways:");
|
||||
println!("✓ API embeddings require OPENAI_API_KEY environment variable");
|
||||
println!("✓ text-embedding-3-small is fast and cost-effective");
|
||||
println!("✓ Batch processing reduces API calls for multiple texts");
|
||||
println!("✓ Similar texts have higher cosine similarity scores");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "api_embed"))]
|
||||
fn main() {
|
||||
eprintln!("This example requires the 'api_embed' feature.");
|
||||
eprintln!("Run with: cargo run --example openai_embedding --features api_embed");
|
||||
std::process::exit(1);
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
//! PDF ingestion example demonstrating how to ingest and search PDF documents.
|
||||
//!
|
||||
//! This example demonstrates PDF text extraction, chunking, and semantic search.
|
||||
//!
|
||||
//! Run with:
|
||||
//! ```bash
|
||||
//! cargo run --example pdf_ingestion -- /path/to/pdf
|
||||
//! ```
|
||||
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use memvid_core::{Memvid, PutOptions, Result, SearchRequest};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// Get PDF path from args
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: cargo run --example pdf_ingestion -- /path/to/pdf");
|
||||
eprintln!("\nExample:");
|
||||
eprintln!(" cargo run --example pdf_ingestion -- examples/1706.03762v7.pdf");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let pdf_path = PathBuf::from(&args[1]);
|
||||
|
||||
if !pdf_path.exists() {
|
||||
eprintln!("ERROR: PDF file not found at {:?}", pdf_path);
|
||||
eprintln!("Usage: cargo run --example pdf_ingestion -- /path/to/pdf");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Create a temporary directory for our memory file
|
||||
let dir = tempdir().expect("failed to create temp dir");
|
||||
let mv2_path: PathBuf = dir.path().join("paper.mv2");
|
||||
|
||||
println!("=== Memvid PDF Ingestion Example ===\n");
|
||||
|
||||
// ========================================
|
||||
// 1. CREATE a new memory file
|
||||
// ========================================
|
||||
println!("1. Creating memory file...");
|
||||
let mut mem = Memvid::create(&mv2_path)?;
|
||||
println!(" Memory created at {:?}\n", mv2_path);
|
||||
|
||||
// ========================================
|
||||
// 2. INGEST the PDF file
|
||||
// ========================================
|
||||
println!("2. Ingesting PDF: {:?}", pdf_path);
|
||||
|
||||
// Read the PDF file
|
||||
let pdf_bytes = std::fs::read(&pdf_path)?;
|
||||
println!(" PDF size: {} bytes", pdf_bytes.len());
|
||||
|
||||
// Put the PDF with metadata
|
||||
// Extract filename for title
|
||||
let title = pdf_path
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("PDF Document")
|
||||
.to_string();
|
||||
|
||||
let options = PutOptions::builder()
|
||||
.title(&title)
|
||||
.uri(format!(
|
||||
"mv2://pdfs/{}",
|
||||
pdf_path.file_name().unwrap_or_default().to_string_lossy()
|
||||
))
|
||||
.build();
|
||||
|
||||
let frame_id = mem.put_bytes_with_options(&pdf_bytes, options)?;
|
||||
println!(" Ingested as frame: {}", frame_id);
|
||||
|
||||
// Commit changes
|
||||
mem.commit()?;
|
||||
println!(" Committed successfully!\n");
|
||||
|
||||
// ========================================
|
||||
// 3. CHECK memory statistics
|
||||
// ========================================
|
||||
println!("3. Memory statistics:");
|
||||
let stats = mem.stats()?;
|
||||
println!(" Frame count: {}", stats.frame_count);
|
||||
println!(" Has lexical index: {}", stats.has_lex_index);
|
||||
println!();
|
||||
|
||||
// ========================================
|
||||
// 4. SEARCH the ingested PDF
|
||||
// ========================================
|
||||
println!("4. Searching the paper...\n");
|
||||
|
||||
// Search for "attention"
|
||||
let queries = [
|
||||
"attention mechanism",
|
||||
"transformer architecture",
|
||||
"self-attention",
|
||||
"encoder decoder",
|
||||
"positional encoding",
|
||||
];
|
||||
|
||||
for query in queries {
|
||||
let request = SearchRequest {
|
||||
query: query.to_string(),
|
||||
top_k: 3,
|
||||
snippet_chars: 150,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
};
|
||||
|
||||
let response = mem.search(request)?;
|
||||
println!(" Query: '{}'", query);
|
||||
println!(
|
||||
" Hits: {} ({}ms)",
|
||||
response.total_hits, response.elapsed_ms
|
||||
);
|
||||
|
||||
for (i, hit) in response.hits.iter().take(2).enumerate() {
|
||||
let snippet = hit
|
||||
.text
|
||||
.chars()
|
||||
.take(100)
|
||||
.collect::<String>()
|
||||
.replace('\n', " ");
|
||||
println!(" {}. {}...", i + 1, snippet);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
// ========================================
|
||||
// 5. VERIFY file integrity
|
||||
// ========================================
|
||||
println!("5. Verifying file integrity...");
|
||||
drop(mem);
|
||||
let report = Memvid::verify(&mv2_path, false)?;
|
||||
println!(" Status: {:?}", report.overall_status);
|
||||
println!();
|
||||
|
||||
println!("=== PDF ingestion example completed! ===");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
//! Benchmark comparing SIMD vs Scalar L2 distance calculations.
|
||||
//!
|
||||
//! Run with: `cargo run --example simd_benchmark --features simd --release`
|
||||
|
||||
use std::hint::black_box;
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() {
|
||||
let num_vectors = 10_000;
|
||||
let dims = 384; // Standard embedding dimension
|
||||
let num_iterations = 100;
|
||||
|
||||
println!("SIMD vs Scalar L2 Distance Benchmark");
|
||||
println!("=====================================");
|
||||
println!("Vectors: {}", num_vectors);
|
||||
println!("Dimensions: {}", dims);
|
||||
println!("Iterations: {}", num_iterations);
|
||||
println!();
|
||||
|
||||
// Generate random vectors
|
||||
let query: Vec<f32> = (0..dims).map(|i| (i as f32 * 0.001) % 1.0).collect();
|
||||
let vectors: Vec<Vec<f32>> = (0..num_vectors)
|
||||
.map(|v| (0..dims).map(|i| ((v + i) as f32 * 0.0017) % 1.0).collect())
|
||||
.collect();
|
||||
|
||||
// Benchmark SIMD version
|
||||
let simd_start = Instant::now();
|
||||
let mut simd_sum = 0.0f32;
|
||||
for _ in 0..num_iterations {
|
||||
for vec in &vectors {
|
||||
simd_sum += black_box(l2_distance_simd(black_box(&query), black_box(vec)));
|
||||
}
|
||||
}
|
||||
let simd_elapsed = simd_start.elapsed();
|
||||
black_box(simd_sum);
|
||||
|
||||
// Benchmark Scalar version
|
||||
let scalar_start = Instant::now();
|
||||
let mut scalar_sum = 0.0f32;
|
||||
for _ in 0..num_iterations {
|
||||
for vec in &vectors {
|
||||
scalar_sum += black_box(l2_distance_scalar(black_box(&query), black_box(vec)));
|
||||
}
|
||||
}
|
||||
let scalar_elapsed = scalar_start.elapsed();
|
||||
black_box(scalar_sum);
|
||||
|
||||
// Results
|
||||
let total_ops = num_vectors * num_iterations;
|
||||
let simd_per_op_ns = simd_elapsed.as_nanos() as f64 / total_ops as f64;
|
||||
let scalar_per_op_ns = scalar_elapsed.as_nanos() as f64 / total_ops as f64;
|
||||
let speedup = scalar_elapsed.as_nanos() as f64 / simd_elapsed.as_nanos() as f64;
|
||||
|
||||
println!("Results:");
|
||||
println!("--------");
|
||||
println!(
|
||||
"SIMD: {:>8.2}ms total, {:>6.1}ns per distance",
|
||||
simd_elapsed.as_secs_f64() * 1000.0,
|
||||
simd_per_op_ns
|
||||
);
|
||||
println!(
|
||||
"Scalar: {:>8.2}ms total, {:>6.1}ns per distance",
|
||||
scalar_elapsed.as_secs_f64() * 1000.0,
|
||||
scalar_per_op_ns
|
||||
);
|
||||
println!();
|
||||
println!("Speedup: {:.2}x", speedup);
|
||||
|
||||
// Verify correctness
|
||||
let simd_result = l2_distance_simd(&query, &vectors[0]);
|
||||
let scalar_result = l2_distance_scalar(&query, &vectors[0]);
|
||||
let diff = (simd_result - scalar_result).abs();
|
||||
println!();
|
||||
println!("Correctness check:");
|
||||
println!(" SIMD result: {:.8}", simd_result);
|
||||
println!(" Scalar result: {:.8}", scalar_result);
|
||||
println!(" Difference: {:.2e} (should be < 1e-5)", diff);
|
||||
assert!(diff < 1e-4, "Results differ too much!");
|
||||
println!(" ✓ Results match!");
|
||||
}
|
||||
|
||||
/// SIMD L2 distance using the wide crate
|
||||
#[cfg(feature = "simd")]
|
||||
fn l2_distance_simd(a: &[f32], b: &[f32]) -> f32 {
|
||||
memvid_core::simd::l2_distance_simd(a, b)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "simd"))]
|
||||
fn l2_distance_simd(a: &[f32], b: &[f32]) -> f32 {
|
||||
l2_distance_scalar(a, b)
|
||||
}
|
||||
|
||||
/// Scalar L2 distance (the OLD implementation)
|
||||
#[inline(never)]
|
||||
fn l2_distance_scalar(a: &[f32], b: &[f32]) -> f32 {
|
||||
a.iter()
|
||||
.zip(b.iter())
|
||||
.map(|(x, y)| (x - y).powi(2))
|
||||
.sum::<f32>()
|
||||
.sqrt()
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
//! Test to demonstrate the implicit OR bug in query parsing
|
||||
//!
|
||||
//! Run with: cargo run --example test_implicit_or_bug --features lex
|
||||
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
let test_file = "/tmp/test_implicit.mv2";
|
||||
|
||||
// Clean up old file if exists
|
||||
let _ = std::fs::remove_file(test_file);
|
||||
|
||||
println!("Creating test .mv2 file...");
|
||||
let mut mem = Memvid::create(test_file)?;
|
||||
|
||||
// Add documents
|
||||
println!("Adding test documents...");
|
||||
mem.put_bytes_with_options(
|
||||
b"I love machine learning",
|
||||
PutOptions::builder().title("Doc 1").build(),
|
||||
)?;
|
||||
mem.put_bytes_with_options(
|
||||
b"I love Python programming",
|
||||
PutOptions::builder().title("Doc 2").build(),
|
||||
)?;
|
||||
mem.put_bytes_with_options(
|
||||
b"Machine learning with Python is awesome",
|
||||
PutOptions::builder().title("Doc 3").build(),
|
||||
)?;
|
||||
mem.commit()?;
|
||||
|
||||
println!();
|
||||
println!("=== TESTING IMPLICIT OPERATOR BEHAVIOR ===");
|
||||
println!("Query: 'machine python'");
|
||||
println!();
|
||||
println!("Expected behavior (AND): Should return 1 result");
|
||||
println!(" - Only Doc 3 has BOTH 'machine' AND 'python'");
|
||||
println!();
|
||||
println!("Current behavior (OR): Returns 3 results");
|
||||
println!(" - Any doc with 'machine' OR 'python'");
|
||||
println!();
|
||||
|
||||
// Search with implicit operator
|
||||
let results = mem.search(SearchRequest {
|
||||
query: "machine python".to_string(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})?;
|
||||
|
||||
println!("ACTUAL RESULTS: {} documents found", results.hits.len());
|
||||
println!();
|
||||
|
||||
for (i, hit) in results.hits.iter().enumerate() {
|
||||
let title = hit
|
||||
.title
|
||||
.as_ref()
|
||||
.unwrap_or(&"Untitled".to_string())
|
||||
.clone();
|
||||
let text_lower = hit.text.to_lowercase();
|
||||
let has_machine = text_lower.contains("machine");
|
||||
let has_python = text_lower.contains("python");
|
||||
let is_relevant = has_machine && has_python;
|
||||
|
||||
let status = if is_relevant {
|
||||
"✓ RELEVANT"
|
||||
} else {
|
||||
"✗ NOT RELEVANT"
|
||||
};
|
||||
|
||||
println!(
|
||||
" {}. {} [machine={}, python={}] {}",
|
||||
i + 1,
|
||||
title,
|
||||
has_machine,
|
||||
has_python,
|
||||
status
|
||||
);
|
||||
}
|
||||
|
||||
println!();
|
||||
|
||||
// Calculate precision
|
||||
let relevant_count = results
|
||||
.hits
|
||||
.iter()
|
||||
.filter(|hit| {
|
||||
let text = hit.text.to_lowercase();
|
||||
text.contains("machine") && text.contains("python")
|
||||
})
|
||||
.count();
|
||||
|
||||
let precision = if results.hits.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
(relevant_count as f64 / results.hits.len() as f64) * 100.0
|
||||
};
|
||||
|
||||
println!(
|
||||
"PRECISION: {:.1}% ({}/{} results are relevant)",
|
||||
precision,
|
||||
relevant_count,
|
||||
results.hits.len()
|
||||
);
|
||||
println!();
|
||||
|
||||
// Verdict
|
||||
if results.hits.len() == 1 && relevant_count == 1 {
|
||||
println!("✓ CORRECT: Query uses implicit AND");
|
||||
println!(" Perfect precision - returns only relevant docs");
|
||||
} else if results.hits.len() > 1 {
|
||||
println!("✗ BUG DETECTED: Query uses implicit OR");
|
||||
println!(" Low precision - returns docs with EITHER term");
|
||||
println!();
|
||||
} else {
|
||||
println!("? NO RESULTS: Check if lex feature is enabled");
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
std::fs::remove_file(test_file)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
//! Whisper transcription example demonstrating audio transcription.
|
||||
//!
|
||||
//! Run with:
|
||||
//! ```bash
|
||||
//! cargo run --example test_whisper --features whisper -- /path/to/audio
|
||||
//! ```
|
||||
|
||||
#[cfg(feature = "whisper")]
|
||||
use memvid_core::{WhisperConfig, WhisperTranscriber};
|
||||
#[cfg(feature = "whisper")]
|
||||
use std::env;
|
||||
#[cfg(feature = "whisper")]
|
||||
use std::path::PathBuf;
|
||||
#[cfg(feature = "whisper")]
|
||||
use std::time::Instant;
|
||||
|
||||
#[cfg(not(feature = "whisper"))]
|
||||
fn main() {
|
||||
eprintln!(
|
||||
"This example requires the `whisper` feature.\n\
|
||||
Re-run with:\n\
|
||||
cargo run --example test_whisper --features whisper -- /path/to/audio"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "whisper")]
|
||||
fn main() {
|
||||
// Get audio path from args
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: cargo run --example test_whisper --features whisper -- /path/to/audio");
|
||||
eprintln!("\nExample:");
|
||||
eprintln!(
|
||||
" cargo run --example test_whisper --features whisper -- examples/call_sale.mp3"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let audio_path = PathBuf::from(&args[1]);
|
||||
|
||||
if !audio_path.exists() {
|
||||
eprintln!("ERROR: Audio file not found at {:?}", audio_path);
|
||||
eprintln!("Usage: cargo run --example test_whisper --features whisper -- /path/to/audio");
|
||||
return;
|
||||
}
|
||||
|
||||
println!("=== Whisper Transcription Example ===\n");
|
||||
println!("Creating Whisper transcriber...");
|
||||
let start = Instant::now();
|
||||
let config = WhisperConfig::default();
|
||||
println!("Model dir: {:?}", config.models_dir);
|
||||
println!("Model name: {}", config.model_name);
|
||||
|
||||
let mut transcriber = WhisperTranscriber::new(&config).expect("Failed to create transcriber");
|
||||
println!("Transcriber created in {:?}", start.elapsed());
|
||||
|
||||
println!("\nTranscribing audio file: {}", audio_path.display());
|
||||
let start = Instant::now();
|
||||
|
||||
match transcriber.transcribe_file(&audio_path) {
|
||||
Ok(result) => {
|
||||
println!("Transcription completed in {:?}", start.elapsed());
|
||||
println!("\n=== Transcription Result ===");
|
||||
println!("Duration: {:.2} seconds", result.duration_secs);
|
||||
println!("Language: {}", result.language);
|
||||
println!("\nText:\n{}", result.text);
|
||||
|
||||
if !result.segments.is_empty() {
|
||||
println!("\n=== Segments ===");
|
||||
for seg in &result.segments {
|
||||
println!("[{:.2}s - {:.2}s] {}", seg.start, seg.end, seg.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Transcription failed: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n=== Transcription example completed! ===");
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
//! Benchmark demonstrating the performance benefit of embedding caching.
|
||||
//!
|
||||
//! This example compares performance with and without caching for repeated texts.
|
||||
//!
|
||||
//! ## Prerequisites
|
||||
//!
|
||||
//! Download the BGE-small model:
|
||||
//!
|
||||
//! ```bash
|
||||
//! mkdir -p ~/.cache/memvid/text-models
|
||||
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
//! ```
|
||||
//!
|
||||
//! ## Run
|
||||
//!
|
||||
//! ```bash
|
||||
//! cargo run --example text_embed_cache_bench --features vec --release
|
||||
//! ```
|
||||
|
||||
use memvid_core::Result;
|
||||
#[cfg(feature = "vec")]
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
println!("╔════════════════════════════════════════════════════════════╗");
|
||||
println!("║ Embedding Cache Benchmark ║");
|
||||
println!("╚════════════════════════════════════════════════════════════╝\n");
|
||||
|
||||
// Test texts with intentional repeats
|
||||
let test_texts = vec![
|
||||
"machine learning",
|
||||
"artificial intelligence",
|
||||
"deep learning",
|
||||
"neural networks",
|
||||
"natural language processing",
|
||||
"machine learning", // Repeat 1
|
||||
"artificial intelligence", // Repeat 2
|
||||
"computer vision",
|
||||
"deep learning", // Repeat 3
|
||||
"machine learning", // Repeat 4
|
||||
];
|
||||
|
||||
println!(
|
||||
"Test data: {} texts ({} unique, {} repeats)\n",
|
||||
test_texts.len(),
|
||||
7, // unique
|
||||
3
|
||||
); // repeats
|
||||
|
||||
// ---------- Test with Cache Enabled ----------
|
||||
println!("──────────────────────────────────────────────────────────");
|
||||
println!("Test 1: WITH Cache (enabled by default)");
|
||||
println!("──────────────────────────────────────────────────────────");
|
||||
|
||||
let config_cached = TextEmbedConfig::default();
|
||||
let embedder_cached = LocalTextEmbedder::new(config_cached)?;
|
||||
|
||||
println!("Processing texts...");
|
||||
let start = Instant::now();
|
||||
for (i, text) in test_texts.iter().enumerate() {
|
||||
let _ = embedder_cached.encode_text(text)?;
|
||||
if i < test_texts.len() - 1 {
|
||||
print!(".");
|
||||
}
|
||||
}
|
||||
println!("!");
|
||||
let cached_time = start.elapsed();
|
||||
|
||||
if let Some(stats) = embedder_cached.cache_stats() {
|
||||
println!("\n📊 Cache Statistics:");
|
||||
println!(" Hits: {}", stats.hits);
|
||||
println!(" Misses: {}", stats.misses);
|
||||
println!(" Size: {}/{}", stats.size, stats.capacity);
|
||||
println!(" Hit Rate: {:.1}%", stats.hit_rate() * 100.0);
|
||||
}
|
||||
|
||||
println!("\n⏱️ Total Time: {:?}", cached_time);
|
||||
|
||||
// ---------- Test with Cache Disabled ----------
|
||||
println!("\n──────────────────────────────────────────────────────────");
|
||||
println!("Test 2: WITHOUT Cache (force disabled)");
|
||||
println!("──────────────────────────────────────────────────────────");
|
||||
|
||||
let config_uncached = TextEmbedConfig {
|
||||
enable_cache: false,
|
||||
..Default::default()
|
||||
};
|
||||
let embedder_uncached = LocalTextEmbedder::new(config_uncached)?;
|
||||
|
||||
println!("Processing texts...");
|
||||
let start = Instant::now();
|
||||
for (i, text) in test_texts.iter().enumerate() {
|
||||
let _ = embedder_uncached.encode_text(text)?;
|
||||
if i < test_texts.len() - 1 {
|
||||
print!(".");
|
||||
}
|
||||
}
|
||||
println!("!");
|
||||
let uncached_time = start.elapsed();
|
||||
|
||||
println!("\n⏱️ Total Time: {:?}", uncached_time);
|
||||
|
||||
// ---------- Results ----------
|
||||
println!("\n╔════════════════════════════════════════════════════════════╗");
|
||||
println!("║ Results ║");
|
||||
println!("╚════════════════════════════════════════════════════════════╝\n");
|
||||
|
||||
let speedup = uncached_time.as_secs_f64() / cached_time.as_secs_f64();
|
||||
|
||||
println!("┌────────────────┬──────────────┬──────────────┐");
|
||||
println!("│ Configuration │ Time │ Speedup │");
|
||||
println!("├────────────────┼──────────────┼──────────────┤");
|
||||
println!(
|
||||
"│ With Cache │ {:>10.3}s │ {:>9.2}x │",
|
||||
cached_time.as_secs_f64(),
|
||||
speedup
|
||||
);
|
||||
println!(
|
||||
"│ Without Cache │ {:>10.3}s │ baseline │",
|
||||
uncached_time.as_secs_f64()
|
||||
);
|
||||
println!("└────────────────┴──────────────┴──────────────┘\n");
|
||||
|
||||
if speedup > 1.0 {
|
||||
println!("✅ Cache provides {:.1}% speedup!", (speedup - 1.0) * 100.0);
|
||||
} else {
|
||||
println!("⚠️ No speedup observed (may indicate all cache misses)");
|
||||
}
|
||||
|
||||
println!("\n💡 Note: Speedup increases with more repeated texts.");
|
||||
println!(" Real-world applications with 50-90% repeated queries");
|
||||
println!(" can see 2-10x improvements!\n");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
//! Example demonstrating local text embedding usage.
|
||||
//!
|
||||
//! This example shows how to:
|
||||
//! - Create a local text embedder with default configuration (BGE-small)
|
||||
//! - Generate embeddings for sample texts
|
||||
//! - Compute cosine similarity between embeddings
|
||||
//! - Batch process multiple texts
|
||||
//! - Use different models (BGE-base, Nomic, GTE-large)
|
||||
//!
|
||||
//! ## Prerequisites
|
||||
//!
|
||||
//! Before running this example, download the BGE-small model:
|
||||
//!
|
||||
//! ```bash
|
||||
//! mkdir -p ~/.cache/memvid/text-models
|
||||
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
//! curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
//! -o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
//! ```
|
||||
//!
|
||||
//! ## Run
|
||||
//!
|
||||
//! ```bash
|
||||
//! cargo run --example text_embedding --features vec
|
||||
//! ```
|
||||
|
||||
use memvid_core::Result;
|
||||
#[cfg(feature = "vec")]
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
#[cfg(feature = "vec")]
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
/// Compute cosine similarity between two vectors
|
||||
fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
assert_eq!(a.len(), b.len(), "Vectors must have same length");
|
||||
|
||||
let dot_product: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
|
||||
let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
|
||||
if norm_a > 0.0 && norm_b > 0.0 {
|
||||
dot_product / (norm_a * norm_b)
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
println!("=== Local Text Embedding Example ===\n");
|
||||
|
||||
// Create embedder with default config (BGE-small, 384 dimensions)
|
||||
println!("Creating local text embedder (BGE-small-en-v1.5)...");
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
println!("Model: {}", embedder.model());
|
||||
println!("Kind: {}", embedder.kind());
|
||||
println!("Dimensions: {}", embedder.dimension());
|
||||
println!("Ready: {}\n", embedder.is_ready());
|
||||
|
||||
// Example 1: Generate embeddings for sample texts
|
||||
println!("--- Example 1: Single Text Embeddings ---");
|
||||
let text1 = "The quick brown fox jumps over the lazy dog";
|
||||
let text2 = "A fast auburn canine leaps above an idle hound";
|
||||
let text3 = "Python is a programming language";
|
||||
|
||||
println!("Generating embeddings...");
|
||||
let emb1 = embedder.embed_text(text1)?;
|
||||
let emb2 = embedder.embed_text(text2)?;
|
||||
let emb3 = embedder.embed_text(text3)?;
|
||||
|
||||
println!("✓ Generated {} embeddings of dimension {}\n", 3, emb1.len());
|
||||
|
||||
// Example 2: Compute semantic similarity
|
||||
println!("--- Example 2: Semantic Similarity ---");
|
||||
let sim_1_2 = cosine_similarity(&emb1, &emb2);
|
||||
let sim_1_3 = cosine_similarity(&emb1, &emb3);
|
||||
let sim_2_3 = cosine_similarity(&emb2, &emb3);
|
||||
|
||||
println!("Text 1: \"{}\"", text1);
|
||||
println!("Text 2: \"{}\"", text2);
|
||||
println!("Text 3: \"{}\"", text3);
|
||||
println!();
|
||||
println!("Similarity (1 ↔ 2): {:.4}", sim_1_2);
|
||||
println!("Similarity (1 ↔ 3): {:.4}", sim_1_3);
|
||||
println!("Similarity (2 ↔ 3): {:.4}", sim_2_3);
|
||||
println!();
|
||||
|
||||
if sim_1_2 > sim_1_3 {
|
||||
println!("✓ As expected, similar texts (1 & 2) have higher similarity!");
|
||||
}
|
||||
println!();
|
||||
|
||||
// Example 3: Batch processing
|
||||
println!("--- Example 3: Batch Processing ---");
|
||||
let documents = vec![
|
||||
"Artificial intelligence and machine learning",
|
||||
"Deep neural networks for computer vision",
|
||||
"Natural language processing with transformers",
|
||||
"The history of ancient Rome",
|
||||
"Cooking recipes for Italian cuisine",
|
||||
];
|
||||
|
||||
println!("Processing {} documents in batch...", documents.len());
|
||||
let batch_embeddings = embedder.embed_batch(&documents)?;
|
||||
println!("✓ Generated {} embeddings\n", batch_embeddings.len());
|
||||
|
||||
// Find most similar pair
|
||||
println!("Finding most similar document pair...");
|
||||
let mut max_sim = 0.0;
|
||||
let mut max_pair = (0, 0);
|
||||
|
||||
for i in 0..batch_embeddings.len() {
|
||||
for j in (i + 1)..batch_embeddings.len() {
|
||||
let sim = cosine_similarity(&batch_embeddings[i], &batch_embeddings[j]);
|
||||
if sim > max_sim {
|
||||
max_sim = sim;
|
||||
max_pair = (i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("Most similar pair (similarity: {:.4}):", max_sim);
|
||||
println!(" [{}] \"{}\"", max_pair.0, documents[max_pair.0]);
|
||||
println!(" [{}] \"{}\"\n", max_pair.1, documents[max_pair.1]);
|
||||
|
||||
// Example 4: Search query use case
|
||||
println!("--- Example 4: Search Query ---");
|
||||
let query = "machine learning algorithms";
|
||||
let query_emb = embedder.embed_text(query)?;
|
||||
|
||||
println!("Query: \"{}\"", query);
|
||||
println!("\nRanked results:");
|
||||
|
||||
let mut scores: Vec<(usize, f32)> = batch_embeddings
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, emb)| (i, cosine_similarity(&query_emb, emb)))
|
||||
.collect();
|
||||
|
||||
scores.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
|
||||
|
||||
for (rank, (idx, score)) in scores.iter().take(3).enumerate() {
|
||||
println!(" {}. [{:.4}] \"{}\"", rank + 1, score, documents[*idx]);
|
||||
}
|
||||
println!();
|
||||
|
||||
// Example 5: Model unloading (memory management)
|
||||
println!("--- Example 5: Memory Management ---");
|
||||
println!("Model loaded: {}", embedder.is_loaded());
|
||||
embedder.unload()?;
|
||||
println!("After unload: {}", embedder.is_loaded());
|
||||
println!("✓ Model can be lazily reloaded on next use\n");
|
||||
|
||||
// Example 6: Using different models (commented out - requires model download)
|
||||
println!("--- Example 6: Different Models ---");
|
||||
println!("Available models:");
|
||||
println!(" - bge-small-en-v1.5 (384d) - Default, fast");
|
||||
println!(" - bge-base-en-v1.5 (768d) - Better quality");
|
||||
println!(" - nomic-embed-text-v1.5 (768d) - Versatile");
|
||||
println!(" - gte-large (1024d) - Highest quality");
|
||||
println!();
|
||||
println!("To use a different model:");
|
||||
println!(" let config = TextEmbedConfig::bge_base();");
|
||||
println!(" let embedder = LocalTextEmbedder::new(config)?;");
|
||||
println!();
|
||||
|
||||
println!("=== Example Complete ===");
|
||||
println!("\nKey takeaways:");
|
||||
println!("✓ Local embeddings run entirely offline (no API calls)");
|
||||
println!("✓ Models are lazy-loaded on first use");
|
||||
println!("✓ Embeddings are L2-normalized for cosine similarity");
|
||||
println!("✓ Batch processing is efficient for multiple texts");
|
||||
println!("✓ Similar texts have higher cosine similarity scores");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user