chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,125 @@
|
||||
# Contributing Translations (i18n)
|
||||
|
||||
Thank you for helping translate Memvid’s documentation and make the project accessible to a global audience.
|
||||
|
||||
This guide explains how to contribute translations for the main `README.md`.
|
||||
|
||||
---
|
||||
|
||||
## What Can Be Translated
|
||||
|
||||
- The main repository `README.md`
|
||||
- Translations are stored in `docs/i18n/`
|
||||
- Each language has a single translation file
|
||||
|
||||
---
|
||||
|
||||
## Translation Workflow
|
||||
|
||||
### 1. Check Existing Issues
|
||||
|
||||
Before starting:
|
||||
- Check open issues to see if your language is already in progress
|
||||
- If an issue exists, comment on it to indicate you want to work on it
|
||||
- Only one contributor should work on a language at a time
|
||||
|
||||
---
|
||||
|
||||
### 2. Create a Translation Issue (If Needed)
|
||||
|
||||
If no issue exists for your language, open a new one using this format:
|
||||
|
||||
**Title:**
|
||||
```
|
||||
README translation: <Language> (<code>)
|
||||
```
|
||||
|
||||
**Labels to apply:**
|
||||
- `i18n`
|
||||
- `documentation`
|
||||
- `help wanted` or `good first issue`
|
||||
|
||||
---
|
||||
|
||||
### 3. Create the Translation File
|
||||
|
||||
1. Copy the main `README.md` from the repository root
|
||||
2. Create a new file in `docs/i18n/` using this format:
|
||||
|
||||
```
|
||||
README.<language-code>.md
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
- `README.es.md`
|
||||
- `README.zh-CN.md`
|
||||
- `README.pt-BR.md`
|
||||
|
||||
Use standard ISO language codes (include region where applicable).
|
||||
|
||||
---
|
||||
|
||||
## Translation Guidelines
|
||||
|
||||
- **Preserve structure**
|
||||
Keep headings, section order, and formatting identical to the original README
|
||||
|
||||
- **Preserve links and badges**
|
||||
Do not modify URLs, badges, or shields
|
||||
|
||||
- **Preserve code blocks**
|
||||
Keep code examples, commands, flags, and API names in English
|
||||
|
||||
- **Maintain technical accuracy**
|
||||
Translate naturally, but do not change meaning or behavior
|
||||
|
||||
- **Language quality matters**
|
||||
Native or fluent speakers are strongly preferred
|
||||
|
||||
---
|
||||
|
||||
## Submitting Your Translation
|
||||
|
||||
1. **Create a new branch:**
|
||||
```bash
|
||||
git checkout -b docs/i18n-<language-code>
|
||||
```
|
||||
|
||||
2. **Add your translation file** under `docs/i18n/`
|
||||
|
||||
3. **Commit your changes:**
|
||||
```bash
|
||||
git commit -m "docs(i18n): add <Language> README translation"
|
||||
```
|
||||
|
||||
4. **Push to your fork** and open a Pull Request
|
||||
|
||||
5. **Reference the translation issue** in your PR description
|
||||
|
||||
---
|
||||
|
||||
## Review Process
|
||||
|
||||
- Maintainers may request:
|
||||
- Clarifications
|
||||
- Formatting fixes
|
||||
- Review by another native speaker
|
||||
- Once approved, the PR will be merged
|
||||
- The corresponding issue will be closed
|
||||
|
||||
---
|
||||
|
||||
## Keeping Translations Up to Date
|
||||
|
||||
When the main `README.md` changes significantly:
|
||||
|
||||
- Existing translations may need updates
|
||||
- Contributors are encouraged to help keep translations in sync
|
||||
|
||||
---
|
||||
|
||||
## Getting Help
|
||||
|
||||
- Open an issue if you have questions about translations
|
||||
- Use GitHub Discussions for coordination
|
||||
- Contact: [contact@memvid.com](mailto:contact@memvid.com)
|
||||
@@ -0,0 +1,160 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
# المساهمة في Memvid (الترجمة العربية)
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid هي طبقة ذاكرة مكونة من ملف واحد لوكلاء الذكاء الاصطناعي (AI Agents)، توفر استرجاعاً فورياً وذاكرة طويلة المدى.</strong>
|
||||
|
||||
ذاكرة دائمة، مؤرشفة، وقابلة للنقل، دون الحاجة إلى قواعد بيانات.
|
||||
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ اترك نجمة (STAR) لدعم المشروع ⭐️</h2>
|
||||
|
||||
---
|
||||
|
||||
## ما هو Memvid؟
|
||||
|
||||
Memvid هو نظام ذاكرة محمول للذكاء الاصطناعي يقوم بتغليف بياناتك، والمتجهات (Embeddings)، وهيكل البحث، والبيانات الوصفية في **ملف واحد فقط**.
|
||||
|
||||
بدلاً من تشغيل خطوط أنابيب RAG معقدة أو قواعد بيانات متجهة تعتمد على الخادم، يتيح Memvid استرجاعاً سريعاً للبيانات مباشرة من الملف.
|
||||
|
||||
النتيجة هي طبقة ذاكرة مستقلة عن النموذج (Model-agnostic) ولا تحتاج إلى بنية تحتية، مما يمنح وكلاء الذكاء الاصطناعي ذاكرة دائمة وطويلة المدى يمكنهم حملها في أي مكان.
|
||||
|
||||
---
|
||||
|
||||
## لماذا "إطارات الفيديو" (Video Frames)؟
|
||||
|
||||
يستلهم Memvid فكرته من ترميز الفيديو، ليس لتخزين الفيديو، بل **لتنظيم ذاكرة الذكاء الاصطناعي كمتسلسلة فائقة الكفاءة من "الإطارات الذكية" (Smart Frames) التي تُضاف باستمرار.**
|
||||
|
||||
"الإطار الذكي" هو وحدة غير قابلة للتغيير تخزن المحتوى مع الطوابع الزمنية، والتحقق من البيانات (Checksums)، والبيانات الوصفية الأساسية. يتم تجميع هذه الإطارات بطريقة تسمح بضغط البيانات وفهرستها والقراءة المتوازية بكفاءة عالية.
|
||||
|
||||
يسمح هذا التصميم بـ:
|
||||
|
||||
- **إضافة البيانات فقط:** الكتابة دون تعديل أو إفساد البيانات الموجودة.
|
||||
- **الاستعلام عبر الزمن:** البحث في حالات الذاكرة السابقة.
|
||||
- **جدول زمني للمعرفة:** فحص كيفية تطور المعرفة بمرور الوقت.
|
||||
- **سلامة البيانات:** ضمان عدم فقدان البيانات عند التعطل بفضل الإطارات الثابتة.
|
||||
|
||||
---
|
||||
|
||||
## المفاهيم الأساسية
|
||||
|
||||
- **محرك الذاكرة الحية:** إضافة وتطوير الذاكرة باستمرار عبر الجلسات.
|
||||
- **كبسولة السياق (`.mv2`):** كبسولات ذاكرة ذاتية الاحتواء وقابلة للمشاركة مع قواعد وصلاحية محددة.
|
||||
- **تصحيح السفر عبر الزمن:** إرجاع أو إعادة تشغيل أو تفريع أي حالة من حالات الذاكرة.
|
||||
- **الاستدعاء الذكي:** وصول محلي للذاكرة في أقل من 5 ملي ثانية مع ذاكرة تخزين مؤقت تنبؤية.
|
||||
- **ذكاء الترميز:** يختار ويحدث تقنيات الضغط تلقائياً بمرور الوقت.
|
||||
|
||||
---
|
||||
|
||||
## حالات الاستخدام
|
||||
|
||||
نظرًا لأن Memvid يعمل دون اتصال بالإنترنت ومستقل عن النماذج، فإنه يُستخدم في:
|
||||
|
||||
- وكلاء الذكاء الاصطناعي طويلي الأمد.
|
||||
- قواعد المعرفة للمؤسسات.
|
||||
- أنظمة الذكاء الاصطناعي التي تعمل "بدون إنترنت أولاً".
|
||||
- فهم الأكواد البرمجية (Codebases).
|
||||
- المساعدين الشخصيين والأنظمة الطبية والقانونية والمالية.
|
||||
|
||||
---
|
||||
|
||||
## أدوات المطورين (SDKs)
|
||||
|
||||
| الحزمة | طريقة التثبيت |
|
||||
| ----------------------------- | --------------------------- |
|
||||
| **واجهة السطر البرمجي (CLI)** | `npm install -g memvid-cli` |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` |
|
||||
| **Python SDK** | `pip install memvid-sdk` |
|
||||
| **Rust** | `cargo add memvid-core` |
|
||||
|
||||
---
|
||||
|
||||
## هيكل الملف
|
||||
|
||||
كل شيء يعيش داخل ملف واحد بصيغة `.mv2`:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ العنوان: النسخة والقدرة الاستيعابية
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ سجل العمليات للتعافي من الأعطال
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ أجزاء البيانات: الإطارات المضغوطة
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ الفهرس اللغوي: البحث النصي الكامل
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ الفهرس المتجهي: البحث بالمتجهات (HNSW)
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ الفهرس الزمني: الترتيب الزمني
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ جدول المحتويات: مواقع الأجزاء
|
||||
└────────────────────────────┘
|
||||
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## الدعم
|
||||
|
||||
هل لديك أسئلة؟
|
||||
البريد الإلكتروني: contact@memvid.com
|
||||
|
||||
**لا تنسَ ترك ⭐ لدعم المشروع!**
|
||||
|
||||
---
|
||||
|
||||
## الترخيص
|
||||
|
||||
رخصة Apache 2.0 — راجع ملف [LICENSE](../../LICENSE) لمزيد من التفاصيل.
|
||||
@@ -0,0 +1,379 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
|
||||
# মেমভিড-এ অবদান (বাংলা অনুবাদ)
|
||||
|
||||
<p align="center">
|
||||
<strong>মেমভিড হল এআই এজেন্টদের জন্য একটি একক-ফাইল মেমরি স্তর যার তাৎক্ষণিক পুনরুদ্ধার এবং দীর্ঘমেয়াদী মেমরি রয়েছে।</strong><br/>
|
||||
ডাটাবেস ছাড়াই স্থায়ী, সংস্করণযুক্ত এবং পোর্টেবল মেমরি।
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">ওয়েবসাইট</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">স্যান্ডবক্সি চেষ্টা করে দেখুন</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">ডক্স</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">আলোচনা</a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/</a>
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ প্রকল্পটি সমর্থন করার জন্য একটি তারকা দিন। ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## মেমভিড কী?
|
||||
|
||||
মেমভিড হল একটি পোর্টেবল এআই মেমরি সিস্টেম যা আপনার ডেটা, এম্বেডিং, অনুসন্ধান কাঠামো এবং মেটাডেটা একটি একক ফাইলে প্যাক করে।
|
||||
|
||||
জটিল RAG পাইপলাইন বা সার্ভার-ভিত্তিক ভেক্টর ডাটাবেস চালানোর পরিবর্তে, Memvid ফাইল থেকে সরাসরি দ্রুত ডেটা পুনরুদ্ধারে সহায়তা করে।
|
||||
|
||||
ফলাফল হল একটি মডেল-অজ্ঞেয়বাদী, অবকাঠামো-মুক্ত মেমোরি স্তর যা AI এজেন্টদের স্থায়ী, দীর্ঘমেয়াদী মেমোরি দেয় যা তারা যেকোনো জায়গায় নিতে পারে।
|
||||
---
|
||||
|
||||
## ভিডিও ফ্রেম কেন?
|
||||
|
||||
মেমভিড ভিডিও এনকোডিং থেকে অনুপ্রেরণা নেয়, ভিডিও সংরক্ষণের জন্য নয়, বরং এআই মেমোরিকে অ্যাপেন্ড-ওনলি, স্মার্ট ফ্রেমের অতি-দক্ষ সিকোয়েন্স হিসেবে সংগঠিত করার জন্য।
|
||||
|
||||
একটি স্মার্ট ফ্রেম হল একটি অ-পরিবর্তনযোগ্য ইউনিট যা টাইমস্ট্যাম্প, চেকসাম এবং মৌলিক মেটাডেটা সহ বিষয়বস্তু সংরক্ষণ করে।
|
||||
ফ্রেমগুলিকে এমনভাবে গোষ্ঠীভুক্ত করা হয় যা দক্ষ কম্প্রেশন, ইনডেক্সিং এবং সমান্তরাল পঠনের সুযোগ করে দেয়।
|
||||
|
||||
এই ফ্রেম-ভিত্তিক নকশাটি সক্ষম করে:
|
||||
|
||||
- বিদ্যমান ডেটা পরিবর্তন বা দূষিত না করে কেবল ডেটা যোগ করা
|
||||
- পূর্ববর্তী মেমরি অবস্থা অনুসন্ধান করা
|
||||
- জ্ঞান কীভাবে বিকশিত হয় তার টাইমলাইন-স্টাইল তদন্ত
|
||||
- প্রতিশ্রুতিবদ্ধ, অপরিবর্তনীয় ফ্রেমের মাধ্যমে ক্র্যাশ সুরক্ষা
|
||||
- ভিডিও এনকোডিং থেকে গৃহীত কৌশল ব্যবহার করে দক্ষ কম্প্রেশন।
|
||||
|
||||
ফলাফল হল একটি একক ফাইল যা AI সিস্টেমের জন্য একটি রিওয়াইন্ডেবল মেমরি টাইমলাইন হিসাবে কাজ করে।
|
||||
|
||||
---
|
||||
|
||||
## মূল ধারণা
|
||||
|
||||
- **লিভিং মেমোরি ইঞ্জিন**
|
||||
|
||||
একটি সেশনের সময় স্থায়ী মেমোরি যোগ করুন, শাখা করুন এবং বিকশিত করুন।
|
||||
|
||||
- **ক্যাপসুল রেফারেন্স (`.mv2`)**
|
||||
|
||||
নিয়ম এবং মেয়াদোত্তীর্ণতা সহ স্বয়ংসম্পূর্ণ, শেয়ারযোগ্য মেমোরি ক্যাপসুল।
|
||||
|
||||
- **টাইম-ট্রাভেল ডিবাগিং**
|
||||
|
||||
যেকোন মেমোরি স্টেট রিওয়াইন্ড, রিপ্লে বা শাখা করুন।
|
||||
|
||||
- **স্মার্ট রিকল**
|
||||
|
||||
প্রেডিক্টিভ ক্যাশিং সহ সাব-5ms স্থানীয় মেমোরি অ্যাক্সেস।
|
||||
|
||||
- **কোডেক ইন্টেলিজেন্স**
|
||||
|
||||
সময়ের সাথে সাথে স্বয়ংক্রিয়ভাবে কম্প্রেশন নির্বাচন এবং আপগ্রেড করে।
|
||||
|
||||
---
|
||||
|
||||
## ব্যবহারের ক্ষেত্রে
|
||||
|
||||
মেমভিড হল একটি পোর্টেবল, সার্ভারলেস মেমরি স্তর যা এআই এজেন্টদের স্থায়ী মেমরি এবং দ্রুত রিকল দেয়। যেহেতু এটি মডেল-অ্যাগনস্টিক, মাল্টি-মডেল এবং সম্পূর্ণ অফলাইনে কাজ করে, তাই ডেভেলপাররা বিভিন্ন বাস্তব-বিশ্ব অ্যাপ্লিকেশনে মেমভিড ব্যবহার করছে।
|
||||
|
||||
- দীর্ঘমেয়াদী এআই এজেন্ট
|
||||
- এন্টারপ্রাইজ জ্ঞান ভিত্তি
|
||||
- অফলাইন-প্রথম এআই সিস্টেম
|
||||
- কোডবেস বোঝা
|
||||
- গ্রাহক সহায়তা এজেন্ট
|
||||
- ওয়ার্কফ্লো অটোমেশন
|
||||
- বিক্রয় এবং বিপণন সহ-পাইলট
|
||||
- ব্যক্তিগত জ্ঞান সহকারী
|
||||
- চিকিৎসা, আইনি এবং আর্থিক এজেন্ট
|
||||
- নিরীক্ষণযোগ্য এবং ডিবাগযোগ্য এআই কর্মপ্রবাহ
|
||||
- কাস্টম অ্যাপ্লিকেশন
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
আপনার পছন্দের ভাষায় Memvid ব্যবহার করুন:
|
||||
|
||||
| Package | Install | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Installation (Rust)
|
||||
|
||||
### আবশ্যকতা
|
||||
|
||||
- **Rust 1.85.0+** — Install from [rustup.rs](https://rustup.rs)
|
||||
|
||||
### আপনার প্রকল্পে যোগ করুন
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Description |
|
||||
| ------------------- | ---------------------------------------------- |
|
||||
| `lex` | Full-text search with BM25 ranking (Tantivy) |
|
||||
| `pdf_extract` | Pure Rust PDF text extraction |
|
||||
| `vec` | Vector similarity search (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings for image search |
|
||||
| `whisper` | Audio transcription with Whisper |
|
||||
| `temporal_track` | Natural language date parsing ("last Tuesday") |
|
||||
| `parallel_segments` | Multi-threaded ingestion |
|
||||
| `encryption` | Password-based encryption capsules (.mv2e) |
|
||||
|
||||
Enable features as needed:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## দ্রুত শুরু
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## তৈরি করুন
|
||||
|
||||
রিপোজিটরিটি ক্লোন করুন:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
ডিবাগ মোডে তৈরি করুন:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
রিলিজ মোডে তৈরি করুন (অপ্টিমাইজ করা):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
স্বতন্ত্র বৈশিষ্ট্য সহ তৈরি করুন:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
## পরীক্ষাগুলি চালান
|
||||
|
||||
সকল পরীক্ষা চালান:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
নিম্নলিখিত আউটপুট দিয়ে পরীক্ষাটি চালান:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
একটি নির্দিষ্ট পরীক্ষা চালান:
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
শুধুমাত্র ইন্টিগ্রেশন পরীক্ষা চালান:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## উদাহরণ
|
||||
|
||||
The `examples/` কার্যকরী ডিরেক্টরিগুলির উদাহরণ হল:
|
||||
|
||||
### মৌলিক ব্যবহার
|
||||
|
||||
এটি তৈরি, পুট, অনুসন্ধান এবং টাইমলাইন ক্রিয়াকলাপগুলি দেখায়:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### পিডিএফ ইনজেকশন
|
||||
|
||||
পিডিএফ ডকুমেন্টগুলি গ্রহণ করুন এবং অনুসন্ধান করুন ("Attention Is All You Need" পেপার ব্যবহার করে):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP ভিজ্যুয়াল সার্চ
|
||||
|
||||
CLIP এম্বেডিং ব্যবহার করে ছবি সার্চ (`ক্লিপ` বৈশিষ্ট্য প্রয়োজন):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### হুইস্পার ট্রান্সক্রিপশন
|
||||
|
||||
অডিও ট্রান্সক্রিপশন (`হুইস্পার` বৈশিষ্ট্য প্রয়োজন):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ফাইল ফরম্যাট
|
||||
|
||||
সবকিছু একটি একক `.mv2` ফাইলের মধ্যে রয়েছে:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
কোন `.wal`, `.lock`, `.shm`, অথবা সাইডকার ফাইল নেই। কখনও না।
|
||||
|
||||
সম্পূর্ণ ফাইল ফর্ম্যাট স্পেসিফিকেশনের জন্য [MV2_SPEC.md](MV2_SPEC.md) দেখুন।
|
||||
|
||||
---
|
||||
|
||||
## সহায়তা
|
||||
|
||||
আপনার কি কোন প্রশ্ন বা প্রতিক্রিয়া আছে?
|
||||
|
||||
ইমেল: contact@memvid.com
|
||||
|
||||
**সমর্থন দেখানোর জন্য ⭐ দিন**
|
||||
|
||||
---
|
||||
|
||||
## লাইসেন্স
|
||||
|
||||
Apache License 2.0 — আরও তথ্যের জন্য [LICENSE](LICENSE) ফাইলটি দেখুন।
|
||||
@@ -0,0 +1,424 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid je jednosouborová paměťová vrstva pro AI agenty s okamžitým vyhledáváním a dlouhodobou pamětí.</strong><br/>
|
||||
Trvalá, verzovaná a přenosná paměť, bez databází.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Zanechte hvězdičku na podporu projektu ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Co je Memvid?
|
||||
|
||||
Memvid je systém pro tvorbu AI pamětí, který balí vaše data, embeddingy, strukturu vyhledávání a metadata do jediného souboru.
|
||||
|
||||
Místo spouštění složitých RAG řešení nebo serverových vektorových databází umožňuje Memvid rychlé vyhledávání přímo ze souboru.
|
||||
|
||||
Výsledkem je modelově nezávislá paměťová vrstva bez infrastruktury, která poskytuje agentům AI trvalou, dlouhodobou paměť, kterou lze přenášet kamkoli.
|
||||
|
||||
---
|
||||
|
||||
## Co jsou inteligentní rámce?
|
||||
|
||||
Memvid čerpá inspiraci z enkódování videa, nikoli za účelem ukládání videa, ale za účelem **organizace paměti AI jako ultraefektivní sekvence inteligentních rámců, do kterých lze data pouze přidávat.**
|
||||
|
||||
Inteligentní rámec je neměnná jednotka, která ukládá obsah spolu s časovými značkami, kontrolními součty a základními metadaty.
|
||||
Rámce jsou seskupeny tak, aby umožňovaly efektivní kompresi, indexování a paralelní čtení.
|
||||
|
||||
Tento design založený na rámcích umožňuje:
|
||||
|
||||
- Pouze zápisy bez úpravy nebo poškození existujících dat
|
||||
- Dotazy na minulé stavy paměti
|
||||
- Kontrolu vývoje znalostí ve stylu časové osy
|
||||
- Bezpečnost proti selhání díky závazným, neměnným rámcům
|
||||
- Efektivní kompresi pomocí technik převzatých z kódování videa
|
||||
|
||||
Výsledkem je jediný soubor, který se chová jako časová osa paměti pro systémy AI, ve které lze snadno hledat.
|
||||
|
||||
---
|
||||
|
||||
## Základní koncepty
|
||||
|
||||
- **Living Memory Engine**
|
||||
Kontinuální přidávání, rozvětvování a vývoj paměti, napříč relacemi.
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
Samostatné, sdílené paměťové kapsle s pravidly a dobou platnosti.
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
Převíjení, přehrávání nebo rozvětvování libovolného stavu paměti.
|
||||
|
||||
- **Smart Recall**
|
||||
Přístup k lokální paměti za méně než 5 ms s prediktivním ukládáním do mezipaměti.
|
||||
|
||||
- **Codec Intelligence**
|
||||
Automaticky vybírá a vylepšuje kompresi v průběhu času.
|
||||
|
||||
---
|
||||
|
||||
## Případy použití
|
||||
|
||||
Memvid je přenosná paměťová vrstva bez serveru, která poskytuje agentům AI trvalou paměť a rychlé vyvolání. Protože je modelově nezávislá, multimodální a funguje zcela offline, vývojáři používají Memvid v široké škále reálných aplikací.
|
||||
|
||||
- Dlouhodobě běžící AI agenti
|
||||
- Firemní znalostní báze
|
||||
- Offline-First AI systémy
|
||||
- Porozumění kódu
|
||||
- Agenti zákaznické podpory
|
||||
- Automatizace pracovních postupů
|
||||
- Asistenti prodeje a marketingu
|
||||
- Osobní znalostní asistenti
|
||||
- Lékařští, právní a finanční agenti
|
||||
- Auditovatelné a laditelné AI pracovní postupy
|
||||
- Vlastní aplikace
|
||||
|
||||
---
|
||||
|
||||
## SDK a CLI
|
||||
|
||||
Používejte Memvid ve svém preferovaném jazyce:
|
||||
|
||||
| Balíček | Instalace | Odkazy |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Instalace (Rust)
|
||||
|
||||
### Požadavky
|
||||
|
||||
- **Rust 1.85.0+** — Instalace z [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Přidejte do svého projektu
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Funkční příznaky
|
||||
|
||||
| Funkce | Popis |
|
||||
| ------------------- | ---------------------------------------------------------- |
|
||||
| `lex` | Fulltextové vyhledávání s hodnocením BM25 (Tantivy) |
|
||||
| `pdf_extract` | Čistá extrakce textu z PDF v Rustu |
|
||||
| `vec` | Vektorové vyhledávání podobnosti (HNSW + lokální vkládání textu přes ONNX) |
|
||||
| `clip` | Vizuální vkládání CLIP pro vyhledávání obrázků |
|
||||
| `whisper` | Přepis zvuku pomocí Whisper |
|
||||
| `temporal_track` | Analýza datumu v přirozeném jazyce ("minulé úterý") |
|
||||
| `parallel_segments` | Vícevláknové načítání |
|
||||
| `encryption` | Kapsle šifrované pomocí hesla (.mv2e) |
|
||||
|
||||
Povolte funkce podle potřeby:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Rychlý start
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Vytvoř nový paměťový soubor
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Přidej dokumenty s metadaty
|
||||
let opts = PutOptions::builder()
|
||||
.title("Zápis jednání")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 plánované diskuze...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Vyhledávání
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Sestavení
|
||||
|
||||
Klonujte repozitář:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
Sestavení v režimu developkment:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Sestavení v režimu production (optimalizované):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Sestavení s konkrétními funkcemi:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Spuštění testů
|
||||
|
||||
Spuštění všech testů:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Spuštění testů s výstupem:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Spuštění konkrétního testu:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Spuštění pouze integračních testů:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Příklady
|
||||
|
||||
Adresář `examples/` obsahuje funkční příklady:
|
||||
|
||||
### Základní použití
|
||||
|
||||
Demonstruje operace vytváření, vkládání, vyhledávání a práci s časovou osou:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Načítání PDF
|
||||
|
||||
Načítání a vyhledávání v dokumentech PDF (demo používá dokument "Attention Is All You Need"):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### Vizuální vyhledávání CLIP
|
||||
|
||||
Vyhledávání obrázků pomocí vložení CLIP (vyžaduje funkci `clip`):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Přepis Whisper
|
||||
|
||||
Přepis zvuku (vyžaduje funkci `whisper`):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Modely vkládání textu
|
||||
|
||||
Funkce `vec` zahrnuje podporu lokálního vkládání textu pomocí modelů ONNX. Před použitím lokálního vkládání textu je nutné ručně stáhnout soubory modelů.
|
||||
|
||||
### Rychlý start: BGE-small (doporučeno)
|
||||
|
||||
Stáhněte si výchozí model BGE-small (384 dimenzí, rychlý a efektivní):
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# Download ONNX model
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# Download tokenizer
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### Dostupné modely
|
||||
|
||||
| Model | Rozměry | Velikost | Nejvhodnější pro |
|
||||
| ----------------------- | ---------- | ------- | --------------------- |
|
||||
| `bge-small-en-v1.5` | 384 | ~120 MB | Výchozí, rychlý |
|
||||
| `bge-base-en-v1.5` | 768 | ~420 MB | Lepší kvalita |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530 MB | Univerzální úkoly |
|
||||
| `gte-large` | 1024 | ~1,3 GB | Nejvyšší kvalita |
|
||||
|
||||
### Další modely
|
||||
|
||||
**BGE-base** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large** (1024 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### Použití v kódu
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Použít výchozí model (BGE-small)
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// Použijte jiný model
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
Kompletní příklad s výpočtem podobnosti a hodnocením vyhledávání najdete v souboru `examples/text_embedding.rs`.
|
||||
|
||||
---
|
||||
|
||||
## Formát souboru
|
||||
|
||||
Vše je uloženo v jediném souboru `.mv2`:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Záhlaví (4 KB) │ Magie, verze, kapacita
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64 MB) │ Obnova po selhání
|
||||
├────────────────────────────┤
|
||||
│ Datové segmenty │ Komprimované rámce
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ Vektory HNSW
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronologické řazení
|
||||
├────────────────────────────┤
|
||||
│ TOC (zápatí) │ Segmentové posuny
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Žádné další soubory, jako je `.wal`, `.lock`, `.shm` nejsou potřeba. Nikdy.
|
||||
|
||||
Kompletní specifikace formátu souboru najdete v [MV2_SPEC.md](MV2_SPEC.md).
|
||||
|
||||
---
|
||||
|
||||
## Podpora
|
||||
|
||||
Máte dotazy nebo připomínky?
|
||||
E-mail: contact@memvid.com
|
||||
|
||||
**Dejte ⭐ a projevte svou podporu**
|
||||
|
||||
---
|
||||
|
||||
## Licence
|
||||
|
||||
Apache License 2.0 — podrobnosti najdete v souboru [LICENSE](LICENSE).
|
||||
@@ -0,0 +1,500 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid es una capa de memoria de un solo archivo para agentes de IA, con recuperación instantánea y memoria a largo plazo.</strong><br/>
|
||||
Memoria persistente, versionada y portable, sin bases de datos.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Deja una STAR para apoyar el proyecto ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Lo más destacado de los benchmarks
|
||||
|
||||
**🚀 Mayor precisión que cualquier otro sistema de memoria:** +35% SOTA en LoCoMo, con recall y razonamiento conversacional de largo horizonte de primer nivel.
|
||||
|
||||
**🧠 Mejor razonamiento multi-hop y temporal:** +76% en multi-hop y +56% en temporal frente al promedio de la industria.
|
||||
|
||||
**⚡ Latencia ultra baja a escala:** 0.025 ms P50 y 0.075 ms P99, con 1,372× más throughput que los enfoques estándar.
|
||||
|
||||
**🔬 Benchmarks totalmente reproducibles:** LoCoMo (10 conversaciones de ~26K tokens), evaluación open source y LLM-as-Judge.
|
||||
|
||||
---
|
||||
|
||||
## ¿Qué es Memvid?
|
||||
|
||||
Memvid es un sistema de memoria portable para IA que empaqueta tus datos, embeddings, estructura de búsqueda y metadatos en un solo archivo.
|
||||
|
||||
En lugar de ejecutar pipelines RAG complejos o bases de datos vectoriales basadas en servidor, Memvid permite una recuperación rápida directamente desde el archivo.
|
||||
|
||||
El resultado es una capa de memoria agnóstica al modelo, sin infraestructura, que da a los agentes de IA una memoria persistente y a largo plazo que pueden llevar a cualquier parte.
|
||||
|
||||
---
|
||||
|
||||
## ¿Por qué fotogramas de vídeo?
|
||||
|
||||
Memvid se inspira en la codificación de vídeo, no para almacenar vídeo, sino para **organizar la memoria de IA como una secuencia de Smart Frames ultrarrápida y append-only.**
|
||||
|
||||
Un Smart Frame es una unidad inmutable que almacena contenido junto con marcas de tiempo (timestamps), checksums y metadatos básicos.
|
||||
Los frames se agrupan de una forma que permite una compresión, indexación y lecturas paralelas eficientes.
|
||||
|
||||
Este diseño basado en frames permite:
|
||||
|
||||
- Escrituras append-only sin modificar ni corromper los datos existentes
|
||||
- Consultas sobre estados pasados de la memoria
|
||||
- Inspección estilo línea temporal (timeline) de cómo evoluciona el conocimiento
|
||||
- Seguridad ante fallos (crash safety) mediante frames confirmados e inmutables
|
||||
- Compresión eficiente usando técnicas adaptadas de la codificación de vídeo
|
||||
|
||||
El resultado es un único archivo que se comporta como una línea temporal de memoria “rebobinable” para sistemas de IA.
|
||||
|
||||
---
|
||||
|
||||
## Conceptos principales
|
||||
|
||||
- **Living Memory Engine**
|
||||
Añade, ramifica (branch) y evoluciona la memoria de forma continua entre sesiones.
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
Cápsulas de memoria autocontenidas y compartibles, con reglas y caducidad.
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
Rebobina, reproduce (replay) o ramifica cualquier estado de memoria.
|
||||
|
||||
- **Smart Recall**
|
||||
Acceso local a memoria en menos de 5ms con caché predictiva.
|
||||
|
||||
- **Codec Intelligence**
|
||||
Selecciona y actualiza la compresión automáticamente con el tiempo.
|
||||
|
||||
---
|
||||
|
||||
## Casos de uso
|
||||
|
||||
Memvid es una capa de memoria portable y serverless que da a los agentes de IA memoria persistente y recuerdo rápido. Como es agnóstica al modelo, multi-modal y funciona totalmente offline, los desarrolladores están usando Memvid en una amplia gama de aplicaciones reales.
|
||||
|
||||
- Agentes de IA de larga duración
|
||||
- Bases de conocimiento empresariales
|
||||
- Sistemas de IA offline-first
|
||||
- Comprensión de codebases
|
||||
- Agentes de soporte al cliente
|
||||
- Automatización de flujos de trabajo
|
||||
- Copilotos de ventas y marketing
|
||||
- Asistentes de conocimiento personal
|
||||
- Agentes médicos, legales y financieros
|
||||
- Flujos de trabajo de IA auditables y depurables
|
||||
- Aplicaciones personalizadas
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
Usa Memvid en tu lenguaje preferido:
|
||||
|
||||
| Package | Install | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Instalación (Rust)
|
||||
|
||||
### Requisitos
|
||||
|
||||
- **Rust 1.85.0+** — Instálalo desde [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Añadir a tu proyecto
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Descripción |
|
||||
| ------------------- | ------------------------------------------------------------------ |
|
||||
| `lex` | Búsqueda full-text con ranking BM25 (Tantivy) |
|
||||
| `pdf_extract` | Extracción de texto PDF 100% en Rust |
|
||||
| `vec` | Búsqueda por similitud vectorial (HNSW + embeddings locales vía ONNX) |
|
||||
| `clip` | Embeddings visuales CLIP para búsqueda de imágenes |
|
||||
| `whisper` | Transcripción de audio con Whisper |
|
||||
| `api_embed` | Embeddings en la nube mediante API (OpenAI) |
|
||||
| `temporal_track` | Interpretación de fechas en lenguaje natural ("el martes pasado") |
|
||||
| `parallel_segments` | Ingesta multi-hilo |
|
||||
| `encryption` | Cápsulas cifradas con contraseña (.mv2e) |
|
||||
| `symspell_cleanup` | Reparación robusta de texto PDF (corrige "emp lo yee" -> "employee") |
|
||||
|
||||
Activa las features según lo necesites:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Inicio rápido
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Build
|
||||
|
||||
Clona el repositorio:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
Compila en modo debug:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Compila en modo release (optimizado):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Compila con features específicas:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ejecutar tests
|
||||
|
||||
Ejecuta todos los tests:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Ejecuta los tests con salida:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Ejecuta un test específico:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Ejecuta solo tests de integración:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ejemplos
|
||||
|
||||
El directorio `examples/` contiene ejemplos funcionales:
|
||||
|
||||
### Uso básico
|
||||
|
||||
Demuestra operaciones de create, put, search y timeline:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Ingesta de PDF
|
||||
|
||||
Ingiere y busca documentos PDF (usa el paper “Attention Is All You Need”):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### Búsqueda visual con CLIP
|
||||
|
||||
Búsqueda de imágenes usando embeddings de CLIP (requiere la feature `clip`):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Transcripción con Whisper
|
||||
|
||||
Transcripción de audio (requiere la feature `whisper`):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Modelos de embeddings de texto
|
||||
|
||||
La feature `vec` incluye soporte para embeddings de texto locales usando modelos ONNX. Antes de usar embeddings locales, necesitas descargar manualmente los archivos del modelo.
|
||||
|
||||
### Inicio rápido: BGE-small (recomendado)
|
||||
|
||||
Descarga el modelo BGE-small por defecto (384 dimensiones, rápido y eficiente):
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# Descargar modelo ONNX
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# Descargar tokenizer
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### Modelos disponibles
|
||||
|
||||
| Modelo | Dimensiones | Tamaño | Mejor para |
|
||||
| ----------------------- | ----------- | ------ | --------------------- |
|
||||
| `bge-small-en-v1.5` | 384 | ~120MB | Opción por defecto, rápido |
|
||||
| `bge-base-en-v1.5` | 768 | ~420MB | Mejor calidad |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530MB | Tareas versátiles |
|
||||
| `gte-large` | 1024 | ~1.3GB | Máxima calidad |
|
||||
|
||||
### Otros modelos
|
||||
|
||||
**BGE-base** (768 dimensiones):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic** (768 dimensiones):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large** (1024 dimensiones):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### Uso en código
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Usar el modelo por defecto (BGE-small)
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// Usar un modelo distinto
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
Consulta `examples/text_embedding.rs` para ver un ejemplo completo con cálculo de similitud y ranking de búsqueda.
|
||||
|
||||
### Consistencia del modelo
|
||||
|
||||
Para evitar mezclar modelos por accidente, por ejemplo consultar un índice BGE-small con embeddings de OpenAI, puedes asociar explícitamente tu instancia de Memvid a un nombre de modelo:
|
||||
|
||||
```rust
|
||||
// Vincula el índice a un modelo concreto.
|
||||
// Si el índice ya fue creado con otro modelo, devolverá un error.
|
||||
mem.set_vec_model("bge-small-en-v1.5")?;
|
||||
```
|
||||
|
||||
Esta vinculación es persistente. Una vez definida, cualquier intento futuro de usar otro nombre de modelo fallará de inmediato con un error `ModelMismatch`.
|
||||
|
||||
---
|
||||
|
||||
## Embeddings por API (OpenAI)
|
||||
|
||||
La feature `api_embed` habilita la generación de embeddings en la nube usando la API de OpenAI.
|
||||
|
||||
### Configuración
|
||||
|
||||
Define tu clave de API de OpenAI:
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY="sk-..."
|
||||
```
|
||||
|
||||
### Uso
|
||||
|
||||
```rust
|
||||
use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Usar el modelo por defecto (text-embedding-3-small)
|
||||
let config = OpenAIConfig::default();
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 1536);
|
||||
|
||||
// Usar un modelo de mayor calidad
|
||||
let config = OpenAIConfig::large(); // text-embedding-3-large (3072 dims)
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
### Modelos disponibles
|
||||
|
||||
| Modelo | Dimensiones | Mejor para |
|
||||
| ------------------------ | ----------- | -------------------------------- |
|
||||
| `text-embedding-3-small` | 1536 | Por defecto, más rápido y económico |
|
||||
| `text-embedding-3-large` | 3072 | Máxima calidad |
|
||||
| `text-embedding-ada-002` | 1536 | Modelo heredado |
|
||||
|
||||
Consulta `examples/openai_embedding.rs` para ver un ejemplo completo.
|
||||
|
||||
---
|
||||
|
||||
## Formato de archivo
|
||||
|
||||
Todo vive en un único archivo `.mv2`:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Sin archivos `.wal`, `.lock`, `.shm` ni sidecars. Nunca.
|
||||
|
||||
Consulta [MV2_SPEC.md](MV2_SPEC.md) para la especificación completa del formato de archivo.
|
||||
|
||||
---
|
||||
|
||||
## Soporte
|
||||
|
||||
¿Tienes preguntas o feedback?
|
||||
Email: contact@memvid.com
|
||||
|
||||
**Deja una ⭐ para mostrar apoyo**
|
||||
|
||||
---
|
||||
|
||||
> **Memvid v1 (memoria basada en QR) está obsoleto**
|
||||
>
|
||||
> Si estás viendo referencias a códigos QR, estás usando información desactualizada.
|
||||
>
|
||||
> Consulta: https://docs.memvid.com/memvid-v1-deprecation
|
||||
|
||||
---
|
||||
|
||||
## Licencia
|
||||
|
||||
Apache License 2.0 — consulta el archivo [LICENSE](LICENSE) para más detalles.
|
||||
@@ -0,0 +1,346 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid est une couche mémoire à fichier unique pour agents IA, avec récupération instantanée et mémoire long terme.</strong><br/>
|
||||
Mémoire persistante, versionnée et portable, sans bases de données.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Laissez une STAR pour soutenir le projet ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Qu'est-ce que Memvid ?
|
||||
|
||||
Memvid est un système de mémoire IA portable qui regroupe vos données, embeddings, structure de recherche et métadonnées dans un seul fichier.
|
||||
|
||||
Au lieu d'exécuter des pipelines RAG complexes ou des bases de données vectorielles côté serveur, Memvid permet une récupération rapide directement depuis le fichier.
|
||||
|
||||
Le résultat est une couche mémoire agnostique au modèle, sans infrastructure, qui donne aux agents IA une mémoire persistante et longue durée qu'ils peuvent emporter partout.
|
||||
|
||||
---
|
||||
|
||||
## Pourquoi des images vidéo ?
|
||||
|
||||
Memvid s'inspire de l'encodage vidéo, non pas pour stocker de la vidéo, mais pour **organiser la mémoire IA en une séquence append-only ultra-efficace de Smart Frames.**
|
||||
|
||||
Une Smart Frame est une unité immuable qui stocke le contenu avec des horodatages, des checksums et des métadonnées de base.
|
||||
Les frames sont regroupées d'une manière qui permet une compression, une indexation et des lectures parallèles efficaces.
|
||||
|
||||
Ce design basé sur les frames permet :
|
||||
|
||||
- Écritures append-only sans modifier ni corrompre les données existantes
|
||||
- Requêtes sur des états mémoire passés
|
||||
- Inspection type timeline de l'évolution des connaissances
|
||||
- Sécurité en cas de crash via des frames immuables et validées
|
||||
- Compression efficace grâce à des techniques adaptées de l'encodage vidéo
|
||||
|
||||
Le résultat est un fichier unique qui se comporte comme une timeline mémoire rembobinable pour les systèmes IA.
|
||||
|
||||
---
|
||||
|
||||
## Concepts de base
|
||||
|
||||
- **Moteur de mémoire vivant**
|
||||
Ajoutez, branchez et faites évoluer la mémoire en continu entre les sessions.
|
||||
|
||||
- **Capsule de Contexte (`.mv2`)**
|
||||
Capsules mémoire autonomes et partageables avec règles et expiration.
|
||||
|
||||
- **Débogage par 'voyage temporel'**
|
||||
Rembobinez, rejouez ou branchez n'importe quel état mémoire.
|
||||
|
||||
- **Rappel intelligent**
|
||||
Accès mémoire local en moins de 5 ms avec cache prédictif.
|
||||
|
||||
- **Intelligence du codec**
|
||||
Sélection et mise à niveau automatiques de la compression au fil du temps.
|
||||
|
||||
---
|
||||
|
||||
## Cas d'usage
|
||||
Memvid est une couche mémoire portable et sans serveur qui donne aux agents IA une mémoire persistante et un rappel rapide. Parce qu'il est agnostique au modèle, multimodal et fonctionne entièrement hors ligne, les développeurs utilisent Memvid pour un large éventail d'applications réelles.
|
||||
|
||||
- Agents IA longue durée
|
||||
- Bases de connaissances d'entreprise
|
||||
- Systèmes IA offline-first
|
||||
- Compréhension de codebase
|
||||
- Agents de support client
|
||||
- Automatisation des workflows
|
||||
- Copilotes ventes et marketing
|
||||
- Assistants de connaissance personnels
|
||||
- Agents médicaux, juridiques et financiers
|
||||
- Workflows IA auditables et débogables
|
||||
- Applications sur mesure
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
Utilisez Memvid dans votre langage préféré :
|
||||
|
||||
| Package | Installation | Liens |
|
||||
|---------|---------|-------|
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Installation (Rust)
|
||||
|
||||
### Prérequis
|
||||
|
||||
- **Rust 1.85.0+** — Installer depuis [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Ajouter à votre projet
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| `lex` | Full-text search with BM25 ranking (Tantivy) |
|
||||
| `pdf_extract` | Pure Rust PDF text extraction |
|
||||
| `vec` | Vector similarity search (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings for image search |
|
||||
| `whisper` | Audio transcription with Whisper |
|
||||
| `temporal_track` | Natural language date parsing ("last Tuesday") |
|
||||
| `parallel_segments` | Multi-threaded ingestion |
|
||||
| `encryption` | Password-based encryption capsules (.mv2e) |
|
||||
|
||||
Activez les features selon vos besoins :
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Démarrage rapide
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Créer un nouveau fichier de mémoire
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Ajouter des documents avec des métadonnées
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Rechercher
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Compiler
|
||||
|
||||
Cloner le repository :
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
Compiler en mode debug :
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Compiler en mode release (optimisé) :
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Compiler avec des features spécifiques :
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Exécuter les tests
|
||||
|
||||
Exécuter tous les tests :
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Exécuter les tests avec sortie :
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Exécuter un test spécifique :
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Exécuter uniquement les tests d'intégration :
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Exemples
|
||||
|
||||
Le répertoire `examples/` contient des exemples fonctionnels :
|
||||
|
||||
### Utilisation de base
|
||||
|
||||
Démontre create, put, search et les opérations de timeline :
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Ingestion de PDF
|
||||
|
||||
Ingérer et rechercher des documents PDF (utilise l'article "Attention Is All You Need") :
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### Recherche visuelle CLIP
|
||||
|
||||
Recherche d'images à l'aide d'embeddings CLIP (nécessite la feature `clip`) :
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Transcription Whisper
|
||||
|
||||
Transcription audio (nécessite la feature `whisper`) :
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Format de fichier
|
||||
|
||||
Tout est dans un seul fichier `.mv2` :
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Pas de `.wal`, `.lock`, `.shm` ou fichiers auxiliaires. Jamais.
|
||||
|
||||
Voir [MV2_SPEC.md](MV2_SPEC.md) pour la spécification complète du format de fichier.
|
||||
|
||||
---
|
||||
|
||||
## Support
|
||||
|
||||
Vous avez des questions ou des retours ?
|
||||
Email : contact@memvid.com
|
||||
|
||||
**Laissez une ⭐ pour montrer votre soutien**
|
||||
|
||||
---
|
||||
|
||||
## Licence
|
||||
|
||||
Apache License 2.0 — voir le fichier [LICENSE](LICENSE) pour plus de détails.
|
||||
@@ -0,0 +1,347 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>मेमविड AI एजेंट के लिए एक सिंगल-फाइल मेमोरी लेयर है जिसमें तुरंत रिट्रीवल और लॉन्ग-टर्म मेमोरी होती है।</strong><br/>
|
||||
बिना डेटाबेस के, स्थायी, वर्शन वाली और पोर्टेबल मेमोरी।
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ प्रोजेक्ट को सपोर्ट करने के लिए एक स्टार दें। ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## What is Memvid?
|
||||
|
||||
मेमविड एक पोर्टेबल AI मेमोरी सिस्टम है जो आपके डेटा, एम्बेडिंग, सर्च स्ट्रक्चर और मेटाडेटा को एक ही फ़ाइल में पैक करता है।
|
||||
|
||||
जटिल RAG पाइपलाइन या सर्वर-आधारित वेक्टर डेटाबेस चलाने के बजाय, Memvid सीधे फ़ाइल से तेज़ी से डेटा रिट्रीव करने में मदद करता है।
|
||||
|
||||
इसका नतीजा एक मॉडल-एग्नोस्टिक, इंफ्रास्ट्रक्चर-फ्री मेमोरी लेयर है जो AI एजेंट को परमानेंट, लॉन्ग-टर्म मेमोरी देती है जिसे वे कहीं भी ले जा सकते हैं।
|
||||
|
||||
---
|
||||
|
||||
## वीडियो फ़्रेम क्यों?
|
||||
|
||||
मेमविड वीडियो एन्कोडिंग से प्रेरणा लेता है, वीडियो स्टोर करने के लिए नहीं, बल्कि**AI मेमोरी को स्मार्ट फ्रेम्स के अपेंड-ओनली, अल्ट्रा-एफ़िशिएंट सीक्वेंस के तौर पर ऑर्गनाइज़ करें।**
|
||||
|
||||
स्मार्ट फ्रेम एक ऐसा यूनिट है जिसे बदला नहीं जा सकता, जो टाइमस्टैम्प, चेकसम और बेसिक मेटाडेटा के साथ कंटेंट स्टोर करता है।
|
||||
फ्रेम को इस तरह से ग्रुप किया जाता है जिससे कुशल कम्प्रेशन, इंडेक्सिंग और पैरेलल रीड संभव हो सके।
|
||||
|
||||
यह फ्रेम-आधारित डिज़ाइन इन चीज़ों को संभव बनाता है:
|
||||
|
||||
- मौजूदा डेटा को संशोधित या दूषित किए बिना केवल डेटा जोड़ना
|
||||
- पिछली मेमोरी स्थितियों पर प्रश्न
|
||||
- ज्ञान कैसे विकसित होता है, इसकी टाइमलाइन-शैली में जांच
|
||||
- प्रतिबद्ध, अपरिवर्तनीय फ्रेम के माध्यम से क्रैश सुरक्षा
|
||||
- वीडियो एन्कोडिंग से अपनाई गई तकनीकों का उपयोग करके कुशल कम्प्रेशन।
|
||||
|
||||
इसका नतीजा एक सिंगल फ़ाइल होती है जो AI सिस्टम के लिए रिवाइंड करने लायक मेमोरी टाइमलाइन की तरह काम करती है।
|
||||
|
||||
---
|
||||
|
||||
## मुख्य अवधारणाएँ
|
||||
|
||||
- **लिविंग मेमोरी इंजन**
|
||||
सेशन के दौरान लगातार मेमोरी को जोड़ें, ब्रांच करें और विकसित करें।
|
||||
|
||||
- **कैप्सूल संदर्भ (`.mv2`)**
|
||||
नियमों और एक्सपायरी के साथ सेल्फ-कंटेन्ड, शेयर करने लायक मेमोरी कैप्सूल।
|
||||
|
||||
- **टाइम-ट्रैवल डिबगिंग**
|
||||
किसी भी मेमोरी स्टेट को रिवाइंड, रिप्ले या ब्रांच करें।
|
||||
|
||||
- **स्मार्ट रिकॉल**
|
||||
प्रेडिक्टिव कैशिंग के साथ सब-5ms लोकल मेमोरी एक्सेस।
|
||||
|
||||
- **कोडेक इंटेलिजेंस**
|
||||
यह समय के साथ कम्प्रेशन को ऑटो-सेलेक्ट और अपग्रेड करता है।
|
||||
|
||||
---
|
||||
|
||||
## उपयोग के मामले
|
||||
|
||||
मेमविड एक पोर्टेबल, सर्वरलेस मेमोरी लेयर है जो AI एजेंट को परमानेंट मेमोरी और तेज़ रिकॉल देता है। क्योंकि यह मॉडल-एग्नोस्टिक, मल्टी-मॉडल है, और पूरी तरह से ऑफ़लाइन काम करता है, इसलिए डेवलपर्स मेमविड का इस्तेमाल कई तरह के रियल-वर्ल्ड एप्लीकेशन में कर रहे हैं।
|
||||
|
||||
- लंबे समय तक चलने वाले AI एजेंट
|
||||
- एंटरप्राइज़ नॉलेज बेस
|
||||
- ऑफ़लाइन-फ़र्स्ट AI सिस्टम
|
||||
- कोडबेस को समझना
|
||||
- कस्टमर सपोर्ट एजेंट
|
||||
- वर्कफ़्लो ऑटोमेशन
|
||||
- सेल्स और मार्केटिंग कोपायलट
|
||||
- पर्सनल नॉलेज असिस्टेंट
|
||||
- मेडिकल, लीगल और फाइनेंशियल एजेंट
|
||||
- ऑडिट करने योग्य और डीबग करने योग्य AI वर्कफ़्लो
|
||||
- कस्टम एप्लीकेशन
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
मेमविड को अपनी पसंदीदा भाषा में इस्तेमाल करें:
|
||||
|
||||
| Package | Install | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Installation (Rust)
|
||||
|
||||
### आवश्यकताएं
|
||||
|
||||
- **Rust 1.85.0+** — Install from [rustup.rs](https://rustup.rs)
|
||||
|
||||
### अपने प्रोजेक्ट में जोड़ें
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Description |
|
||||
| ------------------- | ---------------------------------------------- |
|
||||
| `lex` | Full-text search with BM25 ranking (Tantivy) |
|
||||
| `pdf_extract` | Pure Rust PDF text extraction |
|
||||
| `vec` | Vector similarity search (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings for image search |
|
||||
| `whisper` | Audio transcription with Whisper |
|
||||
| `temporal_track` | Natural language date parsing ("last Tuesday") |
|
||||
| `parallel_segments` | Multi-threaded ingestion |
|
||||
| `encryption` | Password-based encryption capsules (.mv2e) |
|
||||
|
||||
Enable features as needed:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## त्वरित प्रारंभ
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## निर्माण
|
||||
|
||||
रिपॉजिटरी को क्लोन करें:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
डीबग मोड में बिल्ड करें:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
रिलीज़ मोड में बिल्ड करें (ऑप्टिमाइज़्ड):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
विशिष्ट विशेषताओं के साथ बनाएं:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## टेस्ट चलाएँ
|
||||
|
||||
सभी टेस्ट चलाएँ:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
आउटपुट के साथ टेस्ट चलाएँ:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
एक विशिष्ट टेस्ट चलाएँ:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
केवल इंटीग्रेशन टेस्ट चलाएँ:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## उदाहरण
|
||||
|
||||
The `examples/` डायरेक्टरी में काम करने वाले उदाहरण हैं:
|
||||
|
||||
### बेसिक उपयोग
|
||||
|
||||
यह क्रिएट, पुट, सर्च और टाइमलाइन ऑपरेशन दिखाता है:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDF इन्जेक्शन
|
||||
|
||||
PDF डॉक्यूमेंट्स को इन्जेस्ट करें और सर्च करें ("अटेंशन इज़ ऑल यू नीड" पेपर का इस्तेमाल करता है):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP विज़ुअल सर्च
|
||||
|
||||
CLIP एम्बेडिंग का इस्तेमाल करके इमेज सर्च (इसके लिए `clip` फ़ीचर ज़रूरी है):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### व्हिस्पर ट्रांसक्रिप्शन
|
||||
|
||||
ऑडियो ट्रांसक्रिप्शन (`whisper` फीचर ज़रूरी है):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## फ़ाइल फ़ॉर्मेट
|
||||
|
||||
सब कुछ एक ही `.mv2` फ़ाइल में होता है:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
कोई `.wal`, `.lock`, `.shm`, या साइडकार फ़ाइल नहीं। कभी नहीं।
|
||||
|
||||
पूरे फ़ाइल फ़ॉर्मेट स्पेसिफ़िकेशन के लिए [MV2_SPEC.md](MV2_SPEC.md) देखें।
|
||||
|
||||
---
|
||||
|
||||
## सपोर्ट
|
||||
|
||||
क्या आपके कोई सवाल या फीडबैक हैं?
|
||||
Email: contact@memvid.com
|
||||
|
||||
**सपोर्ट दिखाने के लिए ⭐ दें**
|
||||
|
||||
---
|
||||
|
||||
## लाइसेंस
|
||||
|
||||
Apache License 2.0 — ज़्यादा जानकारी के लिए [LICENSE](LICENSE) फ़ाइल देखें।
|
||||
@@ -0,0 +1,429 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvidは、AIエージェントのための即時検索と長期記憶を備えた単一ファイル型メモリレイヤーです。</strong>
|
||||
</br>
|
||||
データベースを必要とせず、永続化、バージョン管理、ポータブル性を備えたメモリを実現します。
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ スターで応援お願いします ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Memvidとは？
|
||||
|
||||
Memvidは、データ、埋め込み、検索構造、メタデータを１つのファイルにパッケージ化するポータブルAIメモリシステムです。
|
||||
|
||||
複雑なRAGパイプラインやサーバーベースのベクトルデータベースを運用する代わりに、Memvidを使用することで直接ファイルから高速な検索が可能になります。
|
||||
|
||||
その結果、モデルに依存せずインフラ不要のメモリレイヤーが実現し、AIエージェントはどこでも使える永続的な長期記憶を持つことができます。
|
||||
|
||||
---
|
||||
|
||||
## スマートフレーム (Smart Frames) とは？
|
||||
|
||||
Memvidは、（ビデオを保存するためではなく）**追記に特化した効率的なスマートフレームのシーケンスとしてAIメモリを整理するため**に、ビデオエンコーディング技術から着想を得ています。
|
||||
|
||||
スマートフレームは、コンテンツをタイムスタンプ、チェックサム、基本メタデータとともに保存する不変（イミュータブル）な単位です。フレームは効率的な圧縮、インデックス作成、並列読み取りができるようグループ化されています。
|
||||
|
||||
このフレームベースの設計により、以下が可能になります。
|
||||
|
||||
- 既存のデータを変更したり破損したりすることなくデータを追加
|
||||
- 過去のメモリ状態に対するクエリ
|
||||
- 知識がどのように進化するかをタイムライン形式で検査
|
||||
- コミットされた不変フレームによるクラッシュ耐性
|
||||
- ビデオエンコーディング技術を応用した効率的な圧縮
|
||||
|
||||
その結果、AIシステムの「巻き戻し可能なメモリタイムライン」のように機能する単一のファイルが生成されます。
|
||||
|
||||
---
|
||||
|
||||
## コアコンセプト
|
||||
|
||||
- **成長するメモリエンジン (Living Memory Engine)**
|
||||
セッションをまたいでメモリを継続的に追加、分岐、進化させます。
|
||||
|
||||
- **カプセル・コンテキスト (`.mv2`)**
|
||||
ルールや有効期限を設定できる、自己完結型で共有可能なメモリカプセル。
|
||||
|
||||
- **タイムトラベル・デバッグ**
|
||||
任意のメモリ状態を巻き戻し、再現、または分岐させることができます。
|
||||
|
||||
- **スマート・リコール**
|
||||
予測キャッシングによる5ミリ秒未満のローカルメモリーアクセス。
|
||||
|
||||
- **コーデック・インテリジェンス**
|
||||
圧縮方式を自動選択し、時間の経過とともにアップグレードします。
|
||||
|
||||
---
|
||||
|
||||
## ユースケース
|
||||
|
||||
Memvidは、AIエージェントに永続的な記憶と高速な呼び出し機能を提供するポータブルでサーバーレスなメモリレイヤーです。モデルに依存せず、マルチモーダルに対応し、完全にオフラインで動作するため、実用的なアプリケーションで幅広く利用されています。
|
||||
|
||||
- 長期稼働AIエージェント
|
||||
- エンタープライズ向けナレッジベース
|
||||
- オフラインファーストAIシステム
|
||||
- コードベースの理解
|
||||
- カスタマーサポートエージェント
|
||||
- ワークフロー自動化
|
||||
- セールス・マーケティング支援
|
||||
- パーソナル・ナレッジ・アシスタント
|
||||
- 医療・法律・金融特化型エージェント
|
||||
- 監査・デバッグ可能なAIワークフロー
|
||||
- カスタムアプリケーション
|
||||
|
||||
---
|
||||
|
||||
## SDK と CLI
|
||||
|
||||
お好みの言語でMemvidを利用できます。
|
||||
|
||||
| パッケージ | インストール | リンク |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## インストール (Rust)
|
||||
|
||||
### 要件
|
||||
|
||||
- **Rust 1.85.0+** - [rustup.rs](https://rustup.rs) からインストールしてください。
|
||||
|
||||
### プロジェクトへの追加
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
|
||||
```
|
||||
|
||||
### 機能フラグ (Feature Flags)
|
||||
|
||||
| 機能 | 説明 |
|
||||
| ------------------- | -------------------------------------------------------------- |
|
||||
| `lex` | BM25ランキングによる全文検索 (Tantivy) |
|
||||
| `pdf_extract` | Pure RustによるPDFテキスト抽出 |
|
||||
| `vec` | ベクトル類似性検索 (HNSW + ONNXによるローカルテキスト埋め込み) |
|
||||
| `clip` | 画像検索用のCLIPビジュアル埋め込み |
|
||||
| `whisper` | Whisperによる音声文字起こし |
|
||||
| `temporal_track` | 自然言語による日付解析 (例: "last Tuesday") |
|
||||
| `parallel_segments` | マルチスレッドによるデータ取り込み |
|
||||
| `encryption` | パスワードベースの暗号化カプセル (.mv2e) |
|
||||
|
||||
以下のように、必要に応じて有効化してください。
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## クイックスタート
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// 新しいメモリファイルを作成
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// メタデータ付きでドキュメントを追加
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// 検索の実行
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ビルド
|
||||
|
||||
リポジトリをクローン：
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
デバッグモードでビルド：
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
リリースモードでビルド（最適化）：
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
特定の機能フラグ付きでビルド：
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## テストの実行
|
||||
|
||||
すべてのテストを実行：
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
標準出力でテストを実行：
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
特定のテストを実行：
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
統合テストのみを実行：
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## サンプル (Examples)
|
||||
|
||||
`examples/` ディレクトリには、実際に動作するサンプルコードが用意されています。
|
||||
|
||||
### 基本的な使い方 (Basic Usage)
|
||||
|
||||
作成 (create)、追加 (put)、検索 (search)、およびタイムライン操作のデモです。
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDFの取り込み (PDF Ingestion)
|
||||
|
||||
PDFドキュメントの取り込みと検索のサンプルです。（論文「Attention Is All You Need」を使用）
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIPによる画像検索 (CLIP Visual Search)
|
||||
|
||||
CLIP埋め込みを使用した画像検索のサンプルです。
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Whisperによる文字起こし (Whisper Transcription)
|
||||
|
||||
音声文字起こしのサンプルです。
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## テキスト埋め込みモデル
|
||||
|
||||
`vec` 機能は、ONNXモデルを使用したローカルでのテキスト埋め込みをサポートしています。利用前にモデルファイルを手動でダウンロードする必要があります。
|
||||
|
||||
### 推奨：BGE-small (デフォルト)
|
||||
|
||||
高速で効率的なBGE-smallモデル（384次元）をダウンロードします。
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# ONNXモデルのダウンロード
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# トークナイザーのダウンロード
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### モデル一覧
|
||||
|
||||
| モデル | 次元数 | サイズ | 最適な用途 |
|
||||
| ----------------------- | ------ | ------ | ------------------------ |
|
||||
| `bge-small-en-v1.5` | 384 | ~120MB | デフォルト（高速・軽量） |
|
||||
| `bge-base-en-v1.5` | 768 | ~420MB | より高い精度が必要な場合 |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530MB | 多目的なタスク |
|
||||
| `gte-large` | 1024 | ~1.3GB | 最高精度 |
|
||||
|
||||
### 他のモデル
|
||||
|
||||
**BGE-base** (768次元):
|
||||
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic** (768次元):
|
||||
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large** (1024次元):
|
||||
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### 使用例
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// デフォルトモデルを使用する場合 (BGE-small)
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// モデルを変更する場合
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
類似性の計算と検索ランキングを含む完全な例については、`examples/text_embedding.rs` を参照してください。
|
||||
|
||||
---
|
||||
|
||||
## ファイル構成
|
||||
|
||||
すべてが単一の `.mv2` ファイルに収められます。
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ ヘッダー (4KB) │ マジックナンバー、バージョン、容量
|
||||
├────────────────────────────┤
|
||||
│ 組み込みWAL (1-64MB) │ クラッシュリカバリ用
|
||||
├────────────────────────────┤
|
||||
│ データセグメント │ 圧縮されたフレーム
|
||||
├────────────────────────────┤
|
||||
│ 全文検索インデックス (Lex) │ Tantivy全文検索
|
||||
├────────────────────────────┤
|
||||
│ ベクトルインデックス (Vec) │ HNSWベクトル
|
||||
├────────────────────────────┤
|
||||
│ タイムインデックス │ 時系列順序
|
||||
├────────────────────────────┤
|
||||
│ TOC (フッター) │ セグメントオフセット
|
||||
└────────────────────────────┘
|
||||
|
||||
```
|
||||
|
||||
`.wal`、`.lock`、`.shm` などのサイドカーファイルは一切生成されません。
|
||||
|
||||
フォーマット仕様の詳細は [MV2_SPEC.md](MV2_SPEC.md) を参照してください。
|
||||
|
||||
---
|
||||
|
||||
## サポート
|
||||
|
||||
ご質問やフィードバックはこちらまでご連絡ください。
|
||||
メール: contact@memvid.com
|
||||
|
||||
**⭐でプロジェクトをサポートしてください。**
|
||||
|
||||
---
|
||||
|
||||
## ライセンス
|
||||
|
||||
Apache License 2.0 - 詳細は [LICENSE](LICENSE) ファイルをご覧ください。
|
||||
@@ -0,0 +1,424 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid는 AI 에이전트를 위한 단일 파일 메모리 레이어로, 인스턴스 검색 및 장기 메모리 기능을 제공합니다.</strong><br/>
|
||||
데이터 베이스 없이 지속적이고, 버전 관리가 용이하며 여러 어플리케이션에 자유로운 적용이 가능합니다.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ STAR로 이 프로젝트를 지원해주세요 ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Memvid란?
|
||||
|
||||
Memvid는 데이터, 임베딩, 검색 구조 및 메타데이터를 단일 파일로 패키징하는 이식 가능한 AI 메모리 시스템입니다.
|
||||
|
||||
복잡한 RAG 파이프라인이나 서버 기반 벡터 데이터베이스를 실행하는 대신, Memvid는 파일에서 직접 빠른 검색을 가능하게 합니다.
|
||||
|
||||
결과적으로 모델에 독립적이며 인프라 구조와는 독립적인 메모리 레이어로, AI 에이전트가 어디서나 휴대할 수 있는 지속적 장기 메모리를 제공합니다
|
||||
|
||||
---
|
||||
|
||||
## Smart Frames란?
|
||||
|
||||
Memvid는 **AI 메모리를 추가 전용(append-only)의 초고효율 Smart Frame 시퀀스로 구성하기 위해** 비디오 인코딩에서 영감을 받았습니다.
|
||||
|
||||
Smart Frame은 타임스탬프, 체크섬 및 기본 메타데이터와 함께 콘텐츠를 저장하는 불변 단위입니다.
|
||||
프레임은 효율적인 압축, 인덱싱 및 병렬 읽기를 허용하는 방식으로 그룹화됩니다.
|
||||
|
||||
이러한 프레임 기반 설계는 다음을 가능하게 합니다:
|
||||
|
||||
- 기존 데이터를 수정하거나 손상시키지 않는 추가 전용(append-only) 쓰기
|
||||
- 과거 메모리 상태에 대한 쿼리
|
||||
- 지식이 어떻게 변화하는지에 대한 타임라인 스타일 검사
|
||||
- 불변 프레임워크를 통한 크래시 안전성
|
||||
- 비디오 인코딩에서 차용한 기술을 사용한 효율적인 압축
|
||||
|
||||
이를 위한 결과물은 AI 시스템을 위한 되감기 가능한 메모리 타임라인처럼 동작하는 단일 파일입니다.
|
||||
|
||||
---
|
||||
|
||||
## 주요 개념
|
||||
|
||||
- **실시간 변화하는 메모리 엔진**
|
||||
세션 간에 메모리를 지속적으로 추가, 분기 및 변화시킵니다.
|
||||
|
||||
- **문맥 캡슐화 (`.mv2`)**
|
||||
규칙과 만료 시간이 포함된 자립형 공유 가능 형대의 메모리 캡슐입니다.
|
||||
|
||||
- **시간 기반 디버깅**
|
||||
임의의 메모리 상태로 되감기, 재생 또는 분기합니다.
|
||||
|
||||
- **예측 기반 호출**
|
||||
예측 캐싱을 사용한 5ms 미만 로컬 메모리 액세스를 제공합니다.
|
||||
|
||||
- **코덱 인텔리전스**
|
||||
시간 경과에 따라 압축을 자동 선택 및 업그레이드합니다.
|
||||
|
||||
---
|
||||
|
||||
## 이용 사례
|
||||
|
||||
Memvid 이동 가능한 서버리스 메모리 레이어로 AI 에이전트에 지속적인 메모리와 빠른 호출을 제공합니다. 이는 모델과 독립적이고, 멀티모달을 지원하며, 인터넷을 사용하지 않으므로, 개발자들은 다양한 실제 어플리케이션에서 Memvid를 활용하고 있습니다.
|
||||
|
||||
- 장기 실행 AI 에이전트
|
||||
- 기업 내의 지식 베이스
|
||||
- 오프라인 우선의 AI 시스템
|
||||
- 코드베이스 이해
|
||||
- 고객 지원 에이전트
|
||||
- 워크플로 자동화
|
||||
- 판매 및 마케팅 코파일럿
|
||||
- 개인 지식 어시스턴트
|
||||
- 의료, 법률 및 금융 에이전트
|
||||
- 모니터링 및 디버깅 가능한 AI 워크플로
|
||||
- 그 외의 여러 애플리케이션
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
원하는 언어로 Memvid를 사용하세요:
|
||||
|
||||
| 패키지 | 설치 커맨드 | 링크 |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## 설치 (Rust)
|
||||
|
||||
### 요구 사항
|
||||
|
||||
- **Rust 1.85.0+** — [rustup.rs](https://rustup.rs)에서 설치 가능합니다.
|
||||
|
||||
### 프로젝트에 추가
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Description |
|
||||
| ------------------- | --------------------------------------------------- |
|
||||
| `lex` | BM25 랭킹 기반 전체 텍스트 검색 (Tantivy) |
|
||||
| `pdf_extract` | Rust 기반 PDF 텍스트 추출 |
|
||||
| `vec` | 벡터 유사도 검색 (HNSW + ONNX) |
|
||||
| `clip` | 이미지 검색을 위한 CLIP 임베딩 |
|
||||
| `whisper` | Whisper 기반 오디오 전사 |
|
||||
| `temporal_track` | 자연어 날짜 추출 ("지난 화요일") |
|
||||
| `parallel_segments` | 멀티-스레딩 처리 |
|
||||
| `encryption` | Password 기반 암호화 (.mv2e) |
|
||||
|
||||
필요한 기능을 아래 방식으로 활성화하세요:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 빌드
|
||||
|
||||
이 레포지토리 클론:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
디버그 모드로 빌드:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
배포 모드로 빌드 (optimized):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
특수 기능을 포함하도록 빌드:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 테스트
|
||||
|
||||
전체 테스트 실행:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
테스트 실행 및 결과 출력:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
특정 테스트 실행:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
인테그레이션 테스트만 실행:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 예시
|
||||
|
||||
`examples/` 디렉토리에 예제가 있습니다:
|
||||
|
||||
### 기본 사용법
|
||||
|
||||
생성, 추가, 검색 및 타임라인 작업을 보여줍니다:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDF 수집
|
||||
|
||||
PDF 문서 수집 및 검색 ("Attention Is All You Need" 논문 사용):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP 이미지 검색
|
||||
|
||||
CLIP 임베딩을 사용한 이미지 검색 (`clip` 기능 필요):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Whisper 전사
|
||||
|
||||
오디오 전사 (`whisper` 기능 필요):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Text Embedding 모델
|
||||
|
||||
`vec` 기능은 ONNX 모델을 사용한 로컬 텍스트 임베딩을 포함합니다. 로컬 텍스트 임베딩을 사용하기 전에 모델 파일을 수동으로 다운로드해야 합니다.
|
||||
|
||||
### Quick Start: BGE-small (추천함)
|
||||
|
||||
기본 BGE-small 모델(384 차원, 빠르고 효율적) 다운로드:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# Download ONNX model
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# Download tokenizer
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### 지원 모델
|
||||
|
||||
| 모델명 | 차원 수 | 크기 | 권장 용도 |
|
||||
| ----------------------- | ---------- | ----- | --------------------- |
|
||||
| `bge-small-en-v1.5` | 384 | ~120MB | 기본 설정, 가장 빠름 |
|
||||
| `bge-base-en-v1.5` | 768 | ~420MB | 꽤 좋은 성능 |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530MB | 다양한 업무 가능 |
|
||||
| `gte-large` | 1024 | ~1.3GB | 가장 좋은 성능 |
|
||||
|
||||
### 타 모델
|
||||
|
||||
**BGE-base** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic** (768 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large** (1024 dimensions):
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### 코드 내 사용법
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// Use default model (BGE-small)
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// Use different model
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
유사도 계산 및 검색 랭킹이 포함된 전체 예제는 `examples/text_embedding.rs`를 참조하세요.
|
||||
|
||||
---
|
||||
|
||||
## 파일 구조
|
||||
|
||||
모든 구성 요소는 단일 `.mv2` 파일 내에 구성됩니다:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
`.wal`, `.lock`, `.shm`, 혹은 그 외의 별도 구성 요소는 없습니다.
|
||||
|
||||
[MV2_SPEC.md](MV2_SPEC.md)에서 파일 세부 형식을 확인할 수 있습니다.
|
||||
|
||||
---
|
||||
|
||||
## Support
|
||||
|
||||
문의 사항은 아래 이메일로 부탁드립니다.
|
||||
Email: contact@memvid.com
|
||||
|
||||
**⭐를 눌러 이 프로젝트를 지원해주세요**
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
Apache License 2.0 — [LICENSE](LICENSE) 파일 참고.
|
||||
@@ -0,0 +1,90 @@
|
||||
# Internationalization (i18n) - README Translations
|
||||
|
||||
This folder contains translated versions of the main [README.md](../../README.md) file for different languages.
|
||||
|
||||
## Purpose
|
||||
|
||||
The `docs/i18n/` directory is dedicated to making Memvid accessible to developers worldwide by providing localized versions of the main README. Each translation helps non-English speakers understand and use Memvid more effectively.
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
docs/i18n/
|
||||
├── README.md # This file
|
||||
├── README.zh-CN.md # Chinese (Simplified) translation
|
||||
├── README.zh-TW.md # Chinese (Traditional) translation
|
||||
├── README.es.md # Spanish translation
|
||||
├── README.fr.md # French translation
|
||||
├── README.de.md # German translation
|
||||
├── README.ja.md # Japanese translation
|
||||
├── README.ko.md # Korean translation
|
||||
├── README.pt-BR.md # Portuguese (Brazil) translation
|
||||
├── README.so.md # Somali translation
|
||||
└── ... # Additional languages
|
||||
```
|
||||
|
||||
## Contributing Translations
|
||||
|
||||
We welcome contributions of README translations! For detailed guidelines, see [Contributing Translations](CONTRIBUTING_TRANSLATIONS.md).
|
||||
|
||||
Here's a quick overview:
|
||||
|
||||
### 1. Check Existing Translations
|
||||
|
||||
Before starting, check if a translation for your language already exists or is in progress.
|
||||
|
||||
### 2. Create a Translation File
|
||||
|
||||
- Use the format: `README.{language-code}.md`
|
||||
- Use standard language codes (e.g., `zh-CN`, `es`, `fr`, `de`, `ja`, `ko`, `pt-BR`, `so`)
|
||||
- Copy the main `README.md` as a starting point
|
||||
|
||||
### 3. Translation Guidelines
|
||||
|
||||
- **Keep the structure**: Maintain the same headings, sections, and formatting as the original
|
||||
- **Preserve links**: Keep all URLs and links unchanged
|
||||
- **Preserve code blocks**: Keep code examples, commands, and technical terms in English (or add comments in the target language)
|
||||
- **Update badges**: Keep badges and shields as they are (they're language-agnostic)
|
||||
- **Maintain accuracy**: Ensure technical accuracy while making the content natural in the target language
|
||||
|
||||
### 4. Submit Your Translation
|
||||
|
||||
1. Create a new file: `README.{language-code}.md` in this directory
|
||||
2. Translate the content while following the guidelines above
|
||||
3. Submit a pull request with:
|
||||
- A clear description of the language being added
|
||||
- Your name/username for attribution (if desired)
|
||||
|
||||
## Language Codes
|
||||
|
||||
Use standard ISO 639-1 or ISO 639-2 language codes:
|
||||
|
||||
- `zh-CN` - Chinese (Simplified)
|
||||
- `zh-TW` - Chinese (Traditional)
|
||||
- `es` - Spanish
|
||||
- `fr` - French
|
||||
- `de` - German
|
||||
- `ja` - Japanese
|
||||
- `ko` - Korean
|
||||
- `pt-BR` - Portuguese (Brazil)
|
||||
- `ru` - Russian
|
||||
- `ar` - Arabic
|
||||
- `hi` - Hindi
|
||||
- `so` - Somali
|
||||
- And more...
|
||||
|
||||
## Maintenance
|
||||
|
||||
Translations should be updated when the main README is significantly changed. Contributors are encouraged to keep their translations in sync with the English version.
|
||||
|
||||
## Questions?
|
||||
|
||||
If you have questions about translations or want to coordinate with other translators, please:
|
||||
- See [Contributing Translations](CONTRIBUTING_TRANSLATIONS.md) for detailed guidelines
|
||||
- Open an issue on GitHub
|
||||
- Join our [Discussions](https://github.com/memvid/memvid/discussions)
|
||||
- Contact: contact@memvid.com
|
||||
|
||||
---
|
||||
|
||||
**Thank you for helping make Memvid accessible to developers worldwide! 🌍**
|
||||
@@ -0,0 +1,347 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid is een geheugenlaag van één bestand voor AI-agenten met directe toegang en langetermijnsgeheugen.</strong><br/>
|
||||
Volhardend en draagbaar geheugen met versiebeheer en zonder databases.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Laat een ster achter om het project te steunen ⭐️</h2>
|
||||
</p>
|
||||
|
||||
## Wat is Memvid?
|
||||
|
||||
Memvid is een draagbaar AI-geheugensysteem dat uw data, embedding, zoekstructuur en metadata in één bestand opslaat.
|
||||
|
||||
In plaats van complexe RAG pijplijnen of servergebaseerde vectordatabases te gebruiken, zal Memvid snelle toegang recht vanuit het bestand toestaan.
|
||||
|
||||
Het resultaat is een model-agnostische, infrastructuurvrije geheugenlaag die AI-agenten een volhardende langetermijnsgeheugen geeft, die ze overal kunnen meenemen.
|
||||
|
||||
---
|
||||
|
||||
## Waarom videoframes?
|
||||
|
||||
Memvid neemt inspiratie uit videos encoderen, niet om de video op te slaan, maar om **het organiseren van AI-geheugen als een ultra-efficiënte sequentie van Smart Frames waarbij je enkel kan toevoegen.**
|
||||
|
||||
Een Smart Frame is een immutabele eenheid die content opslaat samen met zijn tijdstempels, controlesommen en basismetadata.
|
||||
Frames worden gegroupeerd in een manier die voor efficiënte compressie, indexing en parallele lezingen zorgt.
|
||||
|
||||
Dit frame-gebaseerde design maakt het volgende mogelijk:
|
||||
|
||||
- Append-only bijschrijven van data zonder het aanpassen of corrumperen van bestaande data
|
||||
- Zoekopdrachten over vorige geheugenstaten
|
||||
- Tijdlijn-stijl inspectie van hoe kennis evolueert
|
||||
- Crashveiligheid door de vastgelegde immutabele frames
|
||||
- Efficiënte compressie gebruikmakend van technieken aangepast uit video encoderen
|
||||
|
||||
Het resultaat is één bestand dat werkt als een terugspoelbare geheugentijdslijn van AI-systemen.
|
||||
|
||||
---
|
||||
|
||||
## Basisconcepten
|
||||
|
||||
- **Living Memory Engine**
|
||||
Append, vertakt en evolueert geheugen continu over sessies.
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
Autonome, deelbaar geheugencapsules met regels en vervalling.
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
Spoel terug, herspeel, of vertak elke geheugenstatus.
|
||||
|
||||
- **Smart Recall**
|
||||
Sub-5ms lokale geheugentoegang met voorspelbare caching.
|
||||
|
||||
- **Codec Intelligence**
|
||||
Selecteert en verbetert automatisch de compressie doorheen de tijd.
|
||||
|
||||
---
|
||||
|
||||
## Gebruiksgevallen
|
||||
|
||||
Memvid is een draagbare, serverloze geheugenlaag dat AI-agenten een volhardend geheugen en snelle herroepingen geeft. Door zijn model-agnostische, multi-modale en het feit dat het volledig offline werkt, gebruiken ontwikkelaars het over een wijd scala aan real-world applicaties.
|
||||
|
||||
- Lang werkende AI-agenten
|
||||
- Kennisbanken voor ondernemingen
|
||||
- Offline-First AI-systemen
|
||||
- Codebase-begrip
|
||||
- Klantenondersteuningsagenten
|
||||
- Automatisering van de workflow
|
||||
- Verkoop- en marketingcopiloten
|
||||
- Persoonlijke Kennisassistenten
|
||||
- Medische, juridische en financiële adviseurs
|
||||
- Controleerbare en debugbare AI-workflows
|
||||
- Aangepaste toepassingen
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
Gebruik Memvid in je lievelingstaal:
|
||||
|
||||
| Pakket | Installatie | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Installatie (Rust)
|
||||
|
||||
### Benodigdheden
|
||||
|
||||
- **Rust 1.85.0+** — Installeer vanuit [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Voeg dit aan je project toe
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Beschrijving |
|
||||
| ------------------- | ---------------------------------------------- |
|
||||
| `lex` | Full-text search with BM25 ranking (Tantivy) |
|
||||
| `pdf_extract` | Pure Rust PDF text extraction |
|
||||
| `vec` | Vector similarity search (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings for image search |
|
||||
| `whisper` | Audio transcription with Whisper |
|
||||
| `temporal_track` | Natural language date parsing ("last Tuesday") |
|
||||
| `parallel_segments` | Multi-threaded ingestion |
|
||||
| `encryption` | Password-based encryption capsules (.mv2e) |
|
||||
|
||||
Schakel functies in indien nodig:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Build
|
||||
|
||||
Clone de repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
Build in debug modus:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Build in release modus (geoptimaliseerd):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Build with specific features:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tests uitvoeren
|
||||
|
||||
Voer alle tests uit:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Voer tests uit met uitvoer:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Voer een specifieke test uit:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Voer enkel integratie tests uit:
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Voorbeelden
|
||||
|
||||
De `examples/` map bedraagd werkende Voorbeelden:
|
||||
|
||||
### Basisgebruik
|
||||
|
||||
Beeldt create, put, search, and timeline operaties uit:
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDF Ingestion
|
||||
|
||||
PDF-documenten importeren en doorzoeken (gebruikt de "Attention Is All You Need" paper):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP Visual Search
|
||||
|
||||
Afbeeldingen zoeken met behulp van CLIP-integraties (gebruikt `clip` feature):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Whisper Transcription
|
||||
|
||||
Audio transcripties (gebruikt `whisper` feature):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Bestandsformaat
|
||||
|
||||
Alles leeft in één `.mv2` bestand:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, version, capacity
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Crash recovery
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Compressed frames
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy full-text
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Chronological ordering
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Segment offsets
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Geen `.wal`, `.lock`, `.shm`, of sidecar-bestanden. Ooit.
|
||||
|
||||
Zie [MV2_SPEC.md](MV2_SPEC.md) voor de complete bestandsformaat specificaties.
|
||||
|
||||
---
|
||||
|
||||
## Ondersteuning
|
||||
|
||||
Heb je vragen of feedback?
|
||||
Email: contact@memvid.com
|
||||
|
||||
**Laat een ⭐ om je ondersteuning te tonen**
|
||||
|
||||
---
|
||||
|
||||
## Licentie
|
||||
|
||||
Apache License 2.0 — zie het [LICENSE](LICENSE) bestandvoor details.
|
||||
@@ -0,0 +1,345 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)" src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<!-- FLAGS:START -->
|
||||
<p align="center">
|
||||
<a href="../../README.md">🇺🇸 English</a>
|
||||
<a href="README.es.md">🇪🇸 Español</a>
|
||||
<a href="README.fr.md">🇫🇷 Français</a>
|
||||
<a href="README.so.md">🇸🇴 Soomaali</a>
|
||||
<a href="README.ar.md">🇸🇦 العربية</a>
|
||||
<a href="README.nl.md">🇧🇪/🇳🇱 Nederlands</a>
|
||||
<a href="README.hi.md">🇮🇳 हिन्दी</a>
|
||||
<a href="README.bn.md">🇧🇩 বাংলা</a>
|
||||
<a href="README.cs.md">🇨🇿 Čeština</a>
|
||||
<a href="README.ko.md">🇰🇷 한국어</a>
|
||||
<a href="README.ja.md">🇯🇵 日本語</a>
|
||||
<!-- Next Flag -->
|
||||
</p>
|
||||
<!-- FLAGS:END -->
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">Website</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">Try Sandbox</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">Docs</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">Discussions</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid waa nidaam xusuuseed oo hal fayl ah kaas oo loogu talagalay wakiillada AI (AI agents), lehna soo-celin degdeg ah iyo xusuus fog.</strong><br/>
|
||||
Xusuus joogto ah, la raadin karo, lana qaadan karo, iyadoo aan loo baahnayn database-yo kale.
|
||||
</p>
|
||||
|
||||
<h2 align="center">⭐️ Noo saar STAR si aad mashruuca u taageerto ⭐️</h2>
|
||||
|
||||
## Waa maxay Memvid?
|
||||
|
||||
Memvid waa nidaam xusuuseed AI oo la qaadan karo kaas oo kuu keydinaya xogtaada, habka raadinta (embeddings), qaabdhismeedka iyo metadata-daba ku ururiya hal fayl oo keliya.
|
||||
|
||||
Halkii aad ka isticmaali lahayd nidaamyada RAG-ga ee adag ama database-yada vector-ka ee ku shaqeeya server-ka, Memvid wuxuu kuu oggolaanayaa inaad xogta si toos ah uga soo ceshato faylka dhexdiisa si aad u degdeg badan.
|
||||
|
||||
Natiijadu waa lakab xusuuseed ka madax-bannaan nooca modelka iyo kaabayaasha (infrastructure-free), kaas oo siiya wakiillada AI(AI agents) xusuus joogto ah oo fog oo ay meel walba u qaadan karaan.
|
||||
|
||||
---
|
||||
|
||||
## Maxay tahay sababta Frames-ka Muuqaalka (Video Frames)?
|
||||
|
||||
Memvid wuxuu dhiirrigelin ka helayaa habka xogta muuqaalka loo kaydiyo (video encoding), isaga oo aan kaydinayn muuqaal balse u **habaynaya xusuusta AI si isku-xiga (sequence) oo aad u hufan oo "Smart Frames".**
|
||||
|
||||
"Smart Frame" waa unug aan isbeddelayn oo kaydiya macluumaadka oo ay la socdaan waqtiga (timestamps), checksums iyo metadata aasaasi ah. Frames-ka waxaa loo ururiyaa qaab oggolaanaya isku-duubni (compression), tusmeyn (indexing), iyo akhris is-barbar-socda oo hufan.
|
||||
|
||||
Qaabdhismeedkan ku salaysan frames-ka wuxuu suuragelinayaa:
|
||||
|
||||
- Qoraal kaliya oo lagu darayo (Append-only) iyadoo aan la beddelayn ama la kharribayn xogta jirtay
|
||||
- Baaritaan lagu sameyn karo xaaladihii xusuusta ee hore
|
||||
- Kormeeridda habka ay aqoontu u kobcayso iyadoo loo eegayo waqtiga
|
||||
- Badbaadada xogta (crash safety) iyadoo la adeegsanayo frames go'an oo aan isbeddelayn
|
||||
- Isku-duubni hufan oo loo adeegsanayo farsamooyin laga soo minguuriyay kaydinta muuqaallada
|
||||
|
||||
Natiijadu waa hal fayl oo u dhaqmaya sidii jadwal xusuuseed oo dib loo celin karo oo loogu talagalay nidaamyada AI.
|
||||
|
||||
---
|
||||
|
||||
## Fikradaha Muhiimka ah
|
||||
|
||||
- **Living Memory Engine**
|
||||
Si joogto ah ugu dar, u laameey (branch), una kobci xusuusta qeybo kala duwan.
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
Capsule xusuuseed oo isku-filan, la wadaagi karo, lehna sharciyo iyo waqti dhicitaan.
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
Dib u celi, ama qabeey xaalad kasta oo xusuusta ah.
|
||||
|
||||
- **Smart Recall**
|
||||
Soo-celinta xusuusta gudaha wax ka yar 5ms iyadoo la adeegsanayo kaydinta saadaalinta (predictive caching).
|
||||
|
||||
- **Codec Intelligence**
|
||||
Si otomaatig ah u doorta una casriyeeya isku-duubnida (compression) waqtiga ka dib.
|
||||
|
||||
---
|
||||
|
||||
## Meelaha loo adeegsado (Use Cases)
|
||||
|
||||
Memvid waa nidaam xusuuseed oo la qaadan karo oo aan server u baahnayn, kaas oo siiya wakiillada AI(AI agents), xusuus joogto ah iyo soo-celin degdeg ah. Maadaama uu ka madax-bannaan yahay modelka, waxna ku akhriyo qaabab badan (multi-modal), una shaqeeyo si buuxda isagoo aan internet lahayn, horumariyayaashu waxay Memvid u adeegsanayaan hawlo badan:
|
||||
|
||||
- Wakiillada AI(AI Agents) ee muddada dheer shaqeeya
|
||||
- Keydka aqoonta ee shirkadaha
|
||||
- Tageerida AI-ga ee ku shaqeeya offline-ka
|
||||
- Fahamka nidaamyada koodhka (Codebase)
|
||||
- Wakiillada adeegga macmiilka
|
||||
- Otomaatigga shaqada (Workflow Automation)
|
||||
- Kaaliyayaasha iibka iyo suuqgeynta
|
||||
- Kaaliyayaasha aqoonta shakhsi ahaaneed
|
||||
- Wakiillada caafimaadka, sharciga, iyo maaliyadda
|
||||
- Hannaanka shaqada AI-ga oo la baari karo lana saxi karo (Auditable/Debuggable)
|
||||
- Codsiyada gaarka ah (Custom Applications)
|
||||
|
||||
---
|
||||
|
||||
## SDKs & CLI
|
||||
|
||||
Ku isticmaal Memvid luuqadda aad doorbidi lahayd:
|
||||
|
||||
| Package | Install | Links |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## Kushubashada (Installation) (Rust)
|
||||
|
||||
### Shuruudaha
|
||||
|
||||
- **Rust 1.85.0+** — Si aad ugu shubato Guji linkagan [rustup.rs](https://rustup.rs)
|
||||
|
||||
### Ku dar Mashruucaaga
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
| Feature | Sharaxada |
|
||||
| ------------------- | ---------------------------------------------- |
|
||||
| `lex` | Raadinta qoraalka oo dhan oo leh darajada BM25 (Tantivy) |
|
||||
| `pdf_extract` | Soo saarista qoraalka PDF oo saafi ah |
|
||||
| `vec` | Raadinta isku-midka ah ee Vector (HNSW + ONNX) |
|
||||
| `clip` | CLIP visual embeddings oo loogu talagalay raadinta sawirka |
|
||||
| `whisper` | Beddelka codka iyadoo loo baddelayo qoraal lana adeegsanayo Whisper |
|
||||
| `temporal_track` | Turjumidda taariikhda ee luuqadda caadiga ah ("Salaasadii hore") |
|
||||
| `parallel_segments` | Soo gelinta xogta iyadoo la adeegsanayo dhowr nuuc (Multi-threaded) |
|
||||
| `encryption` | capsules-ka xusuusta ee ku xidhan sirta (password) (.mv2e) |
|
||||
|
||||
U furo sifooyinka (features) sida aad ugu baahan tahay:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Bilow Degdeg ah (Quick Start)
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// Create a new memory file
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// Add documents with metadata
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// Search
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Dhisid (Build)
|
||||
|
||||
Qeyb ka soo qaado (Clone) kaydka (Repository):
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
U dhis habka cilad-baadhista (debug mode):
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
U dhis habka rasmiga ah (release mode - la hagaajiyay):
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Ku dhis sifooyin (features) gaar ah:
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tijaabi iskudayga (Run Tests)
|
||||
|
||||
Tijaabi iskudayada oo dhan:
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
Tijaabi iskudayga iyadoo natiijada la arkayo:
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
Tijaab iskuday gaar ah:
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
Tijaabi iskudayada isku-xirka (integration tests) ah oo keliya
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tusaalooyin (Examples)
|
||||
|
||||
Tusaha `examples/` wuxuu ka kooban yahay tusaalooyin shaqaynaya:
|
||||
|
||||
### Adeegsiga Fudud
|
||||
|
||||
Wuxuu muujinayaa samaynta, gelinta, raadinta, iyo hawlgallada waqtiga (timeline):
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### Soo gelinta PDF
|
||||
|
||||
Geli oo baadh dukumiintiyada PDF-ka ah (wuxuu isticmaalaa warqaddii aheyd "Attention Is All You Need"):
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### Raadinta Muuqaalka ee CLIP
|
||||
|
||||
Raadinta sawirka iyadoo la adeegsanayo CLIP (waxay u baahan tahay clip feature):
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Beddelka Codka ee Whisper
|
||||
|
||||
Beddelka codka (waxay u baahan tahay whisper feature):
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Qaabka Faylka (File Format)
|
||||
|
||||
Wax walba waxay ku jiraan hal fayl oo .mv2 ah:
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ Magic, nooca, awoodda
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ Kasoo kabashada burburka
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ Frames la isku-duubay
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy qoraal ka buuxo
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW vectors
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ Siday u kala horreeyaan
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ Meeqaamka qaybaha
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
Ma jiraan faylal .wal, .lock, .shm, ama faylal dhinac socda. Weligaa.
|
||||
|
||||
Fiiri [MV2_SPEC.md](MV2_SPEC.md) si aad u hesho faahfaahinta dhammaystiran ee qaabka faylka.
|
||||
|
||||
---
|
||||
|
||||
## Taageer (Support)
|
||||
|
||||
Ma qabtaa su'aalo ama ra'yi?
|
||||
Email: contact@memvid.com
|
||||
|
||||
**Noo saar ⭐ si aad u muujiso taageeradaada**
|
||||
|
||||
---
|
||||
|
||||
## Shatiga (License)
|
||||
|
||||
Apache License 2.0 — Fiiri faylka [LICENSE](LICENSE) si aad u hesho faahfaahin dheeraad ah.
|
||||
@@ -0,0 +1,517 @@
|
||||
<!-- HEADER:START -->
|
||||
<img width="2000" height="524" alt="Social Cover (9)"
|
||||
src="https://github.com/user-attachments/assets/cf66f045-c8be-494b-b696-b8d7e4fb709c" />
|
||||
<!-- HEADER:END -->
|
||||
|
||||
<div style="height: 16px;"></div>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/17293" target="_blank"><img src="https://trendshift.io/api/badge/repositories/17293" alt="memvid%2Fmemvid | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
<!-- BADGES:END -->
|
||||
|
||||
<p align="center">
|
||||
<strong>Memvid 是专为 AI 智能体设计的单文件记忆层，具备即时检索和长期记忆能力。</strong><br/>
|
||||
持久化、版本化、可移植的记忆，无需数据库。
|
||||
</p>
|
||||
|
||||
<!-- NAV:START -->
|
||||
<p align="center">
|
||||
<a href="https://www.memvid.com">官方网站</a>
|
||||
·
|
||||
<a href="https://sandbox.memvid.com">尝试一下沙箱</a>
|
||||
·
|
||||
<a href="https://docs.memvid.com">文档</a>
|
||||
·
|
||||
<a href="https://github.com/memvid/memvid/discussions">讨论区</a>
|
||||
</p>
|
||||
<!-- NAV:END -->
|
||||
|
||||
<!-- BADGES:START -->
|
||||
<p align="center">
|
||||
<a href="https://crates.io/crates/memvid-core"><img src="https://img.shields.io/crates/v/memvid-core?style=flat-square&logo=rust" alt="Crates.io" /></a>
|
||||
<a href="https://docs.rs/memvid-core"><img src="https://img.shields.io/docsrs/memvid-core?style=flat-square&logo=docs.rs" alt="docs.rs" /></a>
|
||||
<a href="https://github.com/memvid/memvid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="许可证" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/memvid/memvid/stargazers"><img src="https://img.shields.io/github/stars/memvid/memvid?style=flat-square&logo=github" alt="Stars" /></a>
|
||||
<a href="https://github.com/memvid/memvid/network/members"><img src="https://img.shields.io/github/forks/memvid/memvid?style=flat-square&logo=github" alt="Forks" /></a>
|
||||
<a href="https://github.com/memvid/memvid/issues"><img src="https://img.shields.io/github/issues/memvid/memvid?style=flat-square&logo=github" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/2mynS7fcK7"><img src="https://img.shields.io/discord/1442910055233224745?style=flat-square&logo=discord&label=discord" alt="Discord" /></a>
|
||||
</p>
|
||||
|
||||
|
||||
|
||||
<h2 align="center">⭐️ 给项目点个星标支持我们 ⭐️</h2>
|
||||
|
||||
## 基准测试亮点
|
||||
|
||||
**🚀 准确率超越其他记忆系统：** 在 LoCoMo 上领先 SOTA 35%，长时对话回忆与推理能力最佳
|
||||
|
||||
**🧠 卓越的多跳与时序推理：** 比行业平均水平高出 76% 多跳推理，56% 时序推理
|
||||
|
||||
**⚡ 超低延迟高吞吐：** P50 仅 0.025ms，P99 仅 0.075ms，吞吐量是标准的 1,372 倍
|
||||
|
||||
**🔬 完全可复现的基准测试：** LoCoMo（10 次约 26K token 的对话）、开源评估、LLM-as-Judge
|
||||
|
||||
|
||||
## 什么是 Memvid？
|
||||
|
||||
Memvid 是可移植的 AI 记忆系统，将数据、嵌入向量、搜索结构和元数据打包成单个文件。
|
||||
|
||||
无需运行复杂的 RAG 管道或基于服务器的向量数据库，Memvid 支持直接从文件进行快速检索。
|
||||
|
||||
结果是模型无关、无基础设施的记忆层，让 AI 智能体拥有可随身携带的持久化长期记忆。
|
||||
|
||||
|
||||
## 什么是 Smart Frames？
|
||||
|
||||
Memvid 借鉴视频编码的理念，不是为了存储视频，而是**将 AI 记忆组织为仅追加、超高效序列的 Smart Frames。**
|
||||
|
||||
Smart Frame 是存储内容以及时间戳、校验和和基本元数据的不可变单元。
|
||||
帧以允许高效压缩、索引和并行读取的方式分组。
|
||||
|
||||
这种基于帧的设计支持：
|
||||
|
||||
- 仅追加写入，不修改或破坏现有数据
|
||||
- 对过去记忆状态的查询
|
||||
- 知识演化的时间轴式检查
|
||||
- 通过基于提交的不可变帧应对崩溃
|
||||
- 使用基于视频编码技术的高效压缩
|
||||
|
||||
结果是一个表现为 AI 系统可追溯记忆时间线的单文件。
|
||||
|
||||
|
||||
## 核心概念
|
||||
|
||||
- **Living Memory Engine**
|
||||
持续追加、分支和跨会话演进记忆。
|
||||
|
||||
- **Capsule Context (`.mv2`)**
|
||||
自包含、可共享的记忆胶囊，带规则和过期时间。
|
||||
|
||||
- **Time-Travel Debugging**
|
||||
回溯、重放或分支化任何记忆状态。
|
||||
|
||||
- **Smart Recall**
|
||||
小于 5ms 本地记忆访问，具备预测性缓存。
|
||||
|
||||
- **Codec Intelligence**
|
||||
随时间自动选择和升级压缩。
|
||||
|
||||
|
||||
## 使用场景
|
||||
|
||||
Memvid 是无服务器便携记忆层，为 AI 智能体提供持久记忆和快速召回。由于它是模型无关、多模态且完全离线工作，开发者正在各种现实应用中广泛的使用 Memvid。
|
||||
|
||||
- 长期运行的 AI 智能体
|
||||
- 企业知识库
|
||||
- 离线优先 AI 系统
|
||||
- 代码库理解
|
||||
- 客户支持智能体
|
||||
- 工作流自动化
|
||||
- 销售与营销助手
|
||||
- 个人知识助理
|
||||
- 医疗、法律和金融智能体
|
||||
- 可审计和可调试的 AI 工作流
|
||||
- 自定义应用
|
||||
|
||||
|
||||
## SDKs 与 CLI
|
||||
|
||||
在您喜欢的语言中使用 Memvid：
|
||||
|
||||
| 包 | 安装 | 链接 |
|
||||
| --------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------- |
|
||||
| **CLI** | `npm install -g memvid-cli` | [](https://www.npmjs.com/package/memvid-cli) |
|
||||
| **Node.js SDK** | `npm install @memvid/sdk` | [](https://www.npmjs.com/package/@memvid/sdk) |
|
||||
| **Python SDK** | `pip install memvid-sdk` | [](https://pypi.org/project/memvid-sdk/) |
|
||||
| **Rust** | `cargo add memvid-core` | [](https://crates.io/crates/memvid-core) |
|
||||
|
||||
---
|
||||
|
||||
## 安装（Rust）
|
||||
|
||||
### 要求
|
||||
|
||||
- **Rust 1.85.0+** — 从 [rustup.rs](https://rustup.rs) 安装
|
||||
|
||||
### 添加到项目
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = "2.0"
|
||||
```
|
||||
|
||||
### 功能标志
|
||||
|
||||
| 功能 | 描述 |
|
||||
| -------------------- | ---------------------------------------------------------------- |
|
||||
| `lex` | 使用 BM25 排序的全文搜索（Tantivy） |
|
||||
| `pdf_extract` | 纯 Rust PDF 文本提取 |
|
||||
| `vec` | 向量相似搜索（HNSW + 通过 ONNX 的本地文本嵌入） |
|
||||
| `clip` | CLIP 视觉嵌入用于图像搜索 |
|
||||
| `whisper` | 使用 Whisper 进行音频转录 |
|
||||
| `api_embed` | 云 API 嵌入（OpenAI） |
|
||||
| `temporal_track` | 自然语言日期解析（"last Tuesday"） |
|
||||
| `parallel_segments` | 多线程摄取 |
|
||||
| `encryption` | 基于密码的加密胶囊（.mv2e） |
|
||||
| `symspell_cleanup` | 强大的 PDF 文本修复（修复 "emp lo yee" -> "employee"） |
|
||||
|
||||
按需启用功能：
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
memvid-core = { version = "2.0", features = ["lex", "vec", "temporal_track"] }
|
||||
```
|
||||
|
||||
|
||||
## 快速开始
|
||||
|
||||
```rust
|
||||
use memvid_core::{Memvid, PutOptions, SearchRequest};
|
||||
|
||||
fn main() -> memvid_core::Result<()> {
|
||||
// 创建新记忆文件
|
||||
let mut mem = Memvid::create("knowledge.mv2")?;
|
||||
|
||||
// 添加带元数据的文档
|
||||
let opts = PutOptions::builder()
|
||||
.title("Meeting Notes")
|
||||
.uri("mv2://meetings/2024-01-15")
|
||||
.tag("project", "alpha")
|
||||
.build();
|
||||
mem.put_bytes_with_options(b"Q4 planning discussion...", opts)?;
|
||||
mem.commit()?;
|
||||
|
||||
// 搜索
|
||||
let response = mem.search(SearchRequest {
|
||||
query: "planning".into(),
|
||||
top_k: 10,
|
||||
snippet_chars: 200,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
for hit in response.hits {
|
||||
println!("{}: {}", hit.title.unwrap_or_default(), hit.text);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 构建
|
||||
|
||||
克隆仓库：
|
||||
|
||||
```bash
|
||||
git clone https://github.com/memvid/memvid.git
|
||||
cd memvid
|
||||
```
|
||||
|
||||
以调试模式构建：
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
以发布模式构建（优化）：
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
使用特定功能构建：
|
||||
|
||||
```bash
|
||||
cargo build --release --features "lex,vec,temporal_track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 运行测试
|
||||
|
||||
运行所有测试：
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
```
|
||||
|
||||
带输出运行测试：
|
||||
|
||||
```bash
|
||||
cargo test -- --nocapture
|
||||
```
|
||||
|
||||
运行特定测试：
|
||||
|
||||
```bash
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
仅运行集成测试：
|
||||
|
||||
```bash
|
||||
cargo test --test lifecycle
|
||||
cargo test --test search
|
||||
cargo test --test mutation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 示例
|
||||
|
||||
`examples/` 目录包含可运行示例：
|
||||
|
||||
### 基本用法
|
||||
|
||||
演示创建、添加、搜索和时间线操作：
|
||||
|
||||
```bash
|
||||
cargo run --example basic_usage
|
||||
```
|
||||
|
||||
### PDF 提取
|
||||
|
||||
提取和搜索 PDF 文档（使用 "Attention Is All You Need" 论文）：
|
||||
|
||||
```bash
|
||||
cargo run --example pdf_ingestion
|
||||
```
|
||||
|
||||
### CLIP 可视化搜索
|
||||
|
||||
使用 CLIP 嵌入进行图像搜索（需要 `clip` 功能）：
|
||||
|
||||
```bash
|
||||
cargo run --example clip_visual_search --features clip
|
||||
```
|
||||
|
||||
### Whisper 转录
|
||||
|
||||
音频转录（需要 `whisper` 功能）：
|
||||
|
||||
```bash
|
||||
cargo run --example test_whisper --features whisper -- /path/to/audio.mp3
|
||||
```
|
||||
|
||||
**可用模型：**
|
||||
|
||||
| 模型 | 大小 | 速度 | 用例 |
|
||||
| ---------------------- | ------ | ------- | ----------------------------------- |
|
||||
| `whisper-small-en` | 244 MB | 最慢 | 最佳准确度（默认） |
|
||||
| `whisper-tiny-en` | 75 MB | 快 | 平衡 |
|
||||
| `whisper-tiny-en-q8k` | 19 MB | 最快 | 快速测试，资源受限 |
|
||||
|
||||
**模型选择：**
|
||||
|
||||
```bash
|
||||
# 默认（FP32 small，最高准确度）
|
||||
cargo run --example test_whisper --features whisper -- audio.mp3
|
||||
|
||||
# 小型量化（小 75%，更快）
|
||||
MEMVID_WHISPER_MODEL=whisper-tiny-en-q8k cargo run --example test_whisper --features whisper -- audio.mp3
|
||||
```
|
||||
|
||||
**可编程配置：**
|
||||
|
||||
```rust
|
||||
use memvid_core::{WhisperConfig, WhisperTranscriber};
|
||||
|
||||
// 默认 FP32 small 模型
|
||||
let config = WhisperConfig::default();
|
||||
|
||||
// 小型量化模型（更快，更小）
|
||||
let config = WhisperConfig::with_quantization();
|
||||
|
||||
// 特定模型
|
||||
let config = WhisperConfig::with_model("whisper-tiny-en-q8k");
|
||||
|
||||
let transcriber = WhisperTranscriber::new(&config)?;
|
||||
let result = transcriber.transcribe_file("audio.mp3")?;
|
||||
println!("{}", result.text);
|
||||
```
|
||||
|
||||
|
||||
## 文本嵌入模型
|
||||
|
||||
`vec` 功能包括使用 ONNX 模型的本地文本嵌入支持。在使用本地文本嵌入之前，您需要手动下载模型文件。
|
||||
|
||||
### 快速开始：BGE-small（推荐）
|
||||
|
||||
下载默认 BGE-small 模型（384 维，快速高效）：
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cache/memvid/text-models
|
||||
|
||||
# 下载 ONNX 模型
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5.onnx
|
||||
|
||||
# 下载分词器
|
||||
curl -L 'https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-small-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
### 可用模型
|
||||
|
||||
| 模型 | 维度 | 大小 | 最适合 |
|
||||
| ------------------------ | ---------- | ------ | --------------- |
|
||||
| `bge-small-en-v1.5` | 384 | ~120MB | 默认，快速 |
|
||||
| `bge-base-en-v1.5` | 768 | ~420MB | 更好的质量 |
|
||||
| `nomic-embed-text-v1.5` | 768 | ~530MB | 多用途任务 |
|
||||
| `gte-large` | 1024 | ~1.3GB | 最高质量 |
|
||||
|
||||
### 其他模型
|
||||
|
||||
**BGE-base**（768 维）：
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/bge-base-en-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**Nomic**（768 维）：
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5.onnx
|
||||
curl -L 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/nomic-embed-text-v1.5_tokenizer.json
|
||||
```
|
||||
|
||||
**GTE-large**（1024 维）：
|
||||
```bash
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/onnx/model.onnx' \
|
||||
-o ~/.cache/memvid/text-models/gte-large.onnx
|
||||
curl -L 'https://huggingface.co/thenlper/gte-large/resolve/main/tokenizer.json' \
|
||||
-o ~/.cache/memvid/text-models/gte-large_tokenizer.json
|
||||
```
|
||||
|
||||
### 在代码中使用
|
||||
|
||||
```rust
|
||||
use memvid_core::text_embed::{LocalTextEmbedder, TextEmbedConfig};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// 使用默认模型（BGE-small）
|
||||
let config = TextEmbedConfig::default();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 384);
|
||||
|
||||
// 使用不同模型
|
||||
let config = TextEmbedConfig::bge_base();
|
||||
let embedder = LocalTextEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
有关相似度计算和搜索排名的完整示例，请参见 `examples/text_embedding.rs`。
|
||||
|
||||
### 模型一致性
|
||||
|
||||
为防止意外地模型混合（例如，使用 OpenAI 嵌入查询 BGE-small 索引），您可以将 Memvid 实例显式绑定到特定模型名称：
|
||||
|
||||
```rust
|
||||
// 将索引绑定到特定模型。
|
||||
// 如果之前使用不同模型创建索引将返回错误。
|
||||
mem.set_vec_model("bge-small-en-v1.5")?;
|
||||
```
|
||||
|
||||
绑定是持久化的。一旦设置，将来尝试使用不同模型名称将快速失败并返回 `ModelMismatch` 错误。
|
||||
|
||||
|
||||
|
||||
## API 嵌入（OpenAI）
|
||||
|
||||
`api_embed` 功能使用 OpenAI 的 API 启用基于云的嵌入生成。
|
||||
|
||||
### 设置
|
||||
|
||||
设置您的 OpenAI API 密钥：
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY="sk-..."
|
||||
```
|
||||
|
||||
### 用法
|
||||
|
||||
```rust
|
||||
use memvid_core::api_embed::{OpenAIConfig, OpenAIEmbedder};
|
||||
use memvid_core::types::embedding::EmbeddingProvider;
|
||||
|
||||
// 使用默认模型（text-embedding-3-small）
|
||||
let config = OpenAIConfig::default();
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
|
||||
let embedding = embedder.embed_text("hello world")?;
|
||||
assert_eq!(embedding.len(), 1536);
|
||||
|
||||
// 使用更高质量模型
|
||||
let config = OpenAIConfig::large(); // text-embedding-3-large (3072 维)
|
||||
let embedder = OpenAIEmbedder::new(config)?;
|
||||
```
|
||||
|
||||
### 可用模型
|
||||
|
||||
| 模型 | 维度 | 最适合 |
|
||||
| ------------------------ | ---------- | -------------------------- |
|
||||
| `text-embedding-3-small` | 1536 | 默认，最快，最便宜 |
|
||||
| `text-embedding-3-large` | 3072 | 最高质量 |
|
||||
| `text-embedding-ada-002` | 1536 | 传统模型 |
|
||||
|
||||
有关完整示例，请参见 `examples/openai_embedding.rs`。
|
||||
|
||||
|
||||
|
||||
## 文件格式
|
||||
|
||||
所有内容都存储在单个 `.mv2` 文件中：
|
||||
|
||||
```
|
||||
┌────────────────────────────┐
|
||||
│ Header (4KB) │ 魔数，版本，容量
|
||||
├────────────────────────────┤
|
||||
│ Embedded WAL (1-64MB) │ 崩溃恢复
|
||||
├────────────────────────────┤
|
||||
│ Data Segments │ 压缩帧
|
||||
├────────────────────────────┤
|
||||
│ Lex Index │ Tantivy 全文
|
||||
├────────────────────────────┤
|
||||
│ Vec Index │ HNSW 向量
|
||||
├────────────────────────────┤
|
||||
│ Time Index │ 时序排序
|
||||
├────────────────────────────┤
|
||||
│ TOC (Footer) │ 段偏移
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
不会有 `.wal`、`.lock`、`.shm` 或附带文件。永远不会。
|
||||
|
||||
有关完整文件格式规范，请参见 [MV2_SPEC.md](MV2_SPEC.md)。
|
||||
|
||||
|
||||
|
||||
## 支持
|
||||
|
||||
有问题或反馈？
|
||||
邮箱：contact@memvid.com
|
||||
|
||||
**点个 ⭐ 支持我们**
|
||||
|
||||
---
|
||||
|
||||
> **Memvid v1（基于 QR 码的记忆）已弃用**
|
||||
>
|
||||
> 如果您参考的是 QR 码，那么您正在使用过时信息。
|
||||
>
|
||||
> 参见：https://docs.memvid.com/memvid-v1-deprecation
|
||||
|
||||
---
|
||||
|
||||
## 许可证
|
||||
|
||||
Apache License 2.0 — 详细信息请参见 [LICENSE](LICENSE) 文件。
|
||||
@@ -0,0 +1,249 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const I18N_DIR = path.join(__dirname, '..');
|
||||
const ROOT_DIR = path.join(I18N_DIR, '..', '..');
|
||||
const README_PATH = path.join(ROOT_DIR, 'README.md');
|
||||
|
||||
const LANG_MAP = {
|
||||
'aa': { emoji: '🌐', name: 'Afar' },
|
||||
'ab': { emoji: '🌐', name: 'Abkhazian' },
|
||||
'ae': { emoji: '🌐', name: 'Avestan' },
|
||||
'af': { emoji: '🇿🇦', name: 'Afrikaans' },
|
||||
'ak': { emoji: '🌐', name: 'Akan' },
|
||||
'am': { emoji: '🇪🇹', name: 'Amharic' },
|
||||
'an': { emoji: '🌐', name: 'Aragonese' },
|
||||
'ar': { emoji: '🇸🇦', name: 'العربية' },
|
||||
'as': { emoji: '🌐', name: 'Assamese' },
|
||||
'av': { emoji: '🌐', name: 'Avaric' },
|
||||
'ay': { emoji: '🌐', name: 'Aymara' },
|
||||
'az': { emoji: '🇦🇿', name: 'Azerbaijani' },
|
||||
'ba': { emoji: '🌐', name: 'Bashkir' },
|
||||
'be': { emoji: '🇧🇾', name: 'Belarusian' },
|
||||
'bg': { emoji: '🇧🇬', name: 'Bulgarian' },
|
||||
'bi': { emoji: '🌐', name: 'Bislama' },
|
||||
'bm': { emoji: '🌐', name: 'Bambara' },
|
||||
'bn': { emoji: '🇧🇩', name: 'বাংলা' },
|
||||
'bo': { emoji: '🌐', name: 'Tibetan' },
|
||||
'br': { emoji: '🌐', name: 'Breton' },
|
||||
'bs': { emoji: '🇧🇦', name: 'Bosnian' },
|
||||
'ca': { emoji: '🇪🇸', name: 'Catalan' },
|
||||
'ce': { emoji: '🌐', name: 'Chechen' },
|
||||
'ch': { emoji: '🌐', name: 'Chamorro' },
|
||||
'co': { emoji: '🌐', name: 'Corsican' },
|
||||
'cr': { emoji: '🌐', name: 'Cree' },
|
||||
'cs': { emoji: '🇨🇿', name: 'Česko' },
|
||||
'cu': { emoji: '🌐', name: 'Church Slavonic' },
|
||||
'cv': { emoji: '🌐', name: 'Chuvash' },
|
||||
'cy': { emoji: '🇬🇧', name: 'Welsh' },
|
||||
'da': { emoji: '🇩🇰', name: 'Danish' },
|
||||
'de': { emoji: '🇩🇪', name: 'Deutsch' },
|
||||
'dv': { emoji: '🌐', name: 'Divehi' },
|
||||
'dz': { emoji: '🇧🇹', name: 'Dzongkha' },
|
||||
'ee': { emoji: '🌐', name: 'Ewe' },
|
||||
'el': { emoji: '🇬🇷', name: 'Greek' },
|
||||
'en': { emoji: '🇺🇸', name: 'English' },
|
||||
'eo': { emoji: '🌐', name: 'Esperanto' },
|
||||
'es': { emoji: '🇪🇸', name: 'Español' },
|
||||
'et': { emoji: '🇪🇪', name: 'Estonian' },
|
||||
'eu': { emoji: '🌐', name: 'Basque' },
|
||||
'fa': { emoji: '🇮🇷', name: 'Persian' },
|
||||
'ff': { emoji: '🌐', name: 'Fulah' },
|
||||
'fi': { emoji: '🇫🇮', name: 'Finnish' },
|
||||
'fj': { emoji: '🌐', name: 'Fijian' },
|
||||
'fo': { emoji: '🌐', name: 'Faroese' },
|
||||
'fr': { emoji: '🇫🇷', name: 'Français' },
|
||||
'fy': { emoji: '🌐', name: 'Western Frisian' },
|
||||
'ga': { emoji: '🇮🇪', name: 'Irish' },
|
||||
'gd': { emoji: '🌐', name: 'Gaelic' },
|
||||
'gl': { emoji: '🌐', name: 'Galician' },
|
||||
'gn': { emoji: '🌐', name: 'Guarani' },
|
||||
'gu': { emoji: '🌐', name: 'Gujarati' },
|
||||
'gv': { emoji: '🌐', name: 'Manx' },
|
||||
'ha': { emoji: '🇳🇬', name: 'Hausa' },
|
||||
'he': { emoji: '🇮🇱', name: 'Hebrew' },
|
||||
'hi': { emoji: '🇮🇳', name: 'हिन्दी' },
|
||||
'ho': { emoji: '🌐', name: 'Hiri Motu' },
|
||||
'hr': { emoji: '🇭🇷', name: 'Croatian' },
|
||||
'ht': { emoji: '🌐', name: 'Haitian' },
|
||||
'hu': { emoji: '🇭🇺', name: 'Hungarian' },
|
||||
'hy': { emoji: '🇦🇲', name: 'Armenian' },
|
||||
'hz': { emoji: '🌐', name: 'Herero' },
|
||||
'ia': { emoji: '🌐', name: 'Interlingua' },
|
||||
'id': { emoji: '🇮🇩', name: 'Bahasa' },
|
||||
'ie': { emoji: '🌐', name: 'Interlingue' },
|
||||
'ig': { emoji: '🇳🇬', name: 'Igbo' },
|
||||
'ii': { emoji: '🌐', name: 'Sichuan Yi' },
|
||||
'ik': { emoji: '🌐', name: 'Inupiaq' },
|
||||
'io': { emoji: '🌐', name: 'Ido' },
|
||||
'is': { emoji: '🇮🇸', name: 'Icelandic' },
|
||||
'it': { emoji: '🇮🇹', name: 'Italiano' },
|
||||
'iu': { emoji: '🌐', name: 'Inuktitut' },
|
||||
'ja': { emoji: '🇯🇵', name: '日本語' },
|
||||
'jv': { emoji: '🌐', name: 'Javanese' },
|
||||
'ka': { emoji: '🇬🇪', name: 'Georgian' },
|
||||
'kg': { emoji: '🌐', name: 'Kongo' },
|
||||
'ki': { emoji: '🌐', name: 'Kikuyu' },
|
||||
'kj': { emoji: '🌐', name: 'Kuanyama' },
|
||||
'kk': { emoji: '🇰🇿', name: 'Kazakh' },
|
||||
'kl': { emoji: '🌐', name: 'Kalaallisut' },
|
||||
'km': { emoji: '🇰🇭', name: 'Central Khmer' },
|
||||
'kn': { emoji: '🌐', name: 'Kannada' },
|
||||
'ko': { emoji: '🇰🇷', name: '한국어' },
|
||||
'kr': { emoji: '🌐', name: 'Kanuri' },
|
||||
'ks': { emoji: '🌐', name: 'Kashmiri' },
|
||||
'ku': { emoji: '🇮🇶', name: 'Kurdish' },
|
||||
'kv': { emoji: '🌐', name: 'Komi' },
|
||||
'kw': { emoji: '🌐', name: 'Cornish' },
|
||||
'ky': { emoji: '🇰🇬', name: 'Kyrgyz' },
|
||||
'la': { emoji: '🌐', name: 'Latin' },
|
||||
'lb': { emoji: '🌐', name: 'Luxembourgish' },
|
||||
'lg': { emoji: '🌐', name: 'Ganda' },
|
||||
'li': { emoji: '🌐', name: 'Limburgan' },
|
||||
'ln': { emoji: '🌐', name: 'Lingala' },
|
||||
'lo': { emoji: '🇱🇦', name: 'Lao' },
|
||||
'lt': { emoji: '🇱🇹', name: 'Lithuanian' },
|
||||
'lu': { emoji: '🌐', name: 'Luba-Katanga' },
|
||||
'lv': { emoji: '🇱🇻', name: 'Latvian' },
|
||||
'mg': { emoji: '🌐', name: 'Malagasy' },
|
||||
'mh': { emoji: '🌐', name: 'Marshallese' },
|
||||
'mi': { emoji: '🌐', name: 'Maori' },
|
||||
'mk': { emoji: '🇲🇰', name: 'Macedonian' },
|
||||
'ml': { emoji: '🌐', name: 'Malayalam' },
|
||||
'mn': { emoji: '🇲🇳', name: 'Mongolian' },
|
||||
'mr': { emoji: '🌐', name: 'Marathi' },
|
||||
'ms': { emoji: '🇲🇾', name: 'Malay' },
|
||||
'mt': { emoji: '🇲🇹', name: 'Maltese' },
|
||||
'my': { emoji: '🇲🇲', name: 'Burmese' },
|
||||
'na': { emoji: '🌐', name: 'Nauru' },
|
||||
'nb': { emoji: '🌐', name: 'Norwegian Bokmål' },
|
||||
'nd': { emoji: '🌐', name: 'North Ndebele' },
|
||||
'ne': { emoji: '🇳🇵', name: 'Nepali' },
|
||||
'ng': { emoji: '🌐', name: 'Ndonga' },
|
||||
'nl': { emoji: '🇧🇪/🇳🇱', name: 'Nederlands' },
|
||||
'nn': { emoji: '🌐', name: 'Norwegian Nynorsk' },
|
||||
'no': { emoji: '🇳🇴', name: 'Norwegian' },
|
||||
'nr': { emoji: '🌐', name: 'South Ndebele' },
|
||||
'nv': { emoji: '🌐', name: 'Navajo' },
|
||||
'ny': { emoji: '🌐', name: 'Chichewa' },
|
||||
'oc': { emoji: '🌐', name: 'Occitan' },
|
||||
'oj': { emoji: '🌐', name: 'Ojibwa' },
|
||||
'om': { emoji: '🌐', name: 'Oromo' },
|
||||
'or': { emoji: '🌐', name: 'Oriya' },
|
||||
'os': { emoji: '🌐', name: 'Ossetian' },
|
||||
'pa': { emoji: '🌐', name: 'Punjabi' },
|
||||
'pi': { emoji: '🌐', name: 'Pali' },
|
||||
'pl': { emoji: '🇵��', name: 'Polski' },
|
||||
'ps': { emoji: '🇦🇫', name: 'Pashto' },
|
||||
'pt': { emoji: '🇵🇹', name: 'Português' },
|
||||
'qu': { emoji: '🌐', name: 'Quechua' },
|
||||
'rm': { emoji: '🌐', name: 'Romansh' },
|
||||
'rn': { emoji: '🌐', name: 'Rundi' },
|
||||
'ro': { emoji: '🇷🇴', name: 'Romanian' },
|
||||
'ru': { emoji: '🇷🇺', name: 'Русский' },
|
||||
'rw': { emoji: '🌐', name: 'Kinyarwanda' },
|
||||
'sa': { emoji: '🌐', name: 'Sanskrit' },
|
||||
'sc': { emoji: '🌐', name: 'Sardinian' },
|
||||
'sd': { emoji: '🌐', name: 'Sindhi' },
|
||||
'se': { emoji: '🌐', name: 'Northern Sami' },
|
||||
'sg': { emoji: '🌐', name: 'Sango' },
|
||||
'si': { emoji: '🇱🇰', name: 'Sinhala' },
|
||||
'sk': { emoji: '🇸🇰', name: 'Slovak' },
|
||||
'sl': { emoji: '🇸🇮', name: 'Slovenian' },
|
||||
'sm': { emoji: '🌐', name: 'Samoan' },
|
||||
'sn': { emoji: '🌐', name: 'Shona' },
|
||||
'so': { emoji: '🇸🇴', name: 'Soomaali' },
|
||||
'sq': { emoji: '🇦🇱', name: 'Albanian' },
|
||||
'sr': { emoji: '🇷🇸', name: 'Serbian' },
|
||||
'ss': { emoji: '🌐', name: 'Swati' },
|
||||
'st': { emoji: '🇿🇦', name: 'Southern Sotho' },
|
||||
'su': { emoji: '🌐', name: 'Sundanese' },
|
||||
'sv': { emoji: '🇸🇪', name: 'Swedish' },
|
||||
'sw': { emoji: '🇰🇪', name: 'Swahili' },
|
||||
'ta': { emoji: '🌐', name: 'Tamil' },
|
||||
'te': { emoji: '🌐', name: 'Telugu' },
|
||||
'tg': { emoji: '🇹🇯', name: 'Tajik' },
|
||||
'th': { emoji: '🇹🇭', name: 'Thai' },
|
||||
'ti': { emoji: '🌐', name: 'Tigrinya' },
|
||||
'tk': { emoji: '🇹🇲', name: 'Turkmen' },
|
||||
'tl': { emoji: '🇵🇭', name: 'Tagalog' },
|
||||
'tn': { emoji: '🌐', name: 'Tswana' },
|
||||
'to': { emoji: '🌐', name: 'Tonga' },
|
||||
'tr': { emoji: '🇹🇷', name: 'Türkçe' },
|
||||
'ts': { emoji: '🌐', name: 'Tsonga' },
|
||||
'tt': { emoji: '🌐', name: 'Tatar' },
|
||||
'tw': { emoji: '🌐', name: 'Twi' },
|
||||
'ty': { emoji: '🌐', name: 'Tahitian' },
|
||||
'ug': { emoji: '🌐', name: 'Uighur' },
|
||||
'uk': { emoji: '🇺🇦', name: 'Ukrainian' },
|
||||
'ur': { emoji: '🇵🇰', name: 'Urdu' },
|
||||
'uz': { emoji: '🇺🇿', name: 'Uzbek' },
|
||||
've': { emoji: '🌐', name: 'Venda' },
|
||||
'vi': { emoji: '🇻🇳', name: 'Tiếng Việt' },
|
||||
'vo': { emoji: '🌐', name: 'Volapük' },
|
||||
'wa': { emoji: '🌐', name: 'Walloon' },
|
||||
'wo': { emoji: '🌐', name: 'Wolof' },
|
||||
'xh': { emoji: '🇿🇦', name: 'Xhosa' },
|
||||
'yi': { emoji: '🌐', name: 'Yiddish' },
|
||||
'yo': { emoji: '🇳🇬', name: 'Yoruba' },
|
||||
'za': { emoji: '🌐', name: 'Zhuang' },
|
||||
'zh': { emoji: '🇨🇳', name: '中文' },
|
||||
'zh-CN': { emoji: '🇨🇳', name: '中文 (简体)' },
|
||||
'zh-HK': { emoji: '🇭🇰', name: '中文 (繁體)' },
|
||||
'zh-Hans': { emoji: '🇨🇳', name: '中文 (简体)' },
|
||||
'zh-Hant': { emoji: '🇹🇼', name: '中文 (繁體)' },
|
||||
'zh-MO': { emoji: '🇲🇴', name: '中文 (繁體)' },
|
||||
'zh-SG': { emoji: '🇸🇬', name: '中文 (繁體)' },
|
||||
'zh-TW': { emoji: '🇹🇼', name: '中文 (繁體)' },
|
||||
'zu': { emoji: '🇿🇦', name: 'Zulu' },
|
||||
};
|
||||
|
||||
function autoAddFlags() {
|
||||
if (!fs.existsSync(README_PATH)) {
|
||||
console.error('Error: Cannot find ' + README_PATH);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let readmeContent = fs.readFileSync(README_PATH, 'utf-8');
|
||||
const marker = ' <!-- Next Flag -->';
|
||||
|
||||
if (!readmeContent.includes(marker)) {
|
||||
console.warn('Error: <!-- Next Flag --> marker not found in ' + README_PATH);
|
||||
console.warn('Please add the marker where you want new flags to be inserted.');
|
||||
return;
|
||||
}
|
||||
|
||||
const files = fs.readdirSync(I18N_DIR);
|
||||
|
||||
const translationFiles = files.filter(f =>
|
||||
f.startsWith('README') &&
|
||||
f.endsWith('.md') &&
|
||||
f !== 'README.md'
|
||||
);
|
||||
|
||||
let updated = false;
|
||||
|
||||
translationFiles.forEach(file => {
|
||||
const code = file.split('.')[1];
|
||||
const lang = LANG_MAP[code];
|
||||
|
||||
if (!lang) return;
|
||||
|
||||
const flagLink = ` <a href="docs/i18n/${file}">${lang.emoji} ${lang.name}</a>`;
|
||||
|
||||
if (!readmeContent.includes(file)) {
|
||||
readmeContent = readmeContent.replace(marker, flagLink + '\n' + marker);
|
||||
updated = true;
|
||||
console.log('Added ' + code);
|
||||
}
|
||||
});
|
||||
|
||||
if (updated) {
|
||||
fs.writeFileSync(README_PATH, readmeContent, 'utf-8');
|
||||
console.log('Main README updated.');
|
||||
} else {
|
||||
console.log('No new flags to add.');
|
||||
}
|
||||
}
|
||||
|
||||
autoAddFlags();
|
||||
@@ -0,0 +1,114 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const I18N_DIR = path.join(__dirname, '..');
|
||||
const ROOT_DIR = path.join(I18N_DIR, '..', '..');
|
||||
const README_PATH = path.join(ROOT_DIR, 'README.md');
|
||||
|
||||
const MARKERS = {
|
||||
HEADER: {
|
||||
START: '<!-- HEADER:START -->',
|
||||
END: '<!-- HEADER:END -->'
|
||||
},
|
||||
FLAGS: {
|
||||
START: '<!-- FLAGS:START -->',
|
||||
END: '<!-- FLAGS:END -->'
|
||||
},
|
||||
NAV: {
|
||||
START: '<!-- NAV:START -->',
|
||||
END: '<!-- NAV:END -->'
|
||||
},
|
||||
BADGES: {
|
||||
START: '<!-- BADGES:START -->',
|
||||
END: '<!-- BADGES:END -->'
|
||||
}
|
||||
};
|
||||
|
||||
function extractBlock(content, marker) {
|
||||
const startIdx = content.indexOf(marker.START);
|
||||
const endIdx = content.indexOf(marker.END);
|
||||
|
||||
if (startIdx === -1 || endIdx === -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return content.substring(startIdx, endIdx + marker.END.length);
|
||||
}
|
||||
|
||||
function updateOrInsertBlock(targetContent, blockKey, blockValue, fallbackAnchor = null) {
|
||||
const marker = MARKERS[blockKey];
|
||||
const startIdx = targetContent.indexOf(marker.START);
|
||||
const endIdx = targetContent.indexOf(marker.END);
|
||||
|
||||
if (startIdx !== -1 && endIdx !== -1) {
|
||||
const prefix = targetContent.substring(0, startIdx);
|
||||
const suffix = targetContent.substring(endIdx + marker.END.length);
|
||||
return prefix + blockValue + suffix;
|
||||
}
|
||||
|
||||
if (startIdx !== -1 || endIdx !== -1) {
|
||||
let cleaned = targetContent.split(marker.START).join('');
|
||||
cleaned = cleaned.split(marker.END).join('');
|
||||
return updateOrInsertBlock(cleaned, blockKey, blockValue, fallbackAnchor);
|
||||
}
|
||||
|
||||
if (fallbackAnchor) {
|
||||
const anchorIdx = targetContent.indexOf(fallbackAnchor);
|
||||
if (anchorIdx !== -1) {
|
||||
const insertAfterIdx = targetContent.indexOf('\n', anchorIdx) + 1;
|
||||
const prefix = targetContent.substring(0, insertAfterIdx);
|
||||
const suffix = targetContent.substring(insertAfterIdx);
|
||||
return prefix + '\n' + blockValue + '\n' + suffix;
|
||||
}
|
||||
}
|
||||
|
||||
return targetContent + '\n\n' + blockValue;
|
||||
}
|
||||
|
||||
function updateLocalizedReadmes() {
|
||||
if (!fs.existsSync(README_PATH)) {
|
||||
console.error('Error: Cannot find ' + README_PATH);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const readmeContent = fs.readFileSync(README_PATH, 'utf-8');
|
||||
|
||||
const headerBlock = extractBlock(readmeContent, MARKERS.HEADER);
|
||||
const flagsBlock = extractBlock(readmeContent, MARKERS.FLAGS);
|
||||
const navBlock = extractBlock(readmeContent, MARKERS.NAV);
|
||||
const badgesBlock = extractBlock(readmeContent, MARKERS.BADGES);
|
||||
|
||||
if (!headerBlock || !flagsBlock || !navBlock || !badgesBlock) {
|
||||
console.error('Error: Some markers are missing in main README.md');
|
||||
if (!headerBlock) console.error('Missing: HEADER');
|
||||
if (!flagsBlock) console.error('Missing: FLAGS');
|
||||
if (!navBlock) console.error('Missing: NAV');
|
||||
if (!badgesBlock) console.error('Missing: BADGES');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let flagsI18n = flagsBlock.split('href="docs/i18n/').join('href="');
|
||||
flagsI18n = flagsI18n.split('href="README.md"').join('href="../../README.md"');
|
||||
|
||||
const files = fs.readdirSync(I18N_DIR);
|
||||
const targetFiles = files
|
||||
.filter(file => file.startsWith('README.') && file.endsWith('.md') && file !== 'README.md')
|
||||
.map(file => path.join(I18N_DIR, file));
|
||||
|
||||
targetFiles.forEach(filePath => {
|
||||
let content = fs.readFileSync(filePath, 'utf-8');
|
||||
const fileName = path.basename(filePath);
|
||||
|
||||
content = updateOrInsertBlock(content, 'HEADER', headerBlock, '<img');
|
||||
content = updateOrInsertBlock(content, 'FLAGS', flagsI18n, MARKERS.HEADER.END);
|
||||
content = updateOrInsertBlock(content, 'NAV', navBlock, MARKERS.FLAGS.END);
|
||||
content = updateOrInsertBlock(content, 'BADGES', badgesBlock, MARKERS.NAV.END);
|
||||
|
||||
fs.writeFileSync(filePath, content, 'utf-8');
|
||||
console.log('Updated ' + fileName);
|
||||
});
|
||||
|
||||
console.log('\nSuccess: Done.');
|
||||
}
|
||||
|
||||
updateLocalizedReadmes();
|
||||
Reference in New Issue
Block a user