commit 027e11f1605e7c14392076204fbbca9099082f6f Author: wehub-skill-sync Date: Mon Jul 13 21:37:11 2026 +0800 chore: import zh skill qdrant-vector-search diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..cad07b8 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,9 @@ +# WeHub 来源说明 + +- Skill 名称:`qdrant-vector-search` +- 中文类目:向量索引调优 +- 上游仓库:`nousresearch__hermes-agent` +- 上游路径:`optional-skills/mlops/qdrant/SKILL.md` +- 上游链接:https://github.com/nousresearch/hermes-agent/blob/HEAD/optional-skills/mlops/qdrant/SKILL.md +- 本仓库为 WeHub 中文 Skill 汉化包,基于 skill 市场筛选 Top200 清单整理 +- 原作者、版权和许可证信息以上游仓库为准 diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..e83f558 --- /dev/null +++ b/SKILL.md @@ -0,0 +1,497 @@ +--- +name: qdrant-vector-search +description: 高性能向量相似度搜索引擎,适用于 RAG 和语义搜索。当需要构建生产级 RAG 系统,要求快速最近邻搜索、带过滤条件的混合搜索,或需要可扩展的 Rust 高性能向量存储时使用。 +version: 1.0.0 +author: Orchestra Research +license: MIT +dependencies: [qdrant-client>=1.12.0] +platforms: [linux, macos, windows] +metadata: + hermes: + tags: [RAG, Vector Search, Qdrant, Semantic Search, Embeddings, Similarity Search, HNSW, Production, Distributed] + +--- + +# Qdrant - 向量相似度搜索引擎 + +基于 Rust 编写的高性能向量数据库,适用于生产级 RAG 和语义搜索。 + +## 何时使用 Qdrant + +**使用 Qdrant 的场景:** +- 构建需要低延迟的生产级 RAG 系统 +- 需要混合搜索(向量 + 元数据过滤) +- 需要支持分片/复制的水平扩展 +- 希望本地部署并完全掌控数据 +- 每条记录需要多向量存储(稠密 + 稀疏) +- 构建实时推荐系统 + +**主要特性:** +- **Rust 驱动**:内存安全,高性能 +- **丰富过滤**:搜索时可按任意 payload 字段过滤 +- **多向量支持**:每个点支持稠密、稀疏、多稠密向量 +- **量化**:标量、乘积、二进制量化,节省内存 +- **分布式**:Raft 共识、分片、复制 +- **REST + gRPC**:两种 API 功能完全对等 + +**备选方案:** +- **Chroma**:安装更简单,适用于嵌入式场景 +- **FAISS**:追求最高原始速度,适用于研究/批量处理 +- **Pinecone**:全托管,适合零运维偏好 +- **Weaviate**:偏好 GraphQL,内置向量化器 + +## 快速开始 + +### 安装 + +```bash +# Python 客户端 +pip install qdrant-client + +# Docker(推荐用于开发) +docker run -p 6333:6333 -p 6334:6334 qdrant/qdrant + +# Docker 持久化存储 +docker run -p 6333:6333 -p 6334:6334 \ + -v $(pwd)/qdrant_storage:/qdrant/storage \ + qdrant/qdrant +``` + +### 基本用法 + +```python +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams, PointStruct + +# 连接 Qdrant +client = QdrantClient(host="localhost", port=6333) + +# 创建集合 +client.create_collection( + collection_name="documents", + vectors_config=VectorParams(size=384, distance=Distance.COSINE) +) + +# 插入向量及 payload +client.upsert( + collection_name="documents", + points=[ + PointStruct( + id=1, + vector=[0.1, 0.2, ...], # 384 维向量 + payload={"title": "Doc 1", "category": "tech"} + ), + PointStruct( + id=2, + vector=[0.3, 0.4, ...], + payload={"title": "Doc 2", "category": "science"} + ) + ] +) + +# 带过滤条件的搜索 +results = client.search( + collection_name="documents", + query_vector=[0.15, 0.25, ...], + query_filter={ + "must": [{"key": "category", "match": {"value": "tech"}}] + }, + limit=10 +) + +for point in results: + print(f"ID: {point.id}, Score: {point.score}, Payload: {point.payload}") +``` + +## 核心概念 + +### Points - 基本数据单元 + +```python +from qdrant_client.models import PointStruct + +# Point = ID + 向量 + Payload +point = PointStruct( + id=123, # 整数或 UUID 字符串 + vector=[0.1, 0.2, 0.3, ...], # 稠密向量 + payload={ # 任意 JSON 元数据 + "title": "Document title", + "category": "tech", + "timestamp": 1699900000, + "tags": ["python", "ml"] + } +) + +# 批量 upsert(推荐) +client.upsert( + collection_name="documents", + points=[point1, point2, point3], + wait=True # 等待索引完成 +) +``` + +### Collections - 向量容器 + +```python +from qdrant_client.models import VectorParams, Distance, HnswConfigDiff + +# 创建集合并配置 HNSW +client.create_collection( + collection_name="documents", + vectors_config=VectorParams( + size=384, # 向量维度 + distance=Distance.COSINE # COSINE, EUCLID, DOT, MANHATTAN + ), + hnsw_config=HnswConfigDiff( + m=16, # 每节点连接数(默认 16) + ef_construct=100, # 构建时精度(默认 100) + full_scan_threshold=10000 # 低于此值切换为暴力搜索 + ), + on_disk_payload=True # payload 存储到磁盘 +) + +# 查看集合信息 +info = client.get_collection("documents") +print(f"点数: {info.points_count}, 向量数: {info.vectors_count}") +``` + +### 距离度量 + +| 度量方式 | 适用场景 | 范围 | +|--------|----------|-------| +| `COSINE` | 文本嵌入、归一化向量 | 0 到 2 | +| `EUCLID` | 空间数据、图像特征 | 0 到 ∞ | +| `DOT` | 推荐系统、非归一化向量 | -∞ 到 ∞ | +| `MANHATTAN` | 稀疏特征、离散数据 | 0 到 ∞ | + +## 搜索操作 + +### 基本搜索 + +```python +# 简单最近邻搜索 +results = client.search( + collection_name="documents", + query_vector=[0.1, 0.2, ...], + limit=10, + with_payload=True, + with_vectors=False # 不返回向量(更快) +) +``` + +### 带过滤条件的搜索 + +```python +from qdrant_client.models import Filter, FieldCondition, MatchValue, Range + +# 复杂过滤 +results = client.search( + collection_name="documents", + query_vector=query_embedding, + query_filter=Filter( + must=[ + FieldCondition(key="category", match=MatchValue(value="tech")), + FieldCondition(key="timestamp", range=Range(gte=1699000000)) + ], + must_not=[ + FieldCondition(key="status", match=MatchValue(value="archived")) + ] + ), + limit=10 +) + +# 简化过滤语法 +results = client.search( + collection_name="documents", + query_vector=query_embedding, + query_filter={ + "must": [ + {"key": "category", "match": {"value": "tech"}}, + {"key": "price", "range": {"gte": 10, "lte": 100}} + ] + }, + limit=10 +) +``` + +### 批量搜索 + +```python +from qdrant_client.models import SearchRequest + +# 一次请求执行多个查询 +results = client.search_batch( + collection_name="documents", + requests=[ + SearchRequest(vector=[0.1, ...], limit=5), + SearchRequest(vector=[0.2, ...], limit=5, filter={"must": [...]}), + SearchRequest(vector=[0.3, ...], limit=10) + ] +) +``` + +## RAG 集成 + +### 配合 sentence-transformers + +```python +from sentence_transformers import SentenceTransformer +from qdrant_client import QdrantClient +from qdrant_client.models import VectorParams, Distance, PointStruct + +# 初始化 +encoder = SentenceTransformer("all-MiniLM-L6-v2") +client = QdrantClient(host="localhost", port=6333) + +# 创建集合 +client.create_collection( + collection_name="knowledge_base", + vectors_config=VectorParams(size=384, distance=Distance.COSINE) +) + +# 索引文档 +documents = [ + {"id": 1, "text": "Python is a programming language", "source": "wiki"}, + {"id": 2, "text": "Machine learning uses algorithms", "source": "textbook"}, +] + +points = [ + PointStruct( + id=doc["id"], + vector=encoder.encode(doc["text"]).tolist(), + payload={"text": doc["text"], "source": doc["source"]} + ) + for doc in documents +] +client.upsert(collection_name="knowledge_base", points=points) + +# RAG 检索 +def retrieve(query: str, top_k: int = 5) -> list[dict]: + query_vector = encoder.encode(query).tolist() + results = client.search( + collection_name="knowledge_base", + query_vector=query_vector, + limit=top_k + ) + return [{"text": r.payload["text"], "score": r.score} for r in results] + +# 在 RAG 流水线中使用 +context = retrieve("What is Python?") +prompt = f"Context: {context}\n\nQuestion: What is Python?" +``` + +### 配合 LangChain + +```python +from langchain_community.vectorstores import Qdrant +from langchain_community.embeddings import HuggingFaceEmbeddings + +embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2") +vectorstore = Qdrant.from_documents(documents, embeddings, url="http://localhost:6333", collection_name="docs") +retriever = vectorstore.as_retriever(search_kwargs={"k": 5}) +``` + +### 配合 LlamaIndex + +```python +from llama_index.vector_stores.qdrant import QdrantVectorStore +from llama_index.core import VectorStoreIndex, StorageContext + +vector_store = QdrantVectorStore(client=client, collection_name="llama_docs") +storage_context = StorageContext.from_defaults(vector_store=vector_store) +index = VectorStoreIndex.from_documents(documents, storage_context=storage_context) +query_engine = index.as_query_engine() +``` + +## 多向量支持 + +### 命名向量(不同嵌入模型) + +```python +from qdrant_client.models import VectorParams, Distance + +# 包含多种向量类型的集合 +client.create_collection( + collection_name="hybrid_search", + vectors_config={ + "dense": VectorParams(size=384, distance=Distance.COSINE), + "sparse": VectorParams(size=30000, distance=Distance.DOT) + } +) + +# 使用命名向量插入 +client.upsert( + collection_name="hybrid_search", + points=[ + PointStruct( + id=1, + vector={ + "dense": dense_embedding, + "sparse": sparse_embedding + }, + payload={"text": "document text"} + ) + ] +) + +# 搜索指定向量 +results = client.search( + collection_name="hybrid_search", + query_vector=("dense", query_dense), # 指定向量名称 + limit=10 +) +``` + +### 稀疏向量(BM25, SPLADE) + +```python +from qdrant_client.models import SparseVectorParams, SparseIndexParams, SparseVector + +# 创建支持稀疏向量的集合 +client.create_collection( + collection_name="sparse_search", + vectors_config={}, + sparse_vectors_config={"text": SparseVectorParams(index=SparseIndexParams(on_disk=False))} +) + +# 插入稀疏向量 +client.upsert( + collection_name="sparse_search", + points=[PointStruct(id=1, vector={"text": SparseVector(indices=[1, 5, 100], values=[0.5, 0.8, 0.2])}, payload={"text": "document"})] +) +``` + +## 量化(内存优化) + +```python +from qdrant_client.models import ScalarQuantization, ScalarQuantizationConfig, ScalarType + +# 标量量化(内存减少 4 倍) +client.create_collection( + collection_name="quantized", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + quantization_config=ScalarQuantization( + scalar=ScalarQuantizationConfig( + type=ScalarType.INT8, + quantile=0.99, # 裁剪异常值 + always_ram=True # 量化数据常驻 RAM + ) + ) +) + +# 带重评分的搜索 +results = client.search( + collection_name="quantized", + query_vector=query, + search_params={"quantization": {"rescore": True}}, # 对顶部结果重新评分 + limit=10 +) +``` + +## Payload 索引 + +```python +from qdrant_client.models import PayloadSchemaType + +# 创建 payload 索引以加速过滤 +client.create_payload_index( + collection_name="documents", + field_name="category", + field_schema=PayloadSchemaType.KEYWORD +) + +client.create_payload_index( + collection_name="documents", + field_name="timestamp", + field_schema=PayloadSchemaType.INTEGER +) + +# 索引类型:KEYWORD, INTEGER, FLOAT, GEO, TEXT(全文搜索), BOOL +``` + +## 生产部署 + +### Qdrant Cloud + +```python +from qdrant_client import QdrantClient + +# 连接到 Qdrant Cloud +client = QdrantClient( + url="https://your-cluster.cloud.qdrant.io", + api_key="your-api-key" +) +``` + +### 性能调优 + +```python +# 优化搜索速度(更高召回率) +client.update_collection( + collection_name="documents", + hnsw_config=HnswConfigDiff(ef_construct=200, m=32) +) + +# 优化索引速度(批量加载) +client.update_collection( + collection_name="documents", + optimizer_config={"indexing_threshold": 20000} +) +``` + +## 最佳实践 + +1. **批量操作** - 使用批量 upsert/search 提高效率 +2. **Payload 索引** - 为过滤字段建立索引 +3. **量化** - 大集合(>100 万向量)建议启用 +4. **分片** - 集合超过 1000 万向量时使用 +5. **磁盘存储** - 大 payload 时启用 `on_disk_payload` +6. **连接池** - 复用客户端实例 + +## 常见问题 + +**带过滤条件的搜索速度慢:** +```python +# 为过滤字段创建 payload 索引 +client.create_payload_index( + collection_name="docs", + field_name="category", + field_schema=PayloadSchemaType.KEYWORD +) +``` + +**内存不足:** +```python +# 启用量化和磁盘存储 +client.create_collection( + collection_name="large_collection", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + quantization_config=ScalarQuantization(...), + on_disk_payload=True +) +``` + +**连接问题:** +```python +# 设置超时和重试 +client = QdrantClient( + host="localhost", + port=6333, + timeout=30, + prefer_grpc=True # gRPC 提供更好性能 +) +``` + +## 参考文档 + +- **[高级用法](references/advanced-usage.md)** - 分布式模式、混合搜索、推荐 +- **[故障排查](references/troubleshooting.md)** - 常见问题、调试、性能调优 + +## 资源 + +- **GitHub**:https://github.com/qdrant/qdrant(22k+ Stars) +- **文档**:https://qdrant.tech/documentation/ +- **Python 客户端**:https://github.com/qdrant/qdrant-client +- **Cloud**:https://cloud.qdrant.io +- **版本**:1.12.0+ +- **许可证**:Apache 2.0 diff --git a/references/advanced-usage.md b/references/advanced-usage.md new file mode 100644 index 0000000..5f4ec86 --- /dev/null +++ b/references/advanced-usage.md @@ -0,0 +1,648 @@ +# Qdrant 高级使用指南 + +## 分布式部署 + +### 集群搭建 + +Qdrant 使用 Raft 共识算法进行分布式协调。 + +```yaml +# docker-compose.yml for 3-node cluster +version: '3.8' +services: + qdrant-node-1: + image: qdrant/qdrant:latest + ports: + - "6333:6333" + - "6334:6334" + - "6335:6335" + volumes: + - ./node1_storage:/qdrant/storage + environment: + - QDRANT__CLUSTER__ENABLED=true + - QDRANT__CLUSTER__P2P__PORT=6335 + - QDRANT__SERVICE__HTTP_PORT=6333 + - QDRANT__SERVICE__GRPC_PORT=6334 + + qdrant-node-2: + image: qdrant/qdrant:latest + ports: + - "6343:6333" + - "6344:6334" + - "6345:6335" + volumes: + - ./node2_storage:/qdrant/storage + environment: + - QDRANT__CLUSTER__ENABLED=true + - QDRANT__CLUSTER__P2P__PORT=6335 + - QDRANT__CLUSTER__BOOTSTRAP=http://qdrant-node-1:6335 + depends_on: + - qdrant-node-1 + + qdrant-node-3: + image: qdrant/qdrant:latest + ports: + - "6353:6333" + - "6354:6334" + - "6355:6335" + volumes: + - ./node3_storage:/qdrant/storage + environment: + - QDRANT__CLUSTER__ENABLED=true + - QDRANT__CLUSTER__P2P__PORT=6335 + - QDRANT__CLUSTER__BOOTSTRAP=http://qdrant-node-1:6335 + depends_on: + - qdrant-node-1 +``` + +### 分片配置 + +```python +from qdrant_client import QdrantClient +from qdrant_client.models import VectorParams, Distance, ShardingMethod + +client = QdrantClient(host="localhost", port=6333) + +# 创建分片集合 +client.create_collection( + collection_name="large_collection", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + shard_number=6, # 分片数量 + replication_factor=2, # 每个分片的副本数 + write_consistency_factor=1 # 写入所需确认数 +) + +# 检查集群状态 +cluster_info = client.get_cluster_info() +print(f"Peers: {cluster_info.peers}") +print(f"Raft state: {cluster_info.raft_info}") +``` + +### 复制与一致性 + +```python +from qdrant_client.models import WriteOrdering + +# 强一致性写入 +client.upsert( + collection_name="critical_data", + points=points, + ordering=WriteOrdering.STRONG # 等待所有副本 +) + +# 最终一致性(更快) +client.upsert( + collection_name="logs", + points=points, + ordering=WriteOrdering.WEAK # 主节点确认后返回 +) + +# 从特定分片读取 +results = client.search( + collection_name="documents", + query_vector=query, + consistency="majority" # 从多数副本读取 +) +``` + +## 混合搜索 + +### 稠密 + 稀疏向量 + +结合语义(稠密)与关键词(稀疏)搜索: + +```python +from qdrant_client.models import ( + VectorParams, SparseVectorParams, SparseIndexParams, + Distance, PointStruct, SparseVector, Prefetch, Query +) + +# 创建混合集合 +client.create_collection( + collection_name="hybrid", + vectors_config={ + "dense": VectorParams(size=384, distance=Distance.COSINE) + }, + sparse_vectors_config={ + "sparse": SparseVectorParams( + index=SparseIndexParams(on_disk=False) + ) + } +) + +# 插入两种向量类型 +def encode_sparse(text: str) -> SparseVector: + """简单的类 BM25 稀疏编码""" + from collections import Counter + tokens = text.lower().split() + counts = Counter(tokens) + # 将 token 映射到索引(生产环境请使用词表) + indices = [hash(t) % 30000 for t in counts.keys()] + values = list(counts.values()) + return SparseVector(indices=indices, values=values) + +client.upsert( + collection_name="hybrid", + points=[ + PointStruct( + id=1, + vector={ + "dense": dense_encoder.encode("Python programming").tolist(), + "sparse": encode_sparse("Python programming language code") + }, + payload={"text": "Python programming language code"} + ) + ] +) + +# 使用互惠排名融合(RRF)进行混合搜索 +from qdrant_client.models import FusionQuery + +results = client.query_points( + collection_name="hybrid", + prefetch=[ + Prefetch(query=dense_query, using="dense", limit=20), + Prefetch(query=sparse_query, using="sparse", limit=20) + ], + query=FusionQuery(fusion="rrf"), # 合并结果 + limit=10 +) +``` + +### 多阶段搜索 + +```python +from qdrant_client.models import Prefetch, Query + +# 两阶段检索:粗筛后精排 +results = client.query_points( + collection_name="documents", + prefetch=[ + Prefetch( + query=query_vector, + limit=100, # 宽筛第一阶段 + params={"quantization": {"rescore": False}} # 快速近似 + ) + ], + query=Query(nearest=query_vector), + limit=10, + params={"quantization": {"rescore": True}} # 精确重排序 +) +``` + +## 推荐 + +### 物品到物品推荐 + +```python +# 查找相似物品 +recommendations = client.recommend( + collection_name="products", + positive=[1, 2, 3], # 用户喜欢的 ID + negative=[4], # 用户不喜欢的 ID + limit=10 +) + +# 带过滤条件的推荐 +recommendations = client.recommend( + collection_name="products", + positive=[1, 2], + query_filter={ + "must": [ + {"key": "category", "match": {"value": "electronics"}}, + {"key": "in_stock", "match": {"value": True}} + ] + }, + limit=10 +) +``` + +### 从其他集合查找 + +```python +from qdrant_client.models import RecommendStrategy, LookupLocation + +# 使用另一个集合中的向量进行推荐 +results = client.recommend( + collection_name="products", + positive=[ + LookupLocation( + collection_name="user_history", + id="user_123" + ) + ], + strategy=RecommendStrategy.AVERAGE_VECTOR, + limit=10 +) +``` + +## 高级过滤 + +### 嵌套负载过滤 + +```python +from qdrant_client.models import Filter, FieldCondition, MatchValue, NestedCondition + +# 对嵌套对象进行过滤 +results = client.search( + collection_name="documents", + query_vector=query, + query_filter=Filter( + must=[ + NestedCondition( + key="metadata", + filter=Filter( + must=[ + FieldCondition( + key="author.name", + match=MatchValue(value="John") + ) + ] + ) + ) + ] + ), + limit=10 +) +``` + +### 地理过滤 + +```python +from qdrant_client.models import FieldCondition, GeoRadius, GeoPoint + +# 查找半径范围内的结果 +results = client.search( + collection_name="locations", + query_vector=query, + query_filter=Filter( + must=[ + FieldCondition( + key="location", + geo_radius=GeoRadius( + center=GeoPoint(lat=40.7128, lon=-74.0060), + radius=5000 # 米 + ) + ) + ] + ), + limit=10 +) + +# 地理边界框 +from qdrant_client.models import GeoBoundingBox + +results = client.search( + collection_name="locations", + query_vector=query, + query_filter=Filter( + must=[ + FieldCondition( + key="location", + geo_bounding_box=GeoBoundingBox( + top_left=GeoPoint(lat=40.8, lon=-74.1), + bottom_right=GeoPoint(lat=40.6, lon=-73.9) + ) + ) + ] + ), + limit=10 +) +``` + +### 全文搜索 + +```python +from qdrant_client.models import TextIndexParams, TokenizerType + +# 创建文本索引 +client.create_payload_index( + collection_name="documents", + field_name="content", + field_schema=TextIndexParams( + type="text", + tokenizer=TokenizerType.WORD, + min_token_len=2, + max_token_len=15, + lowercase=True + ) +) + +# 全文过滤 +from qdrant_client.models import MatchText + +results = client.search( + collection_name="documents", + query_vector=query, + query_filter=Filter( + must=[ + FieldCondition( + key="content", + match=MatchText(text="machine learning") + ) + ] + ), + limit=10 +) +``` + +## 量化策略 + +### 标量量化(INT8) + +```python +from qdrant_client.models import ScalarQuantization, ScalarQuantizationConfig, ScalarType + +# 内存减少约 4 倍,精度损失极小 +client.create_collection( + collection_name="scalar_quantized", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + quantization_config=ScalarQuantization( + scalar=ScalarQuantizationConfig( + type=ScalarType.INT8, + quantile=0.99, # 裁剪极端值 + always_ram=True # 将量化后的向量保留在 RAM 中 + ) + ) +) +``` + +### 乘积量化 + +```python +from qdrant_client.models import ProductQuantization, ProductQuantizationConfig, CompressionRatio + +# 内存减少约 16 倍,有一定精度损失 +client.create_collection( + collection_name="product_quantized", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + quantization_config=ProductQuantization( + product=ProductQuantizationConfig( + compression=CompressionRatio.X16, + always_ram=True + ) + ) +) +``` + +### 二值量化 + +```python +from qdrant_client.models import BinaryQuantization, BinaryQuantizationConfig + +# 内存减少约 32 倍,需要过采样 +client.create_collection( + collection_name="binary_quantized", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + quantization_config=BinaryQuantization( + binary=BinaryQuantizationConfig(always_ram=True) + ) +) + +# 带过采样的搜索 +results = client.search( + collection_name="binary_quantized", + query_vector=query, + search_params={ + "quantization": { + "rescore": True, + "oversampling": 2.0 # 检索 2 倍候选数量后重排序 + } + }, + limit=10 +) +``` + +## 快照与备份 + +### 创建快照 + +```python +# 创建集合快照 +snapshot_info = client.create_snapshot(collection_name="documents") +print(f"Snapshot: {snapshot_info.name}") + +# 列出快照 +snapshots = client.list_snapshots(collection_name="documents") +for s in snapshots: + print(f"{s.name}: {s.size} bytes") + +# 完整存储快照 +full_snapshot = client.create_full_snapshot() +``` + +### 从快照恢复 + +```python +# 下载快照 +client.download_snapshot( + collection_name="documents", + snapshot_name="documents-2024-01-01.snapshot", + target_path="./backup/" +) + +# 恢复(通过 REST API) +import requests + +response = requests.put( + "http://localhost:6333/collections/documents/snapshots/recover", + json={"location": "file:///backup/documents-2024-01-01.snapshot"} +) +``` + +## 集合别名 + +```python +# 创建别名 +client.update_collection_aliases( + change_aliases_operations=[ + {"create_alias": {"alias_name": "production", "collection_name": "documents_v2"}} + ] +) + +# 蓝绿部署 +# 1. 创建带更新的新集合 +client.create_collection(collection_name="documents_v3", ...) + +# 2. 填充新集合 +client.upsert(collection_name="documents_v3", points=new_points) + +# 3. 原子切换 +client.update_collection_aliases( + change_aliases_operations=[ + {"delete_alias": {"alias_name": "production"}}, + {"create_alias": {"alias_name": "production", "collection_name": "documents_v3"}} + ] +) + +# 通过别名搜索 +results = client.search(collection_name="production", query_vector=query, limit=10) +``` + +## 滚动与迭代 + +### 滚动遍历所有点 + +```python +# 分页迭代 +offset = None +all_points = [] + +while True: + results, offset = client.scroll( + collection_name="documents", + limit=100, + offset=offset, + with_payload=True, + with_vectors=False + ) + all_points.extend(results) + + if offset is None: + break + +print(f"Total points: {len(all_points)}") +``` + +### 带过滤的滚动 + +```python +# 带过滤条件的滚动 +results, _ = client.scroll( + collection_name="documents", + scroll_filter=Filter( + must=[ + FieldCondition(key="status", match=MatchValue(value="active")) + ] + ), + limit=1000 +) +``` + +## 异步客户端 + +```python +import asyncio +from qdrant_client import AsyncQdrantClient + +async def main(): + client = AsyncQdrantClient(host="localhost", port=6333) + + # 异步操作 + await client.create_collection( + collection_name="async_docs", + vectors_config=VectorParams(size=384, distance=Distance.COSINE) + ) + + await client.upsert( + collection_name="async_docs", + points=points + ) + + results = await client.search( + collection_name="async_docs", + query_vector=query, + limit=10 + ) + + return results + +results = asyncio.run(main()) +``` + +## gRPC 客户端 + +```python +from qdrant_client import QdrantClient + +# 优先使用 gRPC 以获得更佳性能 +client = QdrantClient( + host="localhost", + port=6333, + grpc_port=6334, + prefer_grpc=True # 可用时使用 gRPC +) + +# 纯 gRPC 客户端 +from qdrant_client import QdrantClient + +client = QdrantClient( + host="localhost", + grpc_port=6334, + prefer_grpc=True, + https=False +) +``` + +## 多租户 + +### 基于负载的隔离 + +```python +# 单个集合,按租户过滤 +client.upsert( + collection_name="multi_tenant", + points=[ + PointStruct( + id=1, + vector=embedding, + payload={"tenant_id": "tenant_a", "text": "..."} + ) + ] +) + +# 在租户范围内搜索 +results = client.search( + collection_name="multi_tenant", + query_vector=query, + query_filter=Filter( + must=[FieldCondition(key="tenant_id", match=MatchValue(value="tenant_a"))] + ), + limit=10 +) +``` + +### 每租户独立集合 + +```python +# 创建租户集合 +def create_tenant_collection(tenant_id: str): + client.create_collection( + collection_name=f"tenant_{tenant_id}", + vectors_config=VectorParams(size=384, distance=Distance.COSINE) + ) + +# 搜索租户集合 +def search_tenant(tenant_id: str, query_vector: list, limit: int = 10): + return client.search( + collection_name=f"tenant_{tenant_id}", + query_vector=query_vector, + limit=limit + ) +``` + +## 性能监控 + +### 集合统计信息 + +```python +# 集合信息 +info = client.get_collection("documents") +print(f"Points: {info.points_count}") +print(f"Indexed vectors: {info.indexed_vectors_count}") +print(f"Segments: {len(info.segments)}") +print(f"Status: {info.status}") + +# 详细段信息 +for i, segment in enumerate(info.segments): + print(f"Segment {i}: {segment}") +``` + +### 遥测 + +```python +# 获取遥测数据 +telemetry = client.get_telemetry() +print(f"Collections: {telemetry.collections}") +print(f"Operations: {telemetry.operations}") +``` diff --git a/references/troubleshooting.md b/references/troubleshooting.md new file mode 100644 index 0000000..5a99eec --- /dev/null +++ b/references/troubleshooting.md @@ -0,0 +1,631 @@ +# Qdrant 故障排查指南 + +## 安装问题 + +### Docker 问题 + +**错误**:`Cannot connect to Docker daemon` + +**修复**: +```bash +# 启动 Docker 守护进程 +sudo systemctl start docker + +# 或在 Mac/Windows 上使用 Docker Desktop +open -a Docker +``` + +**错误**:`Port 6333 already in use` + +**修复**: +```bash +# 查找占用端口的进程 +lsof -i :6333 + +# 终止进程或使用其他端口 +docker run -p 6334:6333 qdrant/qdrant +``` + +### Python 客户端问题 + +**错误**:`ModuleNotFoundError: No module named 'qdrant_client'` + +**修复**: +```bash +pip install qdrant-client + +# 指定版本 +pip install qdrant-client>=1.12.0 +``` + +**错误**:`grpc._channel._InactiveRpcError` + +**修复**: +```bash +# 安装带 gRPC 支持的版本 +pip install 'qdrant-client[grpc]' + +# 或禁用 gRPC +client = QdrantClient(host="localhost", port=6333, prefer_grpc=False) +``` + +## 连接问题 + +### 无法连接到服务器 + +**错误**:`ConnectionRefusedError: [Errno 111] Connection refused` + +**解决方案**: + +1. **检查服务器是否在运行**: +```bash +docker ps | grep qdrant +curl http://localhost:6333/healthz +``` + +2. **验证端口绑定**: +```bash +# 检查监听端口 +netstat -tlnp | grep 6333 + +# Docker 端口映射 +docker port +``` + +3. **使用正确的主机地址**: +```python +# Linux 上的 Docker +client = QdrantClient(host="localhost", port=6333) + +# Mac/Windows 上有网络问题的 Docker +client = QdrantClient(host="127.0.0.1", port=6333) + +# Docker 网络内部 +client = QdrantClient(host="qdrant", port=6333) +``` + +### 超时错误 + +**错误**:`TimeoutError: Connection timed out` + +**修复**: +```python +# 增加超时时间 +client = QdrantClient( + host="localhost", + port=6333, + timeout=60 # 秒 +) + +# 对于大型操作 +client.upsert( + collection_name="documents", + points=large_batch, + wait=False # 不等待索引完成 +) +``` + +### SSL/TLS 错误 + +**错误**:`ssl.SSLCertVerificationError` + +**修复**: +```python +# Qdrant Cloud +client = QdrantClient( + url="https://cluster.cloud.qdrant.io", + api_key="your-api-key" +) + +# 自签名证书 +client = QdrantClient( + host="localhost", + port=6333, + https=True, + verify=False # 禁用验证(不推荐用于生产环境) +) +``` + +## 集合问题 + +### 集合已存在 + +**错误**:`ValueError: Collection 'documents' already exists` + +**修复**: +```python +# 创建前先检查 +collections = client.get_collections().collections +names = [c.name for c in collections] + +if "documents" not in names: + client.create_collection(...) + +# 或重新创建 +client.recreate_collection( + collection_name="documents", + vectors_config=VectorParams(size=384, distance=Distance.COSINE) +) +``` + +### 集合未找到 + +**错误**:`NotFoundException: Collection 'docs' not found` + +**修复**: +```python +# 列出可用集合 +collections = client.get_collections() +print([c.name for c in collections.collections]) + +# 检查确切名称(区分大小写) +try: + info = client.get_collection("documents") +except Exception as e: + print(f"未找到集合:{e}") +``` + +### 向量维度不匹配 + +**错误**:`ValueError: Vector dimension mismatch. Expected 384, got 768` + +**修复**: +```python +# 检查集合配置 +info = client.get_collection("documents") +print(f"期望维度:{info.config.params.vectors.size}") + +# 使用正确维度重新创建 +client.recreate_collection( + collection_name="documents", + vectors_config=VectorParams(size=768, distance=Distance.COSINE) # 匹配你的嵌入向量 +) +``` + +## 搜索问题 + +### 搜索结果为空 + +**问题**:搜索返回空结果。 + +**解决方案**: + +1. **验证数据是否存在**: +```python +info = client.get_collection("documents") +print(f"点数:{info.points_count}") + +# 滚动查看数据 +points, _ = client.scroll( + collection_name="documents", + limit=10, + with_payload=True +) +print(points) +``` + +2. **检查向量格式**: +```python +# 必须为浮点数列表 +query_vector = embedding.tolist() # 将 numpy 转换为列表 + +# 检查维度 +print(f"查询维度:{len(query_vector)}") +``` + +3. **验证过滤条件**: +```python +# 先不使用过滤器进行测试 +results = client.search( + collection_name="documents", + query_vector=query, + limit=10 + # 无过滤器 +) + +# 然后逐步添加过滤器 +``` + +### 搜索性能缓慢 + +**问题**:搜索耗时过长。 + +**解决方案**: + +1. **创建载荷索引**: +```python +# 为过滤器中使用的字段建立索引 +client.create_payload_index( + collection_name="documents", + field_name="category", + field_schema="keyword" +) +``` + +2. **启用量化**: +```python +client.update_collection( + collection_name="documents", + quantization_config=ScalarQuantization( + scalar=ScalarQuantizationConfig(type=ScalarType.INT8) + ) +) +``` + +3. **调整 HNSW 参数**: +```python +# 更快搜索(精度较低) +client.update_collection( + collection_name="documents", + hnsw_config=HnswConfigDiff(ef_construct=64, m=8) +) + +# 使用 ef 搜索参数 +results = client.search( + collection_name="documents", + query_vector=query, + search_params={"hnsw_ef": 64}, # 值越小越快 + limit=10 +) +``` + +4. **使用 gRPC**: +```python +client = QdrantClient( + host="localhost", + port=6333, + grpc_port=6334, + prefer_grpc=True +) +``` + +### 结果不一致 + +**问题**:相同查询返回不同结果。 + +**解决方案**: + +1. **等待索引完成**: +```python +client.upsert( + collection_name="documents", + points=points, + wait=True # 等待索引更新 +) +``` + +2. **检查副本一致性**: +```python +# 强一致性读取 +results = client.search( + collection_name="documents", + query_vector=query, + consistency="all" # 从所有副本读取 +) +``` + +## 数据写入问题 + +### 批量写入失败 + +**错误**:`PayloadError: Payload too large` + +**修复**: +```python +# 拆分为更小的批次 +def batch_upsert(client, collection, points, batch_size=100): + for i in range(0, len(points), batch_size): + batch = points[i:i + batch_size] + client.upsert( + collection_name=collection, + points=batch, + wait=True + ) + +batch_upsert(client, "documents", large_points_list) +``` + +### 无效的 Point ID + +**错误**:`ValueError: Invalid point ID` + +**修复**: +```python +# 有效的 ID 类型:int 或 UUID 字符串 +from uuid import uuid4 + +# 整数 ID +PointStruct(id=123, vector=vec, payload={}) + +# UUID 字符串 +PointStruct(id=str(uuid4()), vector=vec, payload={}) + +# 无效的 ID +PointStruct(id="custom-string-123", ...) # 请使用 UUID 格式 +``` + +### 载荷验证错误 + +**错误**:`ValidationError: Invalid payload` + +**修复**: +```python +# 确保载荷是 JSON 可序列化的 +import json + +payload = { + "title": "Document", + "count": 42, + "tags": ["a", "b"], + "nested": {"key": "value"} +} + +# 写入前进行验证 +json.dumps(payload) # 不应抛出异常 + +# 避免不可序列化的类型 +# 无效:datetime、numpy 数组、自定义对象 +payload = { + "timestamp": datetime.now().isoformat(), # 转换为字符串 + "vector": embedding.tolist() # 将 numpy 转换为列表 +} +``` + +## 内存问题 + +### 内存不足 + +**错误**:`MemoryError` 或容器被终止 + +**解决方案**: + +1. **启用磁盘存储**: +```python +client.create_collection( + collection_name="large_collection", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + on_disk_payload=True, # 将载荷存储在磁盘上 + hnsw_config=HnswConfigDiff(on_disk=True) # 将 HNSW 存储在磁盘上 +) +``` + +2. **使用量化**: +```python +# 减少 4 倍内存 +client.update_collection( + collection_name="large_collection", + quantization_config=ScalarQuantization( + scalar=ScalarQuantizationConfig( + type=ScalarType.INT8, + always_ram=False # 保留在磁盘上 + ) + ) +) +``` + +3. **增加 Docker 内存限制**: +```bash +docker run -m 8g -p 6333:6333 qdrant/qdrant +``` + +4. **配置 Qdrant 存储**: +```yaml +# config.yaml +storage: + performance: + max_search_threads: 2 + optimizers: + memmap_threshold_kb: 20000 +``` + +### 索引期间内存占用高 + +**修复**: +```python +# 增加批量加载的索引阈值 +client.update_collection( + collection_name="documents", + optimizer_config={ + "indexing_threshold": 50000 # 延迟索引 + } +) + +# 批量插入 +client.upsert(collection_name="documents", points=all_points, wait=False) + +# 然后优化 +client.update_collection( + collection_name="documents", + optimizer_config={ + "indexing_threshold": 10000 # 恢复常规索引 + } +) +``` + +## 集群问题 + +### 节点无法加入集群 + +**问题**:新节点无法加入集群。 + +**修复**: +```bash +# 检查网络连通性 +docker exec qdrant-node-2 ping qdrant-node-1 + +# 验证引导 URL +docker logs qdrant-node-2 | grep bootstrap + +# 检查 Raft 状态 +curl http://localhost:6333/cluster +``` + +### 脑裂 + +**问题**:集群状态不一致。 + +**修复**: +```bash +# 强制领导者选举 +curl -X POST http://localhost:6333/cluster/recover + +# 或重启少数节点 +docker restart qdrant-node-2 qdrant-node-3 +``` + +### 复制延迟 + +**问题**:副本落后。 + +**修复**: +```python +# 检查集合状态 +info = client.get_collection("documents") +print(f"状态:{info.status}") + +# 对关键写入使用强一致性 +client.upsert( + collection_name="documents", + points=points, + ordering=WriteOrdering.STRONG +) +``` + +## 性能调优 + +### 基准测试配置 + +```python +import time +import numpy as np + +def benchmark_search(client, collection, n_queries=100, dimension=384): + # 生成随机查询 + queries = [np.random.rand(dimension).tolist() for _ in range(n_queries)] + + # 预热 + for q in queries[:10]: + client.search(collection_name=collection, query_vector=q, limit=10) + + # 基准测试 + start = time.perf_counter() + for q in queries: + client.search(collection_name=collection, query_vector=q, limit=10) + elapsed = time.perf_counter() - start + + print(f"QPS:{n_queries / elapsed:.2f}") + print(f"延迟:{elapsed / n_queries * 1000:.2f}ms") + +benchmark_search(client, "documents") +``` + +### 最优 HNSW 参数 + +```python +# 高召回率(较慢) +client.create_collection( + collection_name="high_recall", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + hnsw_config=HnswConfigDiff( + m=32, # 更多连接 + ef_construct=200 # 更高构建质量 + ) +) + +# 高速度(较低召回率) +client.create_collection( + collection_name="high_speed", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + hnsw_config=HnswConfigDiff( + m=8, # 更少连接 + ef_construct=64 # 较低构建质量 + ) +) + +# 均衡配置 +client.create_collection( + collection_name="balanced", + vectors_config=VectorParams(size=384, distance=Distance.COSINE), + hnsw_config=HnswConfigDiff( + m=16, # 默认值 + ef_construct=100 # 默认值 + ) +) +``` + +## 调试技巧 + +### 启用详细日志 + +```python +import logging + +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("qdrant_client").setLevel(logging.DEBUG) +``` + +### 查看服务器日志 + +```bash +# Docker 日志 +docker logs -f qdrant + +# 带时间戳 +docker logs --timestamps qdrant + +# 最后 100 行 +docker logs --tail 100 qdrant +``` + +### 检查集合状态 + +```python +# 集合信息 +info = client.get_collection("documents") +print(f"状态:{info.status}") +print(f"点数:{info.points_count}") +print(f"分段数:{len(info.segments)}") +print(f"配置:{info.config}") + +# 采样数据点 +points, _ = client.scroll( + collection_name="documents", + limit=5, + with_payload=True, + with_vectors=True +) +for p in points: + print(f"ID:{p.id},载荷:{p.payload}") +``` + +### 测试连接 + +```python +def test_connection(host="localhost", port=6333): + try: + client = QdrantClient(host=host, port=port, timeout=5) + collections = client.get_collections() + print(f"已连接!集合数:{len(collections.collections)}") + return True + except Exception as e: + print(f"连接失败:{e}") + return False + +test_connection() +``` + +## 获取帮助 + +1. **文档**:https://qdrant.tech/documentation/ +2. **GitHub Issues**:https://github.com/qdrant/qdrant/issues +3. **Discord**:https://discord.gg/qdrant +4. **Stack Overflow**:标签 `qdrant` + +### 报告问题 + +请包含以下信息: +- Qdrant 版本:`curl http://localhost:6333/` +- Python 客户端版本:`pip show qdrant-client` +- 完整的错误回溯信息 +- 最小可复现代码 +- 集合配置