chore: import upstream snapshot with attribution
Ruff Format Check / Ruff Format & Lint (push) Failing after 7m39s
Deploy VitePress site to Pages / build (push) Failing after 9m11s
Deploy VitePress site to Pages / Deploy (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:32:26 +08:00
commit 1443d3fdf9
732 changed files with 196602 additions and 0 deletions
@@ -0,0 +1,413 @@
import asyncio
import os
from types import SimpleNamespace
import pytest
os.environ.setdefault("OPENAI_API_KEY", "test-key")
from yuxi.knowledge.eval import benchmark_generation
from yuxi.knowledge.eval.benchmark_generation import (
build_benchmark_generation_prompt,
clamp_neighbors_count,
collect_kb_chunks,
iter_generated_benchmark_items,
normalize_generation_concurrency_count,
select_graph_enhanced_chunks,
select_neighbor_chunks_by_kb_query,
)
class FakeKnowledgeBase:
pass
class FakeGenerationKnowledgeBase:
def __init__(self, query_results=None):
self.query_results = query_results or []
self.query_calls = []
async def aquery(self, query_text, kb_id, **kwargs):
self.query_calls.append({"query_text": query_text, "kb_id": kb_id, **kwargs})
return self.query_results
class FakeLlm:
def __init__(self, gold_chunk_id="anchor_chunk"):
self.gold_chunk_id = gold_chunk_id
self.prompts = []
async def call(self, prompt, stream):
self.prompts.append(prompt)
return SimpleNamespace(
content=('{"query":"问题","gold_answer":"答案","gold_chunk_ids":["' + self.gold_chunk_id + '"]}')
)
class NoQueryKnowledgeBase(FakeGenerationKnowledgeBase):
async def aquery(self, query_text, kb_id, **kwargs):
raise AssertionError("neighbors_count=1 时不应调用 aquery")
class TrackingLlm:
def __init__(self, content=None, delay=0):
self.content = content or '{"query":"问题","gold_answer":"答案","gold_chunk_ids":["anchor_chunk"]}'
self.delay = delay
self.active_calls = 0
self.max_active_calls = 0
self.calls = 0
async def call(self, prompt, stream):
self.calls += 1
self.active_calls += 1
self.max_active_calls = max(self.max_active_calls, self.active_calls)
try:
if self.delay:
await asyncio.sleep(self.delay)
return SimpleNamespace(content=self.content)
finally:
self.active_calls -= 1
class FakeGraphGenerationKnowledgeBase(FakeGenerationKnowledgeBase):
pass
def make_chunk(
chunk_id: str,
*,
kb_id: str = "db_1",
file_id: str = "file_a",
content: str = "anchor content",
chunk_index: int = 0,
graph_indexed: bool = False,
ent_ids: list[str] | None = None,
):
return SimpleNamespace(
chunk_id=chunk_id,
kb_id=kb_id,
file_id=file_id,
content=content,
chunk_index=chunk_index,
graph_indexed=graph_indexed,
ent_ids=ent_ids,
tags=None,
extraction_result=None,
)
@pytest.fixture(autouse=True)
def fake_chunk_repository(monkeypatch):
class FakeChunkRepository:
chunks = [make_chunk("anchor_chunk")]
async def list_by_kb_id(self, kb_id):
return [chunk for chunk in self.chunks if chunk.kb_id == kb_id]
monkeypatch.setattr(
"yuxi.repositories.knowledge_chunk_repository.KnowledgeChunkRepository",
FakeChunkRepository,
)
return FakeChunkRepository
def test_clamp_neighbors_count():
assert clamp_neighbors_count(-1) == 0
assert clamp_neighbors_count(3) == 3
assert clamp_neighbors_count(11) == 10
def test_normalize_generation_concurrency_count():
assert normalize_generation_concurrency_count(None) == 10
assert normalize_generation_concurrency_count("") == 10
assert normalize_generation_concurrency_count(0) == 1
assert normalize_generation_concurrency_count(-5) == 1
assert normalize_generation_concurrency_count(10000) == 20
def test_build_benchmark_generation_prompt_contains_required_schema():
prompt = build_benchmark_generation_prompt([("chunk_1", "片段内容")])
assert "片段ID=chunk_1" in prompt
assert "query、gold_answer、gold_chunk_ids" in prompt
@pytest.mark.asyncio
async def test_collect_kb_chunks_filters_kb_id(fake_chunk_repository):
fake_chunk_repository.chunks = [
make_chunk("file_a_chunk", content="内容"),
make_chunk("file_b_chunk", kb_id="db_2", file_id="file_b", content="其他"),
]
chunks = await collect_kb_chunks(FakeKnowledgeBase(), "db_1")
assert chunks == [
{
"id": "file_a_chunk",
"content": "内容",
"file_id": "file_a",
"chunk_index": 0,
"graph_indexed": False,
"ent_ids": [],
"tags": [],
"extraction_result": None,
}
]
@pytest.mark.asyncio
async def test_iter_generated_benchmark_items_with_one_chunk_does_not_query(monkeypatch):
fake_llm = FakeLlm()
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
items = [
item
async for item in iter_generated_benchmark_items(
kb_instance=NoQueryKnowledgeBase(),
kb_id="db_1",
count=1,
neighbors_count=1,
llm_model_spec="test-provider:test-model",
)
]
assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}]
assert "片段ID=anchor_chunk" in fake_llm.prompts[0]
@pytest.mark.asyncio
async def test_select_neighbor_chunks_by_kb_query_filters_anchor():
kb = FakeGenerationKnowledgeBase(
query_results=[
{
"content": "anchor content",
"metadata": {"chunk_id": "anchor_chunk", "file_id": "file_a", "chunk_index": 0},
},
{
"content": "neighbor content",
"metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1},
},
]
)
chunks = await select_neighbor_chunks_by_kb_query(
kb_instance=kb,
kb_id="db_1",
anchor_chunk={"id": "anchor_chunk", "content": "anchor content", "file_id": "file_a", "chunk_index": 0},
neighbors_count=1,
)
assert chunks == [{"id": "neighbor_chunk", "content": "neighbor content", "file_id": "file_a", "chunk_index": 1}]
assert kb.query_calls == [
{
"query_text": "anchor content",
"kb_id": "db_1",
"search_mode": "vector",
"final_top_k": 4,
"use_reranker": False,
"similarity_threshold": 0.0,
}
]
@pytest.mark.asyncio
async def test_select_graph_enhanced_chunks_expands_by_ppr_with_anchor_bias(monkeypatch):
calls = []
async def fake_rank(self, kb_id, seed_weights, *, max_nodes, top_k, damping):
calls.append(dict(seed_weights))
if len(calls) == 1:
return [("anchor", 0.9), ("neighbor_1", 0.8)]
return [("anchor", 0.9), ("neighbor_1", 0.8), ("neighbor_2", 0.7)]
monkeypatch.setattr(
"yuxi.knowledge.graphs.milvus_graph_service.MilvusGraphService.query_and_rank_chunks_by_ppr",
fake_rank,
)
chunks_by_id = {
"anchor": {"id": "anchor", "content": "anchor", "ent_ids": ["anchor_entity"]},
"neighbor_1": {"id": "neighbor_1", "content": "neighbor 1", "ent_ids": ["entity_1"]},
"neighbor_2": {"id": "neighbor_2", "content": "neighbor 2", "ent_ids": ["entity_2"]},
}
chunks = await select_graph_enhanced_chunks(
kb_id="db_1",
anchor_chunk=chunks_by_id["anchor"],
chunks_by_id=chunks_by_id,
context_count=3,
graph_expand_top_k=1,
)
assert [chunk["id"] for chunk in chunks] == ["anchor", "neighbor_1", "neighbor_2"]
assert calls[0] == {"anchor_entity": 1.0}
assert calls[1]["anchor_entity"] == 1.0
assert calls[1]["entity_1"] == 0.9
@pytest.mark.asyncio
async def test_iter_generated_benchmark_items_graph_mode_uses_graph_indexed_anchor(monkeypatch, fake_chunk_repository):
fake_chunk_repository.chunks = [
make_chunk(
"vector_anchor",
content="vector content",
chunk_index=0,
graph_indexed=False,
ent_ids=["vector_entity"],
),
make_chunk(
"graph_anchor",
content="graph anchor content",
chunk_index=1,
graph_indexed=True,
ent_ids=["anchor_entity"],
),
make_chunk(
"graph_neighbor",
content="graph neighbor content",
chunk_index=2,
graph_indexed=False,
ent_ids=["neighbor_entity"],
),
]
async def fake_rank(self, kb_id, seed_weights, *, max_nodes, top_k, damping):
assert seed_weights["anchor_entity"] == 1.0
return [("graph_anchor", 0.9), ("graph_neighbor", 0.8)]
fake_llm = FakeLlm(gold_chunk_id="graph_neighbor")
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
monkeypatch.setattr(
"yuxi.knowledge.graphs.milvus_graph_service.MilvusGraphService.query_and_rank_chunks_by_ppr",
fake_rank,
)
kb = FakeGraphGenerationKnowledgeBase()
items = [
item
async for item in iter_generated_benchmark_items(
kb_instance=kb,
kb_id="db_1",
count=1,
neighbors_count=2,
llm_model_spec="test-provider:test-model",
generation_mode="graph_enhanced",
)
]
assert items == [{"query": "问题", "gold_chunk_ids": ["graph_neighbor"], "gold_answer": "答案"}]
assert kb.query_calls == []
assert "片段ID=graph_anchor" in fake_llm.prompts[0]
assert "片段ID=graph_neighbor" in fake_llm.prompts[0]
assert "片段ID=vector_anchor" not in fake_llm.prompts[0]
@pytest.mark.asyncio
async def test_iter_generated_benchmark_items_uses_query_neighbor(monkeypatch):
fake_llm = FakeLlm(gold_chunk_id="neighbor_chunk")
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
kb = FakeGenerationKnowledgeBase(
query_results=[
{
"content": "neighbor content",
"metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1},
}
]
)
items = [
item
async for item in iter_generated_benchmark_items(
kb_instance=kb,
kb_id="db_1",
count=1,
neighbors_count=2,
llm_model_spec="test-provider:test-model",
)
]
assert items == [{"query": "问题", "gold_chunk_ids": ["neighbor_chunk"], "gold_answer": "答案"}]
assert kb.query_calls[0]["query_text"] == "anchor content"
assert kb.query_calls[0]["search_mode"] == "vector"
assert "片段ID=neighbor_chunk" in fake_llm.prompts[0]
@pytest.mark.asyncio
async def test_iter_generated_benchmark_items_falls_back_to_anchor_when_query_empty(monkeypatch):
fake_llm = FakeLlm()
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
items = [
item
async for item in iter_generated_benchmark_items(
kb_instance=FakeGenerationKnowledgeBase(query_results=[]),
kb_id="db_1",
count=1,
neighbors_count=2,
llm_model_spec="test-provider:test-model",
)
]
assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}]
assert "片段ID=anchor_chunk" in fake_llm.prompts[0]
@pytest.mark.asyncio
async def test_iter_generated_benchmark_items_respects_concurrency_count(monkeypatch):
fake_llm = TrackingLlm(delay=0.01)
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
items = [
item
async for item in iter_generated_benchmark_items(
kb_instance=NoQueryKnowledgeBase(),
kb_id="db_1",
count=4,
neighbors_count=1,
concurrency_count=2,
llm_model_spec="test-provider:test-model",
)
]
assert len(items) == 4
assert fake_llm.max_active_calls == 2
@pytest.mark.asyncio
async def test_iter_generated_benchmark_items_returns_at_most_count(monkeypatch):
fake_llm = TrackingLlm(delay=0.01)
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
items = [
item
async for item in iter_generated_benchmark_items(
kb_instance=NoQueryKnowledgeBase(),
kb_id="db_1",
count=3,
neighbors_count=1,
concurrency_count=10,
llm_model_spec="test-provider:test-model",
)
]
assert len(items) == 3
@pytest.mark.asyncio
async def test_iter_generated_benchmark_items_stops_at_max_attempts(monkeypatch):
fake_llm = TrackingLlm(content='{"query":"","gold_answer":"答案","gold_chunk_ids":["anchor_chunk"]}')
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
items = [
item
async for item in iter_generated_benchmark_items(
kb_instance=NoQueryKnowledgeBase(),
kb_id="db_1",
count=2,
neighbors_count=1,
concurrency_count=10,
llm_model_spec="test-provider:test-model",
)
]
assert items == []
assert fake_llm.calls == 50
@@ -0,0 +1,39 @@
import os
os.environ.setdefault("OPENAI_API_KEY", "test-key")
from yuxi.knowledge.eval.evaluator import aggregate_metrics, build_answer_prompt, normalize_query_result
def test_normalize_query_result_supports_dict_and_list():
answer, chunks = normalize_query_result({"answer": "A", "retrieved_chunks": [{"content": "C"}]})
assert answer == "A"
assert chunks == [{"content": "C"}]
answer, chunks = normalize_query_result([{"content": "C"}])
assert answer == ""
assert chunks == [{"content": "C"}]
def test_build_answer_prompt_uses_first_five_non_empty_chunks():
chunks = [{"content": f"内容{i}"} for i in range(6)] + [{"content": ""}]
prompt = build_answer_prompt("问题", chunks)
assert "用户问题:问题" in prompt
assert "内容0" in prompt
assert "内容4" in prompt
assert "内容5" not in prompt
def test_aggregate_metrics_matches_service_output_shape():
metrics, overall_score = aggregate_metrics(
[{"recall@1": 1.0, "f1@1": 0.0}, {"recall@1": 0.0, "f1@1": 1.0}],
[{"score": 1.0}, {"score": 0.0}],
include_overall_score=True,
)
assert metrics["recall@1"] == 0.5
assert metrics["f1@1"] == 0.5
assert metrics["answer_correctness"] == 0.5
assert metrics["overall_score"] == overall_score
@@ -0,0 +1,50 @@
import os
import pytest
os.environ.setdefault("OPENAI_API_KEY", "test-key")
from yuxi.knowledge.eval.metrics import EvaluationMetricsCalculator, RetrievalMetrics
def test_retrieval_metrics_use_metadata_chunk_id():
retrieved_chunks = [
{"metadata": {"chunk_id": "chunk_a"}},
{"metadata": {"chunk_id": "chunk_b"}},
]
metrics = EvaluationMetricsCalculator.calculate_retrieval_metrics(
retrieved_chunks, ["chunk_b", "chunk_c"], k_values=[1, 3]
)
assert metrics["recall@1"] == 0.0
assert metrics["recall@3"] == 0.5
assert metrics["f1@3"] == RetrievalMetrics.f1_score_at_k(["chunk_a", "chunk_b"], ["chunk_b", "chunk_c"], 3)
def test_overall_score_uses_answer_accuracy_when_available():
# 有答案准确率时,综合得分取各题 score 的平均,且与检索指标无关
retrieval = [{"recall@10": 1.0, "f1@10": 0.2}, {"recall@10": 0.0, "f1@10": 0.0}]
answers = [{"score": 1.0}, {"score": 0.0}, {"score": 1.0}, {"score": 1.0}]
score = EvaluationMetricsCalculator.calculate_overall_score(retrieval, answers)
assert score == 0.75
def test_overall_score_uses_recall_at_10_without_answers():
# 无答案准确率时,综合得分取各题 recall@10 的平均,不受 f1/其它 k 影响
retrieval = [
{"recall@1": 0.0, "recall@5": 0.5, "recall@10": 0.8, "f1@10": 0.1},
{"recall@1": 1.0, "recall@5": 1.0, "recall@10": 0.4, "f1@10": 0.9},
]
score = EvaluationMetricsCalculator.calculate_overall_score(retrieval, [])
assert score == pytest.approx(0.6)
def test_overall_score_returns_none_without_any_metrics():
score = EvaluationMetricsCalculator.calculate_overall_score([], [])
assert score is None
@@ -0,0 +1,133 @@
from types import SimpleNamespace
import pytest
from yuxi.knowledge.eval import service as eval_service_module
from yuxi.knowledge.eval.service import EvaluationService, build_evaluation_run_name
class FakeEvaluationRepository:
def __init__(self):
self.created_dataset = None
self.updated_dataset = None
self.dataset = None
self.created_run = None
async def create_dataset(self, payload):
self.created_dataset = payload
async def update_dataset(self, dataset_id, payload):
self.updated_dataset = (dataset_id, payload)
async def get_dataset(self, dataset_id):
return self.dataset
async def create_run(self, payload):
self.created_run = payload
class FakeChunkRepository:
def __init__(self, indexed_count):
self.indexed_count = indexed_count
async def count_graph_indexed_by_kb_id(self, kb_id):
return self.indexed_count
class FakeKnowledgeBaseRepository:
async def get_by_kb_id(self, kb_id):
return SimpleNamespace(query_params={"options": {"top_k": 3}})
@pytest.mark.asyncio
async def test_generate_dataset_saves_generation_params(monkeypatch):
async def fake_enqueue(**kwargs):
return SimpleNamespace(id="task_1")
monkeypatch.setattr(eval_service_module.tasker, "enqueue", fake_enqueue)
service = EvaluationService()
service.eval_repo = FakeEvaluationRepository()
service.chunk_repo = FakeChunkRepository(indexed_count=1)
result = await service.generate_dataset(
kb_id="db_1",
name="dataset",
description="desc",
count=2,
neighbors_count=3,
concurrency_count=4,
llm_model_spec="test:model",
generation_mode="graph_enhanced",
graph_expand_top_k=2,
created_by="user_1",
)
assert result["task_id"] == "task_1"
params = service.eval_repo.created_dataset["build_metadata"]["params"]
assert params["generation_mode"] == "graph_enhanced"
assert params["graph_expand_top_k"] == 2
updated_metadata = service.eval_repo.updated_dataset[1]["build_metadata"]
assert updated_metadata["params"] == params
@pytest.mark.asyncio
async def test_generate_dataset_rejects_graph_mode_without_indexed_chunks():
service = EvaluationService()
service.eval_repo = FakeEvaluationRepository()
service.chunk_repo = FakeChunkRepository(indexed_count=0)
with pytest.raises(ValueError, match="尚未完成图索引"):
await service.generate_dataset(
kb_id="db_1",
name="dataset",
description="desc",
count=2,
neighbors_count=3,
concurrency_count=4,
llm_model_spec="test:model",
generation_mode="graph_enhanced",
graph_expand_top_k=1,
created_by="user_1",
)
assert service.eval_repo.created_dataset is None
def test_build_evaluation_run_name_uses_eval_date_hash_format():
name = build_evaluation_run_name(hash_value="abcdef12")
assert name.startswith("eval-")
assert name.endswith("-abcdef")
assert len(name.split("-")[1]) == 8
@pytest.mark.asyncio
async def test_run_evaluation_saves_custom_name(monkeypatch):
async def fake_enqueue(**kwargs):
return SimpleNamespace(id="task_1")
monkeypatch.setattr(eval_service_module.tasker, "enqueue", fake_enqueue)
repo = FakeEvaluationRepository()
repo.dataset = SimpleNamespace(
dataset_id="dataset_1",
kb_id="db_1",
name="dataset",
item_count=2,
build_metadata={"status": "completed"},
)
service = EvaluationService()
service.eval_repo = repo
service.kb_repo = FakeKnowledgeBaseRepository()
run_id = await service.run_evaluation(
kb_id="db_1",
dataset_id="dataset_1",
name=" 回归评估 ",
model_config={"answer_llm": "test:model"},
created_by="user_1",
)
assert run_id.startswith("run_")
assert repo.created_run["name"] == "回归评估"
assert repo.created_run["retrieval_config"]["top_k"] == 3
assert repo.created_run["retrieval_config"]["answer_llm"] == "test:model"
@@ -0,0 +1,102 @@
from __future__ import annotations
import pytest
import pytest_asyncio
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from yuxi.repositories import knowledge_file_repository as repository_module
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
from yuxi.storage.postgres.models_knowledge import KnowledgeBase, KnowledgeFile
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
class _AsyncSessionContext:
def __init__(self, db):
self.db = db
async def __aenter__(self):
return self.db
async def __aexit__(self, exc_type, *_args):
if exc_type is None:
await self.db.commit()
else:
await self.db.rollback()
return False
@pytest_asyncio.fixture
async def knowledge_session(monkeypatch):
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as conn:
await conn.run_sync(KnowledgeBase.__table__.create)
await conn.run_sync(KnowledgeFile.__table__.create)
session_factory = async_sessionmaker(engine, expire_on_commit=False)
async with session_factory() as session:
monkeypatch.setattr(
repository_module.pg_manager,
"get_async_session_context",
lambda: _AsyncSessionContext(session),
)
yield session
await engine.dispose()
async def test_exists_by_filename_matches_active_file_exactly(knowledge_session):
existing_name = (
"google_drive/shared_drives/engineering/serving-runtime/dsid_e4ff04ebc2a14c1982abc4987753790c__playbook.txt"
)
knowledge_session.add_all(
[
KnowledgeBase(kb_id="kb_1", name="KB 1", description="", kb_type="milvus"),
KnowledgeBase(kb_id="kb_2", name="KB 2", description="", kb_type="milvus"),
KnowledgeFile(
file_id="file_active",
kb_id="kb_1",
filename=existing_name,
status="uploaded",
is_folder=False,
),
KnowledgeFile(
file_id="file_other_kb",
kb_id="kb_2",
filename=existing_name,
status="uploaded",
is_folder=False,
),
KnowledgeFile(
file_id="file_failed",
kb_id="kb_1",
filename="failed.txt",
status="failed",
is_folder=False,
),
KnowledgeFile(
file_id="folder_same_name",
kb_id="kb_1",
filename="folder",
status="done",
is_folder=True,
),
KnowledgeFile(
file_id="legacy_file",
kb_id="kb_1",
filename="legacy.txt",
status="indexed",
is_folder=None,
),
]
)
await knowledge_session.commit()
repo = KnowledgeFileRepository()
assert await repo.exists_by_filename(kb_id="kb_1", filename=existing_name) is True
assert await repo.exists_by_filename(kb_id="kb_1", filename=existing_name.upper()) is False
assert await repo.exists_by_filename(kb_id="kb_1", filename="failed.txt") is False
assert await repo.exists_by_filename(kb_id="kb_1", filename="folder") is False
assert await repo.exists_by_filename(kb_id="kb_1", filename="legacy.txt") is True
assert await repo.exists_by_filename(kb_id="missing", filename=existing_name) is False
@@ -0,0 +1,279 @@
from types import SimpleNamespace
import pytest
from yuxi.knowledge.manager import KnowledgeBaseManager
pytestmark = pytest.mark.asyncio
class FakeKnowledgeBaseClass:
@classmethod
def normalize_additional_params(cls, additional_params):
return dict(additional_params or {})
class FakeKnowledgeBaseRepository:
async def get_by_kb_id(self, kb_id):
if kb_id != "kb_1":
return None
return SimpleNamespace(
kb_id="kb_1",
name="知识库",
description="desc",
kb_type="milvus",
embedding_model_spec="embedding:model",
llm_model_spec="llm:model",
query_params={"options": {}},
additional_params={"chunk_preset_id": "general"},
share_config=None,
mindmap=None,
sample_questions=[],
created_at=None,
)
class FakeKnowledgeFileRepository:
list_calls = []
exists_calls = []
action_id_calls = []
def __init__(self):
self.records = [
SimpleNamespace(
file_id="folder_1",
kb_id="kb_1",
parent_id=None,
filename="资料",
file_type=None,
status="done",
is_folder=True,
path=None,
minio_url=None,
markdown_file=None,
created_at=None,
updated_at=None,
file_size=0,
),
SimpleNamespace(
file_id="file_1",
kb_id="kb_1",
parent_id=None,
filename="alpha.pdf",
file_type="pdf",
status="indexed",
is_folder=False,
path="minio://bucket/file",
minio_url="minio://bucket/file",
markdown_file="minio://bucket/parsed",
created_at=None,
updated_at=None,
file_size=1024,
),
]
async def get_kb_file_stats(self, kb_id):
return {
"row_count": 3,
"file_count": 2,
"folder_count": 1,
"total_size": 1024,
"chunk_count": 9,
"token_count": 128,
"pending_parse_count": 1,
"pending_index_count": 0,
"processing_count": 0,
}
async def get_by_file_id(self, file_id):
return next((record for record in self.records if record.file_id == file_id), None)
async def list_documents(self, **kwargs):
self.__class__.list_calls.append(kwargs)
return self.records, 2
async def list_file_ids_by_exact_statuses(self, **kwargs):
self.__class__.action_id_calls.append(kwargs)
return ["file_2"]
async def exists_by_filename(self, *, kb_id, filename):
self.__class__.exists_calls.append({"kb_id": kb_id, "filename": filename})
return filename == "docs/Guide.md"
async def count_children_by_parent_ids(self, *, kb_id, parent_ids):
return {"folder_1": 1}
@pytest.fixture(autouse=True)
def patch_repositories(monkeypatch):
FakeKnowledgeFileRepository.list_calls = []
FakeKnowledgeFileRepository.exists_calls = []
FakeKnowledgeFileRepository.action_id_calls = []
monkeypatch.setattr(
"yuxi.repositories.knowledge_base_repository.KnowledgeBaseRepository",
FakeKnowledgeBaseRepository,
)
monkeypatch.setattr(
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
FakeKnowledgeFileRepository,
)
monkeypatch.setattr(
"yuxi.knowledge.manager.KnowledgeBaseFactory.is_type_supported",
staticmethod(lambda _kb_type: True),
)
monkeypatch.setattr(
"yuxi.knowledge.manager.KnowledgeBaseFactory.get_kb_class",
staticmethod(lambda _kb_type: FakeKnowledgeBaseClass),
)
async def test_get_database_info_omits_files_by_default():
manager = KnowledgeBaseManager("/tmp/yuxi-test")
result = await manager.get_database_info("kb_1")
assert result["kb_id"] == "kb_1"
assert "files" not in result
assert result["stats"]["file_count"] == 2
assert result["stats"]["total_size"] == 1024
async def test_list_document_files_returns_lightweight_paginated_items():
manager = KnowledgeBaseManager("/tmp/yuxi-test")
result = await manager.list_document_files(
"kb_1",
parent_id="folder_1",
status="indexed",
page=2,
page_size=50,
)
assert result["page"] == 2
assert result["page_size"] == 50
assert result["total"] == 2
assert FakeKnowledgeFileRepository.list_calls == [
{
"kb_id": "kb_1",
"parent_id": "folder_1",
"path_prefix": None,
"status": "indexed",
"page": 2,
"page_size": 50,
"recursive": False,
"files_only": False,
}
]
assert result["items"][0]["has_children"] is True
assert result["items"][1]["file_size"] == 1024
assert result["items"][1]["has_original_file"] is True
assert result["items"][1]["has_parsed_markdown"] is True
returned_keys = set(result["items"][1])
assert "path" not in returned_keys
assert "markdown_file" not in returned_keys
assert "chunk_count" not in returned_keys
assert "token_count" not in returned_keys
assert "processing_params" not in returned_keys
async def test_list_document_files_keeps_virtual_folder_contract():
manager = KnowledgeBaseManager("/tmp/yuxi-test")
virtual_record = SimpleNamespace(
file_id="__virtual_folder__:root:资料/",
kb_id="kb_1",
parent_id=None,
filename="资料",
file_type="folder",
status="done",
is_folder=True,
is_virtual_folder=True,
path_prefix="资料/",
virtual_children_count=3,
path=None,
minio_url=None,
markdown_file=None,
created_at=None,
updated_at=None,
file_size=0,
)
item = manager._file_record_list_item(virtual_record)
assert item["is_folder"] is True
assert item["is_virtual_folder"] is True
assert item["path_prefix"] == "资料/"
assert item["has_children"] is True
assert item["children_count"] == 3
async def test_list_document_files_passes_files_only_and_can_omit_stats():
manager = KnowledgeBaseManager("/tmp/yuxi-test")
result = await manager.list_document_files("kb_1", files_only=True, include_stats=False)
assert "stats" not in result
assert FakeKnowledgeFileRepository.list_calls == [
{
"kb_id": "kb_1",
"parent_id": None,
"path_prefix": None,
"status": None,
"page": 1,
"page_size": 100,
"recursive": False,
"files_only": True,
}
]
async def test_list_document_files_ignores_recursive_without_status_filter():
manager = KnowledgeBaseManager("/tmp/yuxi-test")
result = await manager.list_document_files("kb_1", recursive=True, include_stats=False)
assert result["recursive"] is False
assert FakeKnowledgeFileRepository.list_calls == [
{
"kb_id": "kb_1",
"parent_id": None,
"path_prefix": None,
"status": None,
"page": 1,
"page_size": 100,
"recursive": False,
"files_only": False,
}
]
async def test_document_file_exists_delegates_exact_filename_to_repository():
manager = KnowledgeBaseManager("/tmp/yuxi-test")
assert await manager.document_file_exists("kb_1", " docs/Guide.md ") is True
assert await manager.document_file_exists("kb_1", "docs/guide.md") is False
assert FakeKnowledgeFileRepository.exists_calls == [
{"kb_id": "kb_1", "filename": "docs/Guide.md"},
{"kb_id": "kb_1", "filename": "docs/guide.md"},
]
async def test_list_document_file_ids_by_statuses_delegates_to_repository():
manager = KnowledgeBaseManager("/tmp/yuxi-test")
result = await manager.list_document_file_ids_by_statuses(
"kb_1",
statuses=["parsed", "error_indexing"],
after_file_id="file_1",
limit=500,
)
assert result == ["file_2"]
assert FakeKnowledgeFileRepository.action_id_calls == [
{
"kb_id": "kb_1",
"statuses": ["parsed", "error_indexing"],
"after_file_id": "file_1",
"limit": 500,
}
]
@@ -0,0 +1,244 @@
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@pytest.mark.unit
class TestMinIOClientStatFile:
"""Test MinIOClient.stat_file and astat_file methods."""
def test_stat_file_returns_size(self):
from yuxi.storage.minio.client import MinIOClient
client = MinIOClient()
mock_stat = MagicMock()
mock_stat.size = 1024
client._client = MagicMock()
client._client.stat_object.return_value = mock_stat
result = client.stat_file("knowledgebases", "db/upload/test.pdf")
assert result == 1024
client._client.stat_object.assert_called_once_with(
bucket_name="knowledgebases", object_name="db/upload/test.pdf"
)
def test_stat_file_returns_none_when_not_found(self):
from io import BytesIO
from minio.error import S3Error
from urllib3 import HTTPResponse
from yuxi.storage.minio.client import MinIOClient
client = MinIOClient()
client._client = MagicMock()
resp = HTTPResponse(BytesIO(b""), status=404)
client._client.stat_object.side_effect = S3Error(
resp, "NoSuchKey", "Not found", "resource", "request_id", "host_id"
)
result = client.stat_file("knowledgebases", "db/upload/missing.pdf")
assert result is None
@pytest.mark.asyncio
async def test_astat_file_returns_size(self):
from yuxi.storage.minio.client import MinIOClient
client = MinIOClient()
mock_stat = MagicMock()
mock_stat.size = 2048
client._client = MagicMock()
client._client.stat_object.return_value = mock_stat
result = await client.astat_file("knowledgebases", "db/upload/test.pdf")
assert result == 2048
@pytest.mark.unit
class TestAddFileRecordSizeFallback:
"""Test that add_file_record fills size from MinIO when not provided."""
def _make_test_kb(self, work_dir="/tmp/test_kb"):
from yuxi.knowledge.base import KnowledgeBase
class TestKB(KnowledgeBase):
@property
def kb_type(self):
return "test"
async def _create_kb_instance(self, kb_id, config):
pass
async def _initialize_kb_instance(self, instance):
pass
async def _persist_file_meta(self, file_id, meta):
pass
async def _persist_kb(self, kb_id):
pass
async def _save_metadata(self):
pass
async def refresh_database_stats(self, kb_id):
return {}
async def index_file(self, kb_id, file_id, operator_id=None):
return {}
async def aquery(self, query_text, kb_id, **kwargs):
return []
async def delete_file(self, kb_id, file_id):
pass
async def get_file_basic_info(self, kb_id, file_id):
return {}
async def get_file_content(self, kb_id, file_id):
return {}
async def get_file_info(self, kb_id, file_id):
return {}
async def update_content(self, kb_id, file_ids, params=None):
return []
async def get_query_params_config(self, kb_id, **kwargs):
return {"type": "test", "options": []}
kb = TestKB(work_dir=work_dir)
kb.databases_meta["db1"] = {"metadata": {}}
return kb
@pytest.mark.asyncio
async def test_add_file_record_fetches_size_from_minio_when_missing(self):
kb = self._make_test_kb()
item = "minio://knowledgebases/db1/upload/test_1234567890123.pdf"
params = {
"content_type": "file",
"content_hashes": {item: "abc123"},
}
mock_minio = AsyncMock()
mock_minio.astat_file.return_value = 9999
with patch("yuxi.storage.minio.get_minio_client", return_value=mock_minio):
metadata = await kb.add_file_record("db1", item, params=params)
assert metadata["size"] == 9999
mock_minio.astat_file.assert_called_once()
@pytest.mark.asyncio
async def test_add_file_record_keeps_provided_size(self):
kb = self._make_test_kb()
item = "minio://knowledgebases/db1/upload/test_1234567890123.pdf"
params = {
"content_type": "file",
"content_hashes": {item: "abc123"},
"file_sizes": {item: 5555},
}
mock_minio = AsyncMock()
with patch("yuxi.storage.minio.get_minio_client", return_value=mock_minio):
metadata = await kb.add_file_record("db1", item, params=params)
assert metadata["size"] == 5555
mock_minio.astat_file.assert_not_called()
@pytest.mark.asyncio
async def test_add_file_record_size_fallback_handles_error_gracefully(self):
kb = self._make_test_kb()
item = "minio://knowledgebases/db1/upload/test_1234567890123.pdf"
params = {
"content_type": "file",
"content_hashes": {item: "abc123"},
}
mock_minio = AsyncMock()
mock_minio.astat_file.side_effect = Exception("MinIO connection error")
with patch("yuxi.storage.minio.get_minio_client", return_value=mock_minio):
metadata = await kb.add_file_record("db1", item, params=params)
assert metadata.get("size") is None
@pytest.mark.unit
@pytest.mark.asyncio
async def test_save_metadata_does_not_overwrite_existing_kb_config():
from yuxi.knowledge.base import KnowledgeBase
class TestKB(KnowledgeBase):
@property
def kb_type(self):
return "test"
async def _create_kb_instance(self, kb_id, config):
pass
async def _initialize_kb_instance(self, instance):
pass
async def index_file(self, kb_id, file_id, operator_id=None):
return {}
async def update_content(self, kb_id, file_ids, params=None):
return []
async def aquery(self, query_text, kb_id, **kwargs):
return []
def get_query_params_config(self, kb_id, **kwargs):
return {"type": "test", "options": []}
async def delete_file(self, kb_id, file_id):
pass
async def get_file_basic_info(self, kb_id, file_id):
return {}
async def get_file_content(self, kb_id, file_id):
return {}
async def get_file_info(self, kb_id, file_id):
return {}
class ExistingKbRepo:
def __init__(self):
self.created = []
self.updated = []
async def get_by_kb_id(self, kb_id):
return SimpleNamespace(kb_id=kb_id)
async def create(self, payload):
self.created.append(payload)
async def update(self, kb_id, data):
self.updated.append((kb_id, data))
kb_repo = ExistingKbRepo()
kb = TestKB(work_dir="/tmp/test_kb")
kb.databases_meta["db1"] = {
"name": "Runtime name",
"description": "Runtime description",
"kb_type": "test",
"metadata": {"graph_build_config": {"extractor_options": {"concurrency_count": 5}}},
}
with (
patch("yuxi.repositories.knowledge_base_repository.KnowledgeBaseRepository", return_value=kb_repo),
patch("yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository", return_value=SimpleNamespace()),
patch("yuxi.repositories.evaluation_repository.EvaluationRepository", return_value=SimpleNamespace()),
):
await kb._save_metadata()
assert kb_repo.created == []
assert kb_repo.updated == []
@@ -0,0 +1,54 @@
import pytest
from yuxi.knowledge.utils.kb_utils import prepare_item_metadata
async def test_prepare_item_metadata_preserves_uploaded_file_size():
item = "minio://knowledgebases/db/upload/demo.txt"
params = {
"content_hashes": {item: "hash"},
"file_sizes": {item: 1234},
}
metadata = await prepare_item_metadata(item, "file", "db", params=params)
assert metadata["size"] == 1234
assert "file_sizes" not in (metadata.get("processing_params") or {})
async def test_prepare_item_metadata_uses_source_path_as_display_filename():
item = "minio://knowledgebases/db/upload/intro_1710000000000.md"
params = {
"content_hashes": {item: "hash"},
"source_path": "guides/setup/Intro.MD",
}
metadata = await prepare_item_metadata(item, "file", "db", params=params)
assert metadata["filename"] == "guides/setup/Intro.MD"
assert metadata["file_type"] == "md"
assert metadata["path"] == item
async def test_prepare_item_metadata_preserves_preprocessed_file_size():
item = "minio://knowledgebases/db/upload/page.html"
params = {
"_preprocessed_map": {
item: {
"path": item,
"content_hash": "hash",
"filename": "https://example.com",
"file_size": 5678,
}
}
}
metadata = await prepare_item_metadata(item, "file", "db", params=params)
assert metadata["size"] == 5678
assert "_preprocessed_map" not in (metadata.get("processing_params") or {})
async def test_prepare_item_metadata_rejects_direct_url_content_type():
with pytest.raises(ValueError, match="Unsupported content_type"):
await prepare_item_metadata("https://example.com", "url", "db")
@@ -0,0 +1,391 @@
import asyncio
import types
from yuxi.knowledge.chunking.ragflow_like.nlp import count_tokens
from yuxi.knowledge.base import KnowledgeBase
class FakeKnowledgeBase(KnowledgeBase):
@property
def kb_type(self) -> str:
return "fake"
async def _create_kb_instance(self, slug: str, config: dict):
return None
async def _initialize_kb_instance(self, instance) -> None:
pass
async def index_file(self, slug: str, file_id: str, operator_id: str | None = None) -> dict:
return {}
async def update_content(self, slug: str, file_ids: list[str], params: dict | None = None) -> list[dict]:
return []
async def aquery(self, query_text: str, slug: str, **kwargs) -> list[dict]:
return []
def get_query_params_config(self, slug: str, **kwargs) -> dict:
return {"options": []}
async def delete_file(self, slug: str, file_id: str) -> None:
pass
async def get_file_basic_info(self, slug: str, file_id: str) -> dict:
return {}
async def get_file_content(self, slug: str, file_id: str) -> dict:
return {}
async def get_file_info(self, slug: str, file_id: str) -> dict:
return {}
async def _save_metadata(self) -> None:
pass
def make_kb(tmp_path):
kb = FakeKnowledgeBase(str(tmp_path))
kb.databases_meta = {
"db": {
"name": "Old name",
"description": "Old description",
"kb_type": "fake",
"llm_model_spec": "provider:model-a",
}
}
return kb
def make_file_record(file_id: str, meta: dict):
return types.SimpleNamespace(
file_id=file_id,
kb_id=meta.get("kb_id"),
parent_id=meta.get("parent_id"),
filename=meta.get("filename", ""),
file_type=meta.get("file_type"),
path=meta.get("path"),
minio_url=meta.get("minio_url"),
markdown_file=meta.get("markdown_file"),
status=meta.get("status"),
content_hash=meta.get("content_hash"),
file_size=meta.get("size", meta.get("file_size")),
chunk_count=meta.get("chunk_count", 0),
token_count=meta.get("token_count", 0),
content_type=meta.get("content_type"),
processing_params=meta.get("processing_params"),
is_folder=meta.get("is_folder", False),
error_message=meta.get("error"),
created_by=meta.get("created_by"),
updated_by=meta.get("updated_by"),
created_at=None,
updated_at=None,
original_filename=meta.get("original_filename"),
)
class FakeFileRepository:
def __init__(self, records: dict[str, types.SimpleNamespace]):
self.records = records
self.update_calls = []
async def list_by_kb_id(self, kb_id: str):
return [record for record in self.records.values() if record.kb_id == kb_id]
async def list_by_kb_id_after(
self,
kb_id: str,
*,
after_file_id: str | None = None,
limit: int = 500,
files_only: bool = False,
):
records = [
record
for record in self.records.values()
if record.kb_id == kb_id
and (not after_file_id or record.file_id > after_file_id)
and (not files_only or not record.is_folder)
]
records.sort(key=lambda record: record.file_id)
return records[:limit]
async def update_fields(self, *, file_id: str, data: dict, kb_id: str | None = None):
record = self.records.get(file_id)
if record is None or (kb_id and record.kb_id != kb_id):
return None
for key, value in data.items():
setattr(record, key, value)
self.update_calls.append((file_id, kb_id, dict(data)))
return record
async def get_kb_file_stats(self, kb_id: str):
records = [record for record in self.records.values() if record.kb_id == kb_id]
files = [record for record in records if not record.is_folder]
return {
"row_count": len(records),
"file_count": len(files),
"folder_count": len(records) - len(files),
"total_size": sum(int(record.file_size or 0) for record in files),
"chunk_count": sum(int(record.chunk_count or 0) for record in files),
"token_count": sum(int(record.token_count or 0) for record in files),
"pending_parse_count": sum(1 for record in files if record.status == "uploaded"),
"pending_index_count": sum(1 for record in files if record.status in {"parsed", "error_indexing"}),
"processing_count": sum(1 for record in files if record.status in {"processing", "waiting", "parsing", "indexing"}),
}
def make_file_records(files: dict[str, dict]) -> dict[str, types.SimpleNamespace]:
return {file_id: make_file_record(file_id, meta) for file_id, meta in files.items()}
async def test_create_database_persists_allowed_record_fields(tmp_path, monkeypatch):
created_payloads = []
class FakeKnowledgeBaseRepository:
async def get_by_kb_id(self, kb_id):
return None
async def create(self, payload):
created_payloads.append(payload)
return types.SimpleNamespace(**payload)
async def update(self, kb_id, data):
raise AssertionError("create_database should insert new database metadata")
monkeypatch.setattr(
"yuxi.repositories.knowledge_base_repository.KnowledgeBaseRepository",
FakeKnowledgeBaseRepository,
)
kb = FakeKnowledgeBase(str(tmp_path))
share_config = {"access_level": "user", "department_ids": [], "user_uids": ["root"]}
await kb.create_database(
"New database",
"New description",
embedding_model_spec="provider:embedding",
record_fields={
"share_config": share_config,
"created_by": "root",
"unexpected_field": "ignored",
},
auto_generate_questions=False,
)
assert len(created_payloads) == 1
payload = created_payloads[0]
assert payload["share_config"] == share_config
assert payload["created_by"] == "root"
assert "unexpected_field" not in payload
assert "share_config" not in payload["additional_params"]
assert "created_by" not in payload["additional_params"]
async def test_update_database_keeps_llm_spec_when_field_is_omitted(tmp_path):
kb = make_kb(tmp_path)
result = kb.update_database("db", "New name", "New description")
await asyncio.sleep(0)
assert result["llm_model_spec"] == "provider:model-a"
assert kb.databases_meta["db"]["llm_model_spec"] == "provider:model-a"
async def test_update_database_clears_llm_spec_when_field_is_explicit(tmp_path):
kb = make_kb(tmp_path)
result = kb.update_database("db", "New name", "New description", None, update_llm_model_spec=True)
await asyncio.sleep(0)
assert result["llm_model_spec"] is None
assert kb.databases_meta["db"]["llm_model_spec"] is None
def test_get_database_info_returns_persisted_content_stats(tmp_path):
kb = make_kb(tmp_path)
kb.databases_meta["db"]["metadata"] = {
"stats": {"row_count": 3, "file_count": 2, "chunk_count": 5, "token_count": 25}
}
result = kb.get_database_info("db")
assert result["row_count"] == 3
assert result["stats"]["file_count"] == 2
assert result["stats"]["chunk_count"] == 5
assert result["stats"]["token_count"] == 25
assert result["files"] == {}
assert result["files_truncated"] is True
def test_get_database_info_prefers_metadata_stats(tmp_path):
kb = make_kb(tmp_path)
kb.databases_meta["db"]["metadata"] = {"stats": {"file_count": 2, "chunk_count": 8, "token_count": 40}}
result = kb.get_database_info("db")
assert result["stats"]["file_count"] == 2
assert result["stats"]["chunk_count"] == 8
assert result["stats"]["token_count"] == 40
async def test_refresh_database_stats_persists_metadata(tmp_path, monkeypatch):
kb = make_kb(tmp_path)
kb.databases_meta["db"]["metadata"] = {}
records = make_file_records({
"file-1": {"kb_id": "db", "filename": "alpha.md", "chunk_count": 2, "token_count": 10},
"folder-1": {
"kb_id": "db",
"filename": "folder",
"is_folder": True,
"chunk_count": 99,
"token_count": 99,
},
})
file_repo = FakeFileRepository(records)
persisted_kbs = []
async def persist_kb(kb_id):
persisted_kbs.append((kb_id, dict(kb.databases_meta[kb_id]["metadata"])))
monkeypatch.setattr(
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
lambda: file_repo,
)
kb._persist_kb = persist_kb
stats = await kb.refresh_database_stats("db")
assert stats["file_count"] == 1
assert stats["chunk_count"] == 2
assert stats["token_count"] == 10
assert kb.databases_meta["db"]["metadata"]["stats"] == stats
assert persisted_kbs == [("db", {"stats": stats})]
async def test_repair_missing_file_stats_updates_files_and_database_metadata(tmp_path, monkeypatch):
kb = make_kb(tmp_path)
kb.databases_meta["db"]["metadata"] = {}
records = make_file_records({
"file-1": {"kb_id": "db", "filename": "alpha.md", "chunk_count": 0, "token_count": 0},
"file-2": {"kb_id": "db", "filename": "beta.md", "chunk_count": 1, "token_count": 7},
"folder-1": {
"kb_id": "db",
"filename": "folder",
"is_folder": True,
"chunk_count": 99,
"token_count": 99,
},
})
file_repo = FakeFileRepository(records)
persisted_kbs = []
class FakeChunkRepo:
async def count_by_file_ids(self, file_ids):
assert file_ids == ["file-1", "file-2"]
return {"file-1": 2, "file-2": 3}
async def list_by_file_ids(self, file_ids):
assert file_ids == ["file-1"]
return [
types.SimpleNamespace(file_id="file-1", content="alpha beta"),
types.SimpleNamespace(file_id="file-1", content="中文"),
]
async def persist_kb(kb_id):
persisted_kbs.append((kb_id, dict(kb.databases_meta[kb_id]["metadata"])))
monkeypatch.setattr("yuxi.repositories.knowledge_chunk_repository.KnowledgeChunkRepository", FakeChunkRepo)
monkeypatch.setattr(
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
lambda: file_repo,
)
kb._persist_kb = persist_kb
result = await kb.repair_missing_file_stats("db")
expected_token_count = count_tokens("alpha beta") + count_tokens("中文")
expected_stats = {"file_count": 2, "chunk_count": 5, "token_count": expected_token_count + 7}
assert records["file-1"].chunk_count == 2
assert records["file-1"].token_count == expected_token_count
assert records["file-2"].chunk_count == 3
assert records["file-2"].token_count == 7
for key, value in expected_stats.items():
assert result["stats"][key] == value
assert result["scanned_token_files"] == 1
assert result["updated_chunk_files"] == 2
assert result["updated_token_files"] == 1
assert {file_id for file_id, _, _ in file_repo.update_calls} == {"file-1", "file-2"}
persisted_stats = persisted_kbs[0][1]["stats"]
for key, value in expected_stats.items():
assert persisted_stats[key] == value
async def test_repair_missing_file_stats_skips_unindexed_files(tmp_path, monkeypatch):
kb = make_kb(tmp_path)
kb.databases_meta["db"]["metadata"] = {}
records = make_file_records({
"file-indexed": {
"kb_id": "db",
"filename": "alpha.md",
"status": "indexed",
"chunk_count": 0,
"token_count": 0,
},
"file-uploaded": {
"kb_id": "db",
"filename": "beta.md",
"status": "uploaded",
"chunk_count": 9,
"token_count": 90,
},
"file-parsed": {
"kb_id": "db",
"filename": "gamma.md",
"status": "parsed",
"chunk_count": 3,
"token_count": 30,
},
})
file_repo = FakeFileRepository(records)
class FakeChunkRepo:
async def count_by_file_ids(self, file_ids):
assert file_ids == ["file-indexed"]
return {"file-indexed": 2}
async def list_by_file_ids(self, file_ids):
assert file_ids == ["file-indexed"]
return [types.SimpleNamespace(file_id="file-indexed", content="alpha beta")]
async def persist_kb(kb_id):
pass
monkeypatch.setattr("yuxi.repositories.knowledge_chunk_repository.KnowledgeChunkRepository", FakeChunkRepo)
monkeypatch.setattr(
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
lambda: file_repo,
)
kb._persist_kb = persist_kb
result = await kb.repair_missing_file_stats("db")
expected_token_count = count_tokens("alpha beta")
assert records["file-indexed"].chunk_count == 2
assert records["file-indexed"].token_count == expected_token_count
assert records["file-uploaded"].chunk_count == 0
assert records["file-uploaded"].token_count == 0
assert records["file-parsed"].chunk_count == 0
assert records["file-parsed"].token_count == 0
assert result["stats"]["file_count"] == 3
assert result["stats"]["chunk_count"] == 2
assert result["stats"]["token_count"] == expected_token_count
assert result["scanned_files"] == 3
assert result["scanned_indexed_files"] == 1
assert result["skipped_unindexed_files"] == 2
assert result["updated_files"] == 3
assert {file_id for file_id, _, _ in file_repo.update_calls} == {
"file-indexed",
"file-uploaded",
"file-parsed",
}
@@ -0,0 +1,138 @@
from types import SimpleNamespace
import pytest
from yuxi.knowledge.base import KnowledgeBase
class FakeKnowledgeBase(KnowledgeBase):
@property
def kb_type(self) -> str:
return "fake"
async def _create_kb_instance(self, kb_id: str, config: dict):
return None
async def _initialize_kb_instance(self, instance) -> None:
pass
async def index_file(self, kb_id: str, file_id: str, operator_id: str | None = None) -> dict:
return {}
async def update_content(self, kb_id: str, file_ids: list[str], params: dict | None = None) -> list[dict]:
return []
async def aquery(self, query_text: str, kb_id: str, **kwargs) -> list[dict]:
return []
def get_query_params_config(self, kb_id: str, **kwargs) -> dict:
return {"options": []}
async def delete_file(self, kb_id: str, file_id: str) -> None:
pass
async def get_file_basic_info(self, kb_id: str, file_id: str) -> dict:
return {}
async def get_file_content(self, kb_id: str, file_id: str) -> dict:
return {}
async def get_file_info(self, kb_id: str, file_id: str) -> dict:
return {}
def make_file_record(**overrides):
data = {
"file_id": "file-1",
"kb_id": "db",
"parent_id": None,
"filename": "demo.md",
"file_type": "md",
"path": "minio://knowledgebases/db/upload/demo.md",
"markdown_file": None,
"status": "uploaded",
"content_hash": "hash",
"file_size": 123,
"chunk_count": 0,
"token_count": 0,
"content_type": "file",
"processing_params": {"ocr_engine": "disable"},
"is_folder": False,
"error_message": None,
"created_by": "user",
"updated_by": None,
"created_at": None,
"updated_at": None,
"original_filename": None,
"minio_url": None,
}
data.update(overrides)
return SimpleNamespace(**data)
@pytest.mark.asyncio
async def test_load_metadata_does_not_load_file_records(monkeypatch, tmp_path):
kb = FakeKnowledgeBase(str(tmp_path))
class FakeKbRepo:
async def get_all(self):
return [
SimpleNamespace(
kb_id="db",
name="Docs",
description="",
kb_type="fake",
embedding_model_spec=None,
llm_model_spec=None,
query_params=None,
additional_params={"chunk_preset_id": "general"},
created_at=None,
)
]
def fail_resolve_processing_params(*args, **kwargs):
raise AssertionError("startup metadata loading should not normalize every file")
monkeypatch.setattr("yuxi.repositories.knowledge_base_repository.KnowledgeBaseRepository", lambda: FakeKbRepo())
monkeypatch.setattr("yuxi.knowledge.base.resolve_processing_params", fail_resolve_processing_params)
await kb._load_metadata()
assert kb._metadata_loaded is True
assert set(kb.databases_meta) == {"db"}
assert not hasattr(kb, "files_meta")
@pytest.mark.asyncio
async def test_update_file_params_lazy_loads_single_file(monkeypatch, tmp_path):
kb = FakeKnowledgeBase(str(tmp_path))
kb.databases_meta["db"] = {"metadata": {"chunk_preset_id": "general"}}
class FakeFileRepo:
def __init__(self):
self.updated = []
async def get_by_file_id(self, file_id: str):
assert file_id == "file-1"
return make_file_record()
async def update_fields(self, *, file_id: str, kb_id: str | None = None, data: dict):
self.updated.append((file_id, kb_id, data))
return make_file_record(
processing_params=data["processing_params"],
updated_by=data.get("updated_by"),
)
file_repo = FakeFileRepo()
monkeypatch.setattr("yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository", lambda: file_repo)
await kb.update_file_params("db", "file-1", {"chunk_preset_id": "qa"}, operator_id="user-2")
assert not hasattr(kb, "files_meta")
assert len(file_repo.updated) == 1
file_id, kb_id, update_data = file_repo.updated[0]
assert file_id == "file-1"
assert kb_id == "db"
assert update_data["processing_params"]["chunk_preset_id"] == "qa"
assert update_data["processing_params"]["ocr_engine"] == "disable"
assert update_data["updated_by"] == "user-2"
@@ -0,0 +1,51 @@
from yuxi.knowledge.graphs.milvus_graph_service import MilvusGraphService
from yuxi.knowledge.implementations.milvus import MilvusKB, _retrieval_config_options
def test_milvus_retrieval_config_exposes_graph_and_dependencies():
options = _retrieval_config_options()
by_key = {option["key"]: option for option in options}
assert by_key["use_graph_retrieval"]["default"] is False
assert by_key["graph_max_nodes"]["default"] == 10000
assert by_key["graph_max_nodes"]["depend_on"] == ("use_graph_retrieval", True)
assert by_key["graph_top_k"]["depend_on"] == ("use_graph_retrieval", True)
assert by_key["reranker_model"]["depend_on"] == ("use_reranker", True)
def test_graph_ppr_ranks_chunk_nodes_from_seed_entities():
subgraph = {
"nodes": [
{"id": "e1", "type": "Entity", "properties": {"entity_id": "seed"}},
{"id": "c1", "type": "Chunk", "properties": {"chunk_id": "chunk_a"}},
{"id": "e2", "type": "Entity", "properties": {"entity_id": "other"}},
{"id": "c2", "type": "Chunk", "properties": {"chunk_id": "chunk_b"}},
],
"edges": [
{"source_id": "e1", "target_id": "c1"},
{"source_id": "e1", "target_id": "e2"},
{"source_id": "e2", "target_id": "c2"},
],
}
ranked = MilvusGraphService.rank_chunks_by_ppr(subgraph, {"seed": 1.0}, top_k=2, damping=0.85)
assert [chunk_id for chunk_id, _ in ranked] == ["chunk_a", "chunk_b"]
def test_rrf_fusion_merges_chunk_and_graph_rankings():
kb = object.__new__(MilvusKB)
base_chunks = [
{"content": "base a", "metadata": {"chunk_id": "a"}, "score": 0.9},
{"content": "base b", "metadata": {"chunk_id": "b"}, "score": 0.8},
]
graph_chunks = [
{"content": "graph b", "metadata": {"chunk_id": "b"}, "score": 0.7, "graph_score": 0.7},
{"content": "graph c", "metadata": {"chunk_id": "c"}, "score": 0.6, "graph_score": 0.6},
]
fused = kb._fuse_chunk_rankings(base_chunks, graph_chunks, graph_weight=1.0)
assert [chunk["metadata"]["chunk_id"] for chunk in fused] == ["b", "a", "c"]
assert fused[0]["graph_score"] == 0.7
assert fused[0]["fusion_sources"] == ["chunk", "graph"]
@@ -0,0 +1,118 @@
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
from yuxi.knowledge.utils import mindmap_utils as mm
def make_kb(**overrides):
data = {
"kb_id": "kb_1",
"name": "知识库",
"mindmap": {"content": "知识库", "children": [{"content": "tracked.pdf", "children": []}]},
"mindmap_file_ids": {"tracked": "tracked.pdf"},
"mindmap_metadata": {},
}
data.update(overrides)
return SimpleNamespace(**data)
def make_file(file_id: str, filename: str, *, kb_id: str = "kb_1"):
return SimpleNamespace(
file_id=file_id,
kb_id=kb_id,
filename=filename,
file_type="pdf",
status="indexed",
is_folder=False,
created_at=None,
)
class FakeKnowledgeBaseRepository:
def __init__(self, kb):
self.kb = kb
self.updates = []
async def get_by_kb_id(self, kb_id):
return self.kb if kb_id == self.kb.kb_id else None
async def update(self, kb_id, data):
self.updates.append((kb_id, data))
@pytest.mark.asyncio
async def test_get_mindmap_diff_keeps_tracked_file_outside_first_page(monkeypatch):
kb_repo = FakeKnowledgeBaseRepository(make_kb())
class FakeFileRepository:
async def list_documents(self, **kwargs):
assert kwargs["page_size"] == mm.MINDMAP_FILE_PAGE_SIZE
return [make_file("new", "new.pdf")], 100
async def list_by_file_ids(self, file_ids):
assert file_ids == ["tracked"]
return [make_file("tracked", "tracked.pdf")]
monkeypatch.setattr(mm, "KnowledgeBaseRepository", lambda: kb_repo)
monkeypatch.setattr(
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
FakeFileRepository,
)
result = await mm.get_mindmap_diff("kb_1")
assert result["removed_file_ids"] == []
assert result["unchanged_count"] == 1
assert result["added_files"] == [{"file_id": "new", "filename": "new.pdf", "type": "pdf"}]
assert result["current_files_truncated"] is True
@pytest.mark.asyncio
async def test_generate_database_mindmap_loads_selected_file_ids_directly(monkeypatch):
kb_repo = FakeKnowledgeBaseRepository(make_kb(mindmap=None, mindmap_file_ids=None))
class FakeFileRepository:
async def list_documents(self, **kwargs):
raise AssertionError("selected file generation should query by file id")
async def list_by_file_ids(self, file_ids):
assert file_ids == ["outside-page"]
return [make_file("outside-page", "outside.pdf")]
class FakeModel:
async def call(self, messages, stream):
assert "outside.pdf" in messages[1]["content"]
return SimpleNamespace(content='{"content":"知识库","children":[{"content":"outside.pdf","children":[]}]}')
monkeypatch.setattr(mm, "KnowledgeBaseRepository", lambda: kb_repo)
monkeypatch.setattr(
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
FakeFileRepository,
)
monkeypatch.setattr(mm, "select_model", lambda model_spec: FakeModel())
result = await mm.generate_database_mindmap("kb_1", file_ids=["outside-page"])
assert result["file_count"] == 1
assert result["original_file_count"] == 1
assert kb_repo.updates[0][1]["mindmap_file_ids"] == {"outside-page": "outside.pdf"}
@pytest.mark.asyncio
async def test_generate_database_mindmap_rejects_missing_selected_files(monkeypatch):
kb_repo = FakeKnowledgeBaseRepository(make_kb(mindmap=None, mindmap_file_ids=None))
class FakeFileRepository:
async def list_by_file_ids(self, file_ids):
return []
monkeypatch.setattr(mm, "KnowledgeBaseRepository", lambda: kb_repo)
monkeypatch.setattr(
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
FakeFileRepository,
)
with pytest.raises(HTTPException, match="选择的文件不存在"):
await mm.generate_database_mindmap("kb_1", file_ids=["missing"])
@@ -0,0 +1,171 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
from yuxi.knowledge.base import KnowledgeBase
from yuxi.services.file_preview import MAX_BINARY_PREVIEW_SIZE_BYTES
class FakeKnowledgeBase(KnowledgeBase):
@property
def kb_type(self) -> str:
return "fake"
async def _create_kb_instance(self, kb_id: str, config: dict):
return None
async def _initialize_kb_instance(self, instance) -> None:
pass
async def index_file(self, kb_id: str, file_id: str, operator_id: str | None = None) -> dict:
return {}
async def update_content(self, kb_id: str, file_ids: list[str], params: dict | None = None) -> list[dict]:
return []
async def aquery(self, query_text: str, kb_id: str, **kwargs) -> list[dict]:
return []
def get_query_params_config(self, kb_id: str, **kwargs) -> dict:
return {"options": []}
async def delete_file(self, kb_id: str, file_id: str) -> None:
pass
async def get_file_basic_info(self, kb_id: str, file_id: str) -> dict:
return {}
async def get_file_content(self, kb_id: str, file_id: str) -> dict:
return {}
async def get_file_info(self, kb_id: str, file_id: str) -> dict:
return {}
async def _save_metadata(self) -> None:
pass
class FakeMinioClient:
KB_BUCKETS = {"parsed": "knowledgebases"}
def __init__(self) -> None:
self.objects: dict[tuple[str, str], bytes] = {}
async def astat_file(self, bucket_name: str, object_name: str) -> int | None:
content = self.objects.get((bucket_name, object_name))
return len(content) if content is not None else None
async def adownload_file(self, bucket_name: str, object_name: str) -> bytes:
return self.objects[(bucket_name, object_name)]
async def aupload_file(
self,
bucket_name: str,
object_name: str,
data: bytes,
content_type: str | None = None,
) -> SimpleNamespace:
assert content_type == "application/pdf"
self.objects[(bucket_name, object_name)] = data
return SimpleNamespace(url=f"http://localhost:9000/{bucket_name}/{object_name}")
def make_kb(tmp_path) -> FakeKnowledgeBase:
kb = FakeKnowledgeBase(str(tmp_path))
kb.databases_meta["db1"] = {"metadata": {}}
kb.test_file_meta = {
"file_id": "file1",
"kb_id": "db1",
"filename": "demo.docx",
"path": "minio://knowledgebases/db1/upload/demo.docx",
"markdown_file": "minio://knowledgebases/db1/parsed/file1.md",
"status": "parsed",
}
async def load_file_meta(kb_id: str, file_id: str, *, refresh: bool = False) -> dict:
assert kb_id == "db1"
assert file_id == "file1"
return kb.test_file_meta
kb._load_file_meta = load_file_meta
return kb
def test_office_file_entry_exposes_logical_file_availability(tmp_path) -> None:
kb = make_kb(tmp_path)
entry = kb._knowledge_file_entry("db1", "file1", kb.test_file_meta)
assert entry["has_original_file"] is True
assert entry["has_parsed_markdown"] is True
assert "preview_modes" not in entry
assert "default_preview_mode" not in entry
@pytest.mark.asyncio
async def test_read_office_pdf_preview_converts_and_caches_pdf(tmp_path, monkeypatch) -> None:
kb = make_kb(tmp_path)
minio_client = FakeMinioClient()
minio_client.objects[("knowledgebases", "db1/upload/demo.docx")] = b"office"
minio_client.objects[("knowledgebases", "db1/parsed/file1.md")] = b"# parsed"
convert_calls = 0
async def fake_convert(filename: str, content: bytes) -> bytes:
nonlocal convert_calls
convert_calls += 1
assert filename == "demo.docx"
assert content == b"office"
return b"%PDF-1.4\nconverted"
monkeypatch.setattr("yuxi.storage.minio.get_minio_client", lambda: minio_client)
monkeypatch.setattr("yuxi.knowledge.base.convert_office_to_pdf", fake_convert)
response = await kb.read_file_preview("db1", "file1")
cached_response = await kb.read_file_preview("db1", "file1")
assert response["preview_type"] == "pdf"
assert response["supported"] is True
assert response["binary"] is True
assert response["content"] == b"%PDF-1.4\nconverted"
assert response["media_type"] == "application/pdf"
assert cached_response["content"] == b"%PDF-1.4\nconverted"
assert minio_client.objects[("knowledgebases", "db1/preview/file1.pdf")] == b"%PDF-1.4\nconverted"
assert convert_calls == 1
@pytest.mark.asyncio
async def test_non_docx_pptx_office_files_do_not_get_pdf_preview(tmp_path, monkeypatch) -> None:
kb = make_kb(tmp_path)
kb.test_file_meta["filename"] = "demo.xlsx"
minio_client = FakeMinioClient()
minio_client.objects[("knowledgebases", "db1/upload/demo.docx")] = b"PK\x03\x04excel"
monkeypatch.setattr("yuxi.storage.minio.get_minio_client", lambda: minio_client)
entry = kb._knowledge_file_entry("db1", "file1", kb.test_file_meta)
response = await kb.read_file_preview("db1", "file1")
assert entry["has_original_file"] is True
assert entry["has_parsed_markdown"] is True
assert response["preview_type"] == "unsupported"
assert response["supported"] is False
@pytest.mark.asyncio
async def test_read_file_preview_rejects_large_original_before_download(tmp_path, monkeypatch) -> None:
kb = make_kb(tmp_path)
kb.test_file_meta["filename"] = "large.pdf"
kb.test_file_meta["size"] = MAX_BINARY_PREVIEW_SIZE_BYTES + 1
async def fail_read(_path: str) -> bytes:
raise AssertionError("large preview should not download file content")
monkeypatch.setattr(kb, "_read_minio_bytes", fail_read)
response = await kb.read_file_preview("db1", "file1")
assert response["preview_type"] == "unsupported"
assert response["supported"] is False
assert response["limit"] == MAX_BINARY_PREVIEW_SIZE_BYTES
@@ -0,0 +1,289 @@
from __future__ import annotations
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import pytest
import yuxi.knowledge.parser.paddleocr_api as paddleocr_api
from yuxi.knowledge.parser.base import DocumentParserException
from yuxi.knowledge.parser.paddleocr_api import PaddleOCRPPOCRv6Parser, PaddleOCRVLParser
@dataclass
class FakeResponse:
status_code: int
json_body: dict[str, Any] | None = None
text: str = ""
content: bytes = b""
headers: dict[str, str] | None = None
def json(self) -> dict[str, Any]:
assert self.json_body is not None
return self.json_body
def _build_file(tmp_path: Path, suffix: str = ".png") -> Path:
file_path = tmp_path / f"sample{suffix}"
file_path.write_bytes(b"fake image")
return file_path
def test_paddleocr_vl_submits_model_specific_payload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
file_path = _build_file(tmp_path)
submitted: dict[str, Any] = {}
def fake_post(url, headers=None, data=None, files=None, json=None, timeout=None): # noqa: A002
submitted["url"] = url
submitted["data"] = data
submitted["json"] = json
submitted["files"] = files
return FakeResponse(200, {"code": 0, "data": {"jobId": "job-vl"}})
def fake_get(url, headers=None, timeout=None):
if url.endswith("/job-vl"):
return FakeResponse(200, {"data": {"state": "done", "resultUrl": {"jsonUrl": "https://result.test/vl"}}})
row = {
"result": {
"layoutParsingResults": [
{"markdown": {"text": "VL markdown", "images": {}}, "outputImages": {"debug": "ignored"}}
]
}
}
return FakeResponse(200, text=json.dumps(row))
monkeypatch.setattr(paddleocr_api.requests, "post", fake_post)
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
parser = PaddleOCRVLParser(api_token="token")
result = parser.process_file(
str(file_path),
params={
"optional_payload": {
"useChartRecognition": True,
"useTextlineOrientation": True,
"ignored": True,
}
},
)
assert result == "VL markdown"
assert submitted["data"]["model"] == "PaddleOCR-VL-1.6"
optional_payload = json.loads(submitted["data"]["optionalPayload"])
assert optional_payload == {
"useDocOrientationClassify": False,
"useDocUnwarping": False,
"useChartRecognition": True,
}
def test_paddleocr_pp_ocrv6_submits_model_specific_payload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
file_path = _build_file(tmp_path)
submitted: dict[str, Any] = {}
def fake_post(url, headers=None, data=None, files=None, json=None, timeout=None): # noqa: A002
submitted["data"] = data
return FakeResponse(200, {"data": {"jobId": "job-ocr"}})
def fake_get(url, headers=None, timeout=None):
if url.endswith("/job-ocr"):
return FakeResponse(200, {"data": {"state": "done", "resultUrl": {"jsonUrl": "https://result.test/ocr"}}})
row = {
"result": {
"ocrResults": [
{"prunedResult": {"rec_texts": ["PaddleOCR API Test", "", "Invoice total: 123.45"]}}
]
}
}
return FakeResponse(200, text=json.dumps(row))
monkeypatch.setattr(paddleocr_api.requests, "post", fake_post)
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
parser = PaddleOCRPPOCRv6Parser(api_token="token")
result = parser.process_file(
str(file_path),
params={
"optional_payload": {
"useTextlineOrientation": True,
"useChartRecognition": True,
"ignored": True,
}
},
)
assert result == "PaddleOCR API Test\nInvoice total: 123.45"
assert submitted["data"]["model"] == "PP-OCRv6"
optional_payload = json.loads(submitted["data"]["optionalPayload"])
assert optional_payload == {
"useDocOrientationClassify": False,
"useDocUnwarping": False,
"useTextlineOrientation": True,
}
def test_paddleocr_url_input_uses_json_payload(monkeypatch: pytest.MonkeyPatch) -> None:
submitted: dict[str, Any] = {}
def fake_post(url, headers=None, data=None, files=None, json=None, timeout=None): # noqa: A002
submitted["headers"] = headers
submitted["data"] = data
submitted["files"] = files
submitted["json"] = json
return FakeResponse(200, {"data": {"jobId": "job-url"}})
def fake_get(url, headers=None, timeout=None):
if url.endswith("/job-url"):
return FakeResponse(200, {"data": {"state": "done", "resultUrl": {"jsonUrl": "https://result.test/url"}}})
row = {"result": {"ocrResults": [{"prunedResult": {"rec_texts": ["url text"]}}]}}
return FakeResponse(200, text=json.dumps(row))
monkeypatch.setattr(paddleocr_api.requests, "post", fake_post)
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
parser = PaddleOCRPPOCRv6Parser(api_token="token")
result = parser.process_file("https://example.test/file.png?signature=abc")
assert result == "url text"
assert submitted["headers"]["Content-Type"] == "application/json"
assert submitted["data"] is None
assert submitted["files"] is None
assert submitted["json"]["fileUrl"] == "https://example.test/file.png?signature=abc"
assert submitted["json"]["model"] == "PP-OCRv6"
def test_paddleocr_poll_handles_pending_running_and_done(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
file_path = _build_file(tmp_path)
states = iter(["pending", "running", "done"])
sleep_calls: list[float] = []
monkeypatch.setattr(
paddleocr_api.requests,
"post",
lambda *args, **kwargs: FakeResponse(200, {"data": {"jobId": "job-poll"}}),
)
def fake_get(url, headers=None, timeout=None):
if url.endswith("/job-poll"):
state = next(states)
data: dict[str, Any] = {"state": state}
if state == "done":
data["resultUrl"] = {"jsonUrl": "https://result.test/poll"}
return FakeResponse(200, {"data": data})
row = {"result": {"ocrResults": [{"prunedResult": {"rec_texts": ["done text"]}}]}}
return FakeResponse(200, text=json.dumps(row))
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
monkeypatch.setattr(paddleocr_api.time, "sleep", lambda seconds: sleep_calls.append(seconds))
parser = PaddleOCRPPOCRv6Parser(api_token="token")
result = parser.process_file(str(file_path), params={"poll_interval_seconds": 0.25})
assert result == "done text"
assert sleep_calls == [0.25, 0.25]
def test_paddleocr_failed_job_raises_parser_exception(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
file_path = _build_file(tmp_path)
monkeypatch.setattr(
paddleocr_api.requests,
"post",
lambda *args, **kwargs: FakeResponse(200, {"data": {"jobId": "job-failed"}}),
)
monkeypatch.setattr(
paddleocr_api.requests,
"get",
lambda *args, **kwargs: FakeResponse(
200,
{"data": {"state": "failed", "errorMsg": "quota exceeded"}},
),
)
parser = PaddleOCRVLParser(api_token="token")
with pytest.raises(DocumentParserException, match="quota exceeded"):
parser.process_file(str(file_path))
def test_paddleocr_missing_token_health_and_parse_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
file_path = _build_file(tmp_path)
monkeypatch.delenv("PADDLEOCR_API_TOKEN", raising=False)
parser = PaddleOCRVLParser()
health = parser.check_health()
assert health["status"] == "unavailable"
assert "PADDLEOCR_API_TOKEN" in health["message"]
with pytest.raises(DocumentParserException, match="PADDLEOCR_API_TOKEN"):
parser.process_file(str(file_path))
def test_paddleocr_configured_token_health_does_not_submit_job() -> None:
parser = PaddleOCRVLParser(api_token="token")
health = parser.check_health()
assert health["status"] == "configured"
assert "解析时验证" in health["message"]
assert health["details"]["model"] == "PaddleOCR-VL-1.6"
def test_paddleocr_vl_uploads_markdown_images(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
file_path = _build_file(tmp_path)
uploaded: dict[str, Any] = {}
monkeypatch.setattr(
paddleocr_api.requests,
"post",
lambda *args, **kwargs: FakeResponse(200, {"data": {"jobId": "job-images"}}),
)
def fake_get(url, headers=None, timeout=None):
if url.endswith("/job-images"):
return FakeResponse(
200,
{"data": {"state": "done", "resultUrl": {"jsonUrl": "https://result.test/images"}}},
)
if url == "https://image.test/table.png":
return FakeResponse(200, content=b"image-bytes", headers={"Content-Type": "image/png"})
row = {
"result": {
"layoutParsingResults": [
{
"markdown": {
"text": "before ![](images/table.png) after",
"images": {"images/table.png": "https://image.test/table.png"},
}
}
]
}
}
return FakeResponse(200, text=json.dumps(row))
class FakeMinioClient:
def ensure_bucket_exists(self, bucket_name):
uploaded["bucket_name"] = bucket_name
def upload_file(self, bucket_name, object_name, data):
uploaded["object_name"] = object_name
uploaded["data"] = data
return type("UploadResult", (), {"url": "minio://public/kb/table.png"})()
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
monkeypatch.setattr(paddleocr_api, "get_minio_client", lambda: FakeMinioClient())
monkeypatch.setattr(paddleocr_api.time, "time", lambda: 1.0)
parser = PaddleOCRVLParser(api_token="token")
result = parser.process_file(
str(file_path),
params={"image_bucket": "public", "image_prefix": "kb/images"},
)
assert result == "before ![](minio://public/kb/table.png) after"
assert uploaded == {
"bucket_name": "public",
"object_name": "kb/images/1000000_table.png",
"data": b"image-bytes",
}
@@ -0,0 +1,322 @@
from __future__ import annotations
import asyncio
import base64
import time
from pathlib import Path
from types import SimpleNamespace
import fitz
import pandas as pd
import pytest
import yuxi.knowledge.parser.unified as parser_unified
from docx import Document
from PIL import Image
from yuxi.knowledge.parser import Parser
from yuxi.knowledge.parser.factory import DocumentProcessorFactory
DATA_DIR = Path(__file__).resolve().parents[2] / "data"
def _build_pdf(file_path: Path, text: str) -> None:
doc = fitz.open()
page = doc.new_page()
page.insert_text((72, 72), text)
doc.save(str(file_path))
doc.close()
def _build_docx(file_path: Path, text: str) -> None:
document = Document()
document.add_paragraph(text)
document.save(str(file_path))
def _build_png(file_path: Path) -> None:
image = Image.new("RGB", (120, 80), "white")
image.save(str(file_path))
def test_parser_parse_pdf_file_returns_markdown_text(tmp_path: Path):
file_path = tmp_path / "parser_test.pdf"
_build_pdf(file_path, "Parser PDF content")
markdown = Parser.parse(str(file_path), params={"ocr_engine": "disable"})
assert isinstance(markdown, str)
assert "Parser" in markdown
assert "content" in markdown
assert len(markdown.strip()) > 0
def test_parser_parse_docx_file_returns_markdown_text(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
file_path = tmp_path / "parser_test.docx"
_build_docx(file_path, "Parser DOCX content")
# 避免测试依赖 docling 行为,直接验证统一 parser 可回退到 python-docx。
def _raise_docling_error(*args, **kwargs):
raise RuntimeError("force fallback to python-docx")
monkeypatch.setattr(parser_unified, "_convert_with_docling", _raise_docling_error)
markdown = Parser.parse(str(file_path))
assert isinstance(markdown, str)
assert "Parser DOCX content" in markdown
assert len(markdown.strip()) > 0
def test_convert_csv_to_markdown_preserves_column_dtypes(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
file_path = tmp_path / "parser_test.csv"
file_path.write_text("id,score\n9007199254740993,2.5\n", encoding="utf-8")
captured_dtypes: list[dict[str, object]] = []
original_to_markdown = pd.DataFrame.to_markdown
def _capture_dtypes(dataframe: pd.DataFrame, *args, **kwargs) -> str:
captured_dtypes.append(dataframe.dtypes.to_dict())
return original_to_markdown(dataframe, *args, **kwargs)
monkeypatch.setattr(pd.DataFrame, "to_markdown", _capture_dtypes)
markdown = parser_unified._convert_csv_to_markdown(file_path)
assert markdown
assert str(captured_dtypes[0]["id"]) == "int64"
def test_convert_with_docling_reinserts_image_links_in_document_order(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
):
file_path = tmp_path / "parser_test.docx"
file_path.write_bytes(b"fake docx")
first_image = base64.b64encode(b"first image").decode()
second_image = base64.b64encode(b"second image").decode()
fake_doc = SimpleNamespace(
pictures=[
SimpleNamespace(image=SimpleNamespace(uri=f"data:image/png;base64,{first_image}")),
SimpleNamespace(image=SimpleNamespace(uri="https://example.test/remote.png")),
SimpleNamespace(image=SimpleNamespace(uri=f"data:image/png;base64,{second_image}")),
],
export_to_markdown=lambda: "before\n<!-- image -->\nremote\n<!-- image -->\nbetween\n<!-- image -->\nafter",
)
fake_result = SimpleNamespace(status=SimpleNamespace(name="SUCCESS"), document=fake_doc)
uploaded_images: list[bytes] = []
class FakeConverter:
def convert(self, path: Path):
assert path == file_path
return fake_result
def _fake_upload_image_to_minio(image_data, filename, bucket_name, object_prefix):
uploaded_images.append(image_data)
return f"https://example.test/{len(uploaded_images)}.png"
monkeypatch.setattr(parser_unified, "_get_docling_converter", lambda: FakeConverter())
monkeypatch.setattr(parser_unified, "_upload_image_to_minio", _fake_upload_image_to_minio)
image_timestamps = iter([1.0, 2.0])
monkeypatch.setattr(parser_unified.time, "time", lambda: next(image_timestamps))
markdown = parser_unified._convert_with_docling(file_path)
assert uploaded_images == [b"first image", b"second image"]
assert markdown == (
"before\n"
"![image_1000000.png](https://example.test/1.png)\n"
"remote\n"
"\n"
"between\n"
"![image_2000000.png](https://example.test/2.png)\n"
"after"
)
def test_convert_with_docling_keeps_image_placeholder_when_upload_fails(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
):
file_path = tmp_path / "parser_test.docx"
file_path.write_bytes(b"fake docx")
image = base64.b64encode(b"image data").decode()
fake_doc = SimpleNamespace(
pictures=[SimpleNamespace(image=SimpleNamespace(uri=f"data:image/png;base64,{image}"))],
export_to_markdown=lambda: "before\n<!-- image -->\nafter",
)
fake_result = SimpleNamespace(status=SimpleNamespace(name="SUCCESS"), document=fake_doc)
class FakeConverter:
def convert(self, path: Path):
assert path == file_path
return fake_result
def _raise_upload_error(*args, **kwargs):
raise RuntimeError("upload failed")
monkeypatch.setattr(parser_unified, "_get_docling_converter", lambda: FakeConverter())
monkeypatch.setattr(parser_unified, "_upload_image_to_minio", _raise_upload_error)
monkeypatch.setattr(parser_unified.time, "time", lambda: 1.0)
markdown = parser_unified._convert_with_docling(file_path)
assert markdown == "before\n[图片: image_1000000.png]\nafter"
def test_parser_parse_png_file_returns_markdown_text_with_mocked_ocr(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
):
file_path = tmp_path / "parser_test.png"
_build_png(file_path)
async def _fake_parse_image_async(file, params=None):
return "Parser PNG content"
monkeypatch.setattr(parser_unified, "parse_image_async", _fake_parse_image_async)
markdown = Parser.parse(str(file_path), params={"ocr_engine": "rapid_ocr"})
assert isinstance(markdown, str)
assert "Parser PNG content" in markdown
assert len(markdown.strip()) > 0
def test_parse_image_uses_ocr_engine_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
file_path = tmp_path / "parser_test.png"
_build_png(file_path)
captured = {}
def _fake_process_file(processor_type, file, params=None):
captured["processor_type"] = processor_type
captured["file"] = file
captured["params"] = params
return "OCR content"
monkeypatch.setattr(DocumentProcessorFactory, "process_file", _fake_process_file)
result = parser_unified.parse_image(
str(file_path),
params={
"ocr_engine": "mineru_ocr",
"backend": "old-backend",
"ocr_engine_config": {"backend": "pipeline", "formula_enable": False},
},
)
assert result == "OCR content"
assert captured["processor_type"] == "mineru_ocr"
assert captured["file"] == str(file_path)
assert captured["params"]["backend"] == "pipeline"
assert captured["params"]["formula_enable"] is False
def test_parse_image_ignores_enable_ocr(tmp_path: Path) -> None:
file_path = tmp_path / "parser_test.png"
_build_png(file_path)
with pytest.raises(ValueError, match="必须启用OCR"):
parser_unified.parse_image(str(file_path), params={"ocr_engine": "disable", "enable_ocr": "rapid_ocr"})
@pytest.mark.asyncio
async def test_parser_aparse_pdf_file_returns_markdown_text(tmp_path: Path):
file_path = tmp_path / "parser_test_async.pdf"
_build_pdf(file_path, "Async Parser PDF content")
markdown = await Parser.aparse(str(file_path), params={"ocr_engine": "disable"})
assert isinstance(markdown, str)
assert "Async" in markdown
assert "content" in markdown
assert len(markdown.strip()) > 0
@pytest.mark.asyncio
async def test_parser_aparse_docx_does_not_block_event_loop(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
file_path = tmp_path / "parser_test_async.docx"
file_path.write_bytes(b"fake docx")
completion_order: list[str] = []
def _slow_docling_conversion(*args, **kwargs) -> str:
time.sleep(0.1)
return "Async DOCX content"
async def _parse_document() -> None:
await Parser.aparse(str(file_path))
completion_order.append("parse")
async def _record_event_loop_progress() -> None:
await asyncio.sleep(0.01)
completion_order.append("event_loop")
monkeypatch.setattr(parser_unified, "_convert_with_docling", _slow_docling_conversion)
await asyncio.gather(_parse_document(), _record_event_loop_progress())
assert completion_order == ["event_loop", "parse"]
def test_parse_pdf_uses_config_default_ocr_when_engine_missing(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
import yuxi
file_path = tmp_path / "parser_test.pdf"
_build_pdf(file_path, "Parser PDF content")
captured = {}
def _fake_process_file(processor_type, file, params=None):
captured["processor_type"] = processor_type
captured["file"] = file
captured["params"] = params
return "default OCR content"
monkeypatch.setattr(yuxi.config, "default_ocr_engine", "mineru_ocr")
monkeypatch.setattr(DocumentProcessorFactory, "process_file", _fake_process_file)
result = parser_unified.parse_pdf(str(file_path), params={})
assert result == "default OCR content"
assert captured["processor_type"] == "mineru_ocr"
assert captured["file"] == str(file_path)
def test_parse_pdf_keeps_explicit_disable_when_default_ocr_enabled(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
import yuxi
file_path = tmp_path / "parser_test.pdf"
_build_pdf(file_path, "Parser PDF content")
monkeypatch.setattr(yuxi.config, "default_ocr_engine", "mineru_ocr")
result = parser_unified.parse_pdf(str(file_path), params={"ocr_engine": "disable"})
assert "Parser PDF content" in result
@pytest.mark.asyncio
async def test_parser_aparse_image_file_with_mineru_when_available():
file_path = DATA_DIR / "测试图片.png"
assert file_path.exists(), f"测试文件不存在: {file_path}"
health = await asyncio.to_thread(DocumentProcessorFactory.check_health, "mineru_ocr")
if health.get("status") != "healthy":
pytest.skip(f"mineru_ocr 不可用: {health.get('message', 'unknown')}")
markdown = await Parser.aparse(
str(file_path),
params={"ocr_engine": "mineru_ocr", "backend": "pipeline"},
)
assert isinstance(markdown, str)
assert len(markdown) > 100
assert len(markdown.strip()) > 0
@@ -0,0 +1,108 @@
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
from yuxi.knowledge.utils import sample_question_utils as sq
def test_parse_sample_questions_content_strips_json_fence():
questions = sq.parse_sample_questions_content('```json\n{"questions": ["什么是测试?"]}\n```')
assert questions == ["什么是测试?"]
def test_parse_sample_questions_content_rejects_invalid_payload():
with pytest.raises(ValueError, match="问题格式"):
sq.parse_sample_questions_content('{"items": []}')
@pytest.mark.asyncio
async def test_generate_database_sample_questions_rejects_empty_files(monkeypatch):
class FakeKnowledgeBase:
async def get_database_info(self, kb_id: str, include_files: bool = False) -> dict:
return {"name": "空知识库", "kb_type": "milvus", "files": {}}
monkeypatch.setattr(sq, "knowledge_base", FakeKnowledgeBase())
monkeypatch.setattr(
sq.KnowledgeBaseFactory,
"get_kb_class",
lambda _kb_type: SimpleNamespace(supports_documents=True),
)
with pytest.raises(HTTPException) as exc_info:
await sq.generate_database_sample_questions("kb_1")
assert exc_info.value.status_code == 400
assert "没有文件" in exc_info.value.detail
@pytest.mark.asyncio
async def test_generate_database_sample_questions_saves_and_returns_questions(monkeypatch):
saved: dict = {}
class FakeKnowledgeBase:
async def get_database_info(self, kb_id: str, include_files: bool = False) -> dict:
return {
"name": "测试知识库",
"kb_type": "milvus",
"files": {"file_1": {"filename": "demo.md", "file_type": "md"}},
}
class FakeModel:
async def call(self, messages, stream: bool = False):
assert messages[0]["role"] == "system"
assert "demo.md" in messages[1]["content"]
return SimpleNamespace(content='{"questions": ["如何使用 demo"]}')
class FakeRepository:
async def update(self, kb_id: str, data: dict) -> None:
saved[kb_id] = data["sample_questions"]
async def get_by_kb_id(self, kb_id: str):
return SimpleNamespace(name="测试知识库", sample_questions=saved.get(kb_id))
monkeypatch.setattr(sq, "knowledge_base", FakeKnowledgeBase())
monkeypatch.setattr(
sq.KnowledgeBaseFactory,
"get_kb_class",
lambda _kb_type: SimpleNamespace(supports_documents=True),
)
monkeypatch.setattr(sq, "select_model", lambda model_spec: FakeModel())
monkeypatch.setattr(sq, "KnowledgeBaseRepository", lambda: FakeRepository())
generated = await sq.generate_database_sample_questions("kb_1", count=1)
stored = await sq.get_database_sample_questions("kb_1")
assert generated["questions"] == ["如何使用 demo"]
assert generated["count"] == 1
assert stored["questions"] == ["如何使用 demo"]
@pytest.mark.asyncio
async def test_generate_database_sample_questions_maps_invalid_json(monkeypatch):
class FakeKnowledgeBase:
async def get_database_info(self, kb_id: str, include_files: bool = False) -> dict:
return {
"name": "测试知识库",
"kb_type": "milvus",
"files": {"file_1": {"filename": "demo.md", "file_type": "md"}},
}
class FakeModel:
async def call(self, messages, stream: bool = False):
return SimpleNamespace(content="not json")
monkeypatch.setattr(sq, "knowledge_base", FakeKnowledgeBase())
monkeypatch.setattr(
sq.KnowledgeBaseFactory,
"get_kb_class",
lambda _kb_type: SimpleNamespace(supports_documents=True),
)
monkeypatch.setattr(sq, "select_model", lambda model_spec: FakeModel())
with pytest.raises(HTTPException) as exc_info:
await sq.generate_database_sample_questions("kb_1")
assert exc_info.value.status_code == 500
assert "AI返回格式错误" in exc_info.value.detail