chore: import upstream snapshot with attribution

backend/test/unit/knowledge/eval/test_benchmark_generation.py

@@ -0,0 +1,413 @@
+import asyncio
+import os
+from types import SimpleNamespace
+
+import pytest
+
+os.environ.setdefault("OPENAI_API_KEY", "test-key")
+
+from yuxi.knowledge.eval import benchmark_generation
+from yuxi.knowledge.eval.benchmark_generation import (
+    build_benchmark_generation_prompt,
+    clamp_neighbors_count,
+    collect_kb_chunks,
+    iter_generated_benchmark_items,
+    normalize_generation_concurrency_count,
+    select_graph_enhanced_chunks,
+    select_neighbor_chunks_by_kb_query,
+)
+
+
+class FakeKnowledgeBase:
+    pass
+
+
+class FakeGenerationKnowledgeBase:
+    def __init__(self, query_results=None):
+        self.query_results = query_results or []
+        self.query_calls = []
+
+    async def aquery(self, query_text, kb_id, **kwargs):
+        self.query_calls.append({"query_text": query_text, "kb_id": kb_id, **kwargs})
+        return self.query_results
+
+
+class FakeLlm:
+    def __init__(self, gold_chunk_id="anchor_chunk"):
+        self.gold_chunk_id = gold_chunk_id
+        self.prompts = []
+
+    async def call(self, prompt, stream):
+        self.prompts.append(prompt)
+        return SimpleNamespace(
+            content=('{"query":"问题","gold_answer":"答案","gold_chunk_ids":["' + self.gold_chunk_id + '"]}')
+        )
+
+
+class NoQueryKnowledgeBase(FakeGenerationKnowledgeBase):
+    async def aquery(self, query_text, kb_id, **kwargs):
+        raise AssertionError("neighbors_count=1 时不应调用 aquery")
+
+
+class TrackingLlm:
+    def __init__(self, content=None, delay=0):
+        self.content = content or '{"query":"问题","gold_answer":"答案","gold_chunk_ids":["anchor_chunk"]}'
+        self.delay = delay
+        self.active_calls = 0
+        self.max_active_calls = 0
+        self.calls = 0
+
+    async def call(self, prompt, stream):
+        self.calls += 1
+        self.active_calls += 1
+        self.max_active_calls = max(self.max_active_calls, self.active_calls)
+        try:
+            if self.delay:
+                await asyncio.sleep(self.delay)
+            return SimpleNamespace(content=self.content)
+        finally:
+            self.active_calls -= 1
+
+
+class FakeGraphGenerationKnowledgeBase(FakeGenerationKnowledgeBase):
+    pass
+
+
+def make_chunk(
+    chunk_id: str,
+    *,
+    kb_id: str = "db_1",
+    file_id: str = "file_a",
+    content: str = "anchor content",
+    chunk_index: int = 0,
+    graph_indexed: bool = False,
+    ent_ids: list[str] | None = None,
+):
+    return SimpleNamespace(
+        chunk_id=chunk_id,
+        kb_id=kb_id,
+        file_id=file_id,
+        content=content,
+        chunk_index=chunk_index,
+        graph_indexed=graph_indexed,
+        ent_ids=ent_ids,
+        tags=None,
+        extraction_result=None,
+    )
+
+
+@pytest.fixture(autouse=True)
+def fake_chunk_repository(monkeypatch):
+    class FakeChunkRepository:
+        chunks = [make_chunk("anchor_chunk")]
+
+        async def list_by_kb_id(self, kb_id):
+            return [chunk for chunk in self.chunks if chunk.kb_id == kb_id]
+
+    monkeypatch.setattr(
+        "yuxi.repositories.knowledge_chunk_repository.KnowledgeChunkRepository",
+        FakeChunkRepository,
+    )
+    return FakeChunkRepository
+
+
+def test_clamp_neighbors_count():
+    assert clamp_neighbors_count(-1) == 0
+    assert clamp_neighbors_count(3) == 3
+    assert clamp_neighbors_count(11) == 10
+
+
+def test_normalize_generation_concurrency_count():
+    assert normalize_generation_concurrency_count(None) == 10
+    assert normalize_generation_concurrency_count("") == 10
+    assert normalize_generation_concurrency_count(0) == 1
+    assert normalize_generation_concurrency_count(-5) == 1
+    assert normalize_generation_concurrency_count(10000) == 20
+
+
+def test_build_benchmark_generation_prompt_contains_required_schema():
+    prompt = build_benchmark_generation_prompt([("chunk_1", "片段内容")])
+
+    assert "片段ID=chunk_1" in prompt
+    assert "query、gold_answer、gold_chunk_ids" in prompt
+
+
+@pytest.mark.asyncio
+async def test_collect_kb_chunks_filters_kb_id(fake_chunk_repository):
+    fake_chunk_repository.chunks = [
+        make_chunk("file_a_chunk", content="内容"),
+        make_chunk("file_b_chunk", kb_id="db_2", file_id="file_b", content="其他"),
+    ]
+
+    chunks = await collect_kb_chunks(FakeKnowledgeBase(), "db_1")
+
+    assert chunks == [
+        {
+            "id": "file_a_chunk",
+            "content": "内容",
+            "file_id": "file_a",
+            "chunk_index": 0,
+            "graph_indexed": False,
+            "ent_ids": [],
+            "tags": [],
+            "extraction_result": None,
+        }
+    ]
+
+
+@pytest.mark.asyncio
+async def test_iter_generated_benchmark_items_with_one_chunk_does_not_query(monkeypatch):
+    fake_llm = FakeLlm()
+    monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
+
+    items = [
+        item
+        async for item in iter_generated_benchmark_items(
+            kb_instance=NoQueryKnowledgeBase(),
+            kb_id="db_1",
+            count=1,
+            neighbors_count=1,
+            llm_model_spec="test-provider:test-model",
+        )
+    ]
+
+    assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}]
+    assert "片段ID=anchor_chunk" in fake_llm.prompts[0]
+
+
+@pytest.mark.asyncio
+async def test_select_neighbor_chunks_by_kb_query_filters_anchor():
+    kb = FakeGenerationKnowledgeBase(
+        query_results=[
+            {
+                "content": "anchor content",
+                "metadata": {"chunk_id": "anchor_chunk", "file_id": "file_a", "chunk_index": 0},
+            },
+            {
+                "content": "neighbor content",
+                "metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1},
+            },
+        ]
+    )
+
+    chunks = await select_neighbor_chunks_by_kb_query(
+        kb_instance=kb,
+        kb_id="db_1",
+        anchor_chunk={"id": "anchor_chunk", "content": "anchor content", "file_id": "file_a", "chunk_index": 0},
+        neighbors_count=1,
+    )
+
+    assert chunks == [{"id": "neighbor_chunk", "content": "neighbor content", "file_id": "file_a", "chunk_index": 1}]
+    assert kb.query_calls == [
+        {
+            "query_text": "anchor content",
+            "kb_id": "db_1",
+            "search_mode": "vector",
+            "final_top_k": 4,
+            "use_reranker": False,
+            "similarity_threshold": 0.0,
+        }
+    ]
+
+
+@pytest.mark.asyncio
+async def test_select_graph_enhanced_chunks_expands_by_ppr_with_anchor_bias(monkeypatch):
+    calls = []
+
+    async def fake_rank(self, kb_id, seed_weights, *, max_nodes, top_k, damping):
+        calls.append(dict(seed_weights))
+        if len(calls) == 1:
+            return [("anchor", 0.9), ("neighbor_1", 0.8)]
+        return [("anchor", 0.9), ("neighbor_1", 0.8), ("neighbor_2", 0.7)]
+
+    monkeypatch.setattr(
+        "yuxi.knowledge.graphs.milvus_graph_service.MilvusGraphService.query_and_rank_chunks_by_ppr",
+        fake_rank,
+    )
+    chunks_by_id = {
+        "anchor": {"id": "anchor", "content": "anchor", "ent_ids": ["anchor_entity"]},
+        "neighbor_1": {"id": "neighbor_1", "content": "neighbor 1", "ent_ids": ["entity_1"]},
+        "neighbor_2": {"id": "neighbor_2", "content": "neighbor 2", "ent_ids": ["entity_2"]},
+    }
+
+    chunks = await select_graph_enhanced_chunks(
+        kb_id="db_1",
+        anchor_chunk=chunks_by_id["anchor"],
+        chunks_by_id=chunks_by_id,
+        context_count=3,
+        graph_expand_top_k=1,
+    )
+
+    assert [chunk["id"] for chunk in chunks] == ["anchor", "neighbor_1", "neighbor_2"]
+    assert calls[0] == {"anchor_entity": 1.0}
+    assert calls[1]["anchor_entity"] == 1.0
+    assert calls[1]["entity_1"] == 0.9
+
+
+@pytest.mark.asyncio
+async def test_iter_generated_benchmark_items_graph_mode_uses_graph_indexed_anchor(monkeypatch, fake_chunk_repository):
+    fake_chunk_repository.chunks = [
+        make_chunk(
+            "vector_anchor",
+            content="vector content",
+            chunk_index=0,
+            graph_indexed=False,
+            ent_ids=["vector_entity"],
+        ),
+        make_chunk(
+            "graph_anchor",
+            content="graph anchor content",
+            chunk_index=1,
+            graph_indexed=True,
+            ent_ids=["anchor_entity"],
+        ),
+        make_chunk(
+            "graph_neighbor",
+            content="graph neighbor content",
+            chunk_index=2,
+            graph_indexed=False,
+            ent_ids=["neighbor_entity"],
+        ),
+    ]
+
+    async def fake_rank(self, kb_id, seed_weights, *, max_nodes, top_k, damping):
+        assert seed_weights["anchor_entity"] == 1.0
+        return [("graph_anchor", 0.9), ("graph_neighbor", 0.8)]
+
+    fake_llm = FakeLlm(gold_chunk_id="graph_neighbor")
+    monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
+    monkeypatch.setattr(
+        "yuxi.knowledge.graphs.milvus_graph_service.MilvusGraphService.query_and_rank_chunks_by_ppr",
+        fake_rank,
+    )
+    kb = FakeGraphGenerationKnowledgeBase()
+
+    items = [
+        item
+        async for item in iter_generated_benchmark_items(
+            kb_instance=kb,
+            kb_id="db_1",
+            count=1,
+            neighbors_count=2,
+            llm_model_spec="test-provider:test-model",
+            generation_mode="graph_enhanced",
+        )
+    ]
+
+    assert items == [{"query": "问题", "gold_chunk_ids": ["graph_neighbor"], "gold_answer": "答案"}]
+    assert kb.query_calls == []
+    assert "片段ID=graph_anchor" in fake_llm.prompts[0]
+    assert "片段ID=graph_neighbor" in fake_llm.prompts[0]
+    assert "片段ID=vector_anchor" not in fake_llm.prompts[0]
+
+
+@pytest.mark.asyncio
+async def test_iter_generated_benchmark_items_uses_query_neighbor(monkeypatch):
+    fake_llm = FakeLlm(gold_chunk_id="neighbor_chunk")
+    monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
+    kb = FakeGenerationKnowledgeBase(
+        query_results=[
+            {
+                "content": "neighbor content",
+                "metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1},
+            }
+        ]
+    )
+
+    items = [
+        item
+        async for item in iter_generated_benchmark_items(
+            kb_instance=kb,
+            kb_id="db_1",
+            count=1,
+            neighbors_count=2,
+            llm_model_spec="test-provider:test-model",
+        )
+    ]
+
+    assert items == [{"query": "问题", "gold_chunk_ids": ["neighbor_chunk"], "gold_answer": "答案"}]
+    assert kb.query_calls[0]["query_text"] == "anchor content"
+    assert kb.query_calls[0]["search_mode"] == "vector"
+    assert "片段ID=neighbor_chunk" in fake_llm.prompts[0]
+
+
+@pytest.mark.asyncio
+async def test_iter_generated_benchmark_items_falls_back_to_anchor_when_query_empty(monkeypatch):
+    fake_llm = FakeLlm()
+    monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
+
+    items = [
+        item
+        async for item in iter_generated_benchmark_items(
+            kb_instance=FakeGenerationKnowledgeBase(query_results=[]),
+            kb_id="db_1",
+            count=1,
+            neighbors_count=2,
+            llm_model_spec="test-provider:test-model",
+        )
+    ]
+
+    assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}]
+    assert "片段ID=anchor_chunk" in fake_llm.prompts[0]
+
+
+@pytest.mark.asyncio
+async def test_iter_generated_benchmark_items_respects_concurrency_count(monkeypatch):
+    fake_llm = TrackingLlm(delay=0.01)
+    monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
+
+    items = [
+        item
+        async for item in iter_generated_benchmark_items(
+            kb_instance=NoQueryKnowledgeBase(),
+            kb_id="db_1",
+            count=4,
+            neighbors_count=1,
+            concurrency_count=2,
+            llm_model_spec="test-provider:test-model",
+        )
+    ]
+
+    assert len(items) == 4
+    assert fake_llm.max_active_calls == 2
+
+
+@pytest.mark.asyncio
+async def test_iter_generated_benchmark_items_returns_at_most_count(monkeypatch):
+    fake_llm = TrackingLlm(delay=0.01)
+    monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
+
+    items = [
+        item
+        async for item in iter_generated_benchmark_items(
+            kb_instance=NoQueryKnowledgeBase(),
+            kb_id="db_1",
+            count=3,
+            neighbors_count=1,
+            concurrency_count=10,
+            llm_model_spec="test-provider:test-model",
+        )
+    ]
+
+    assert len(items) == 3
+
+
+@pytest.mark.asyncio
+async def test_iter_generated_benchmark_items_stops_at_max_attempts(monkeypatch):
+    fake_llm = TrackingLlm(content='{"query":"","gold_answer":"答案","gold_chunk_ids":["anchor_chunk"]}')
+    monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
+
+    items = [
+        item
+        async for item in iter_generated_benchmark_items(
+            kb_instance=NoQueryKnowledgeBase(),
+            kb_id="db_1",
+            count=2,
+            neighbors_count=1,
+            concurrency_count=10,
+            llm_model_spec="test-provider:test-model",
+        )
+    ]
+
+    assert items == []
+    assert fake_llm.calls == 50

backend/test/unit/knowledge/eval/test_evaluator.py

@@ -0,0 +1,39 @@
+import os
+
+os.environ.setdefault("OPENAI_API_KEY", "test-key")
+
+from yuxi.knowledge.eval.evaluator import aggregate_metrics, build_answer_prompt, normalize_query_result
+
+
+def test_normalize_query_result_supports_dict_and_list():
+    answer, chunks = normalize_query_result({"answer": "A", "retrieved_chunks": [{"content": "C"}]})
+    assert answer == "A"
+    assert chunks == [{"content": "C"}]
+
+    answer, chunks = normalize_query_result([{"content": "C"}])
+    assert answer == ""
+    assert chunks == [{"content": "C"}]
+
+
+def test_build_answer_prompt_uses_first_five_non_empty_chunks():
+    chunks = [{"content": f"内容{i}"} for i in range(6)] + [{"content": ""}]
+
+    prompt = build_answer_prompt("问题", chunks)
+
+    assert "用户问题：问题" in prompt
+    assert "内容0" in prompt
+    assert "内容4" in prompt
+    assert "内容5" not in prompt
+
+
+def test_aggregate_metrics_matches_service_output_shape():
+    metrics, overall_score = aggregate_metrics(
+        [{"recall@1": 1.0, "f1@1": 0.0}, {"recall@1": 0.0, "f1@1": 1.0}],
+        [{"score": 1.0}, {"score": 0.0}],
+        include_overall_score=True,
+    )
+
+    assert metrics["recall@1"] == 0.5
+    assert metrics["f1@1"] == 0.5
+    assert metrics["answer_correctness"] == 0.5
+    assert metrics["overall_score"] == overall_score

backend/test/unit/knowledge/eval/test_metrics.py

@@ -0,0 +1,50 @@
+import os
+
+import pytest
+
+os.environ.setdefault("OPENAI_API_KEY", "test-key")
+
+from yuxi.knowledge.eval.metrics import EvaluationMetricsCalculator, RetrievalMetrics
+
+
+def test_retrieval_metrics_use_metadata_chunk_id():
+    retrieved_chunks = [
+        {"metadata": {"chunk_id": "chunk_a"}},
+        {"metadata": {"chunk_id": "chunk_b"}},
+    ]
+
+    metrics = EvaluationMetricsCalculator.calculate_retrieval_metrics(
+        retrieved_chunks, ["chunk_b", "chunk_c"], k_values=[1, 3]
+    )
+
+    assert metrics["recall@1"] == 0.0
+    assert metrics["recall@3"] == 0.5
+    assert metrics["f1@3"] == RetrievalMetrics.f1_score_at_k(["chunk_a", "chunk_b"], ["chunk_b", "chunk_c"], 3)
+
+
+def test_overall_score_uses_answer_accuracy_when_available():
+    # 有答案准确率时，综合得分取各题 score 的平均，且与检索指标无关
+    retrieval = [{"recall@10": 1.0, "f1@10": 0.2}, {"recall@10": 0.0, "f1@10": 0.0}]
+    answers = [{"score": 1.0}, {"score": 0.0}, {"score": 1.0}, {"score": 1.0}]
+
+    score = EvaluationMetricsCalculator.calculate_overall_score(retrieval, answers)
+
+    assert score == 0.75
+
+
+def test_overall_score_uses_recall_at_10_without_answers():
+    # 无答案准确率时，综合得分取各题 recall@10 的平均，不受 f1/其它 k 影响
+    retrieval = [
+        {"recall@1": 0.0, "recall@5": 0.5, "recall@10": 0.8, "f1@10": 0.1},
+        {"recall@1": 1.0, "recall@5": 1.0, "recall@10": 0.4, "f1@10": 0.9},
+    ]
+
+    score = EvaluationMetricsCalculator.calculate_overall_score(retrieval, [])
+
+    assert score == pytest.approx(0.6)
+
+
+def test_overall_score_returns_none_without_any_metrics():
+    score = EvaluationMetricsCalculator.calculate_overall_score([], [])
+
+    assert score is None

backend/test/unit/knowledge/eval/test_service_generation.py

@@ -0,0 +1,133 @@
+from types import SimpleNamespace
+
+import pytest
+
+from yuxi.knowledge.eval import service as eval_service_module
+from yuxi.knowledge.eval.service import EvaluationService, build_evaluation_run_name
+
+
+class FakeEvaluationRepository:
+    def __init__(self):
+        self.created_dataset = None
+        self.updated_dataset = None
+        self.dataset = None
+        self.created_run = None
+
+    async def create_dataset(self, payload):
+        self.created_dataset = payload
+
+    async def update_dataset(self, dataset_id, payload):
+        self.updated_dataset = (dataset_id, payload)
+
+    async def get_dataset(self, dataset_id):
+        return self.dataset
+
+    async def create_run(self, payload):
+        self.created_run = payload
+
+
+class FakeChunkRepository:
+    def __init__(self, indexed_count):
+        self.indexed_count = indexed_count
+
+    async def count_graph_indexed_by_kb_id(self, kb_id):
+        return self.indexed_count
+
+
+class FakeKnowledgeBaseRepository:
+    async def get_by_kb_id(self, kb_id):
+        return SimpleNamespace(query_params={"options": {"top_k": 3}})
+
+
+@pytest.mark.asyncio
+async def test_generate_dataset_saves_generation_params(monkeypatch):
+    async def fake_enqueue(**kwargs):
+        return SimpleNamespace(id="task_1")
+
+    monkeypatch.setattr(eval_service_module.tasker, "enqueue", fake_enqueue)
+    service = EvaluationService()
+    service.eval_repo = FakeEvaluationRepository()
+    service.chunk_repo = FakeChunkRepository(indexed_count=1)
+
+    result = await service.generate_dataset(
+        kb_id="db_1",
+        name="dataset",
+        description="desc",
+        count=2,
+        neighbors_count=3,
+        concurrency_count=4,
+        llm_model_spec="test:model",
+        generation_mode="graph_enhanced",
+        graph_expand_top_k=2,
+        created_by="user_1",
+    )
+
+    assert result["task_id"] == "task_1"
+    params = service.eval_repo.created_dataset["build_metadata"]["params"]
+    assert params["generation_mode"] == "graph_enhanced"
+    assert params["graph_expand_top_k"] == 2
+    updated_metadata = service.eval_repo.updated_dataset[1]["build_metadata"]
+    assert updated_metadata["params"] == params
+
+
+@pytest.mark.asyncio
+async def test_generate_dataset_rejects_graph_mode_without_indexed_chunks():
+    service = EvaluationService()
+    service.eval_repo = FakeEvaluationRepository()
+    service.chunk_repo = FakeChunkRepository(indexed_count=0)
+
+    with pytest.raises(ValueError, match="尚未完成图索引"):
+        await service.generate_dataset(
+            kb_id="db_1",
+            name="dataset",
+            description="desc",
+            count=2,
+            neighbors_count=3,
+            concurrency_count=4,
+            llm_model_spec="test:model",
+            generation_mode="graph_enhanced",
+            graph_expand_top_k=1,
+            created_by="user_1",
+        )
+
+    assert service.eval_repo.created_dataset is None
+
+
+def test_build_evaluation_run_name_uses_eval_date_hash_format():
+    name = build_evaluation_run_name(hash_value="abcdef12")
+
+    assert name.startswith("eval-")
+    assert name.endswith("-abcdef")
+    assert len(name.split("-")[1]) == 8
+
+
+@pytest.mark.asyncio
+async def test_run_evaluation_saves_custom_name(monkeypatch):
+    async def fake_enqueue(**kwargs):
+        return SimpleNamespace(id="task_1")
+
+    monkeypatch.setattr(eval_service_module.tasker, "enqueue", fake_enqueue)
+    repo = FakeEvaluationRepository()
+    repo.dataset = SimpleNamespace(
+        dataset_id="dataset_1",
+        kb_id="db_1",
+        name="dataset",
+        item_count=2,
+        build_metadata={"status": "completed"},
+    )
+    service = EvaluationService()
+    service.eval_repo = repo
+    service.kb_repo = FakeKnowledgeBaseRepository()
+
+    run_id = await service.run_evaluation(
+        kb_id="db_1",
+        dataset_id="dataset_1",
+        name="  回归评估  ",
+        model_config={"answer_llm": "test:model"},
+        created_by="user_1",
+    )
+
+    assert run_id.startswith("run_")
+    assert repo.created_run["name"] == "回归评估"
+    assert repo.created_run["retrieval_config"]["top_k"] == 3
+    assert repo.created_run["retrieval_config"]["answer_llm"] == "test:model"