chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
from yuxi.agents.skills.buildin import BUILTIN_SKILLS
|
||||
|
||||
|
||||
def _mysql_reporter_dir() -> Path:
|
||||
for spec in BUILTIN_SKILLS:
|
||||
if spec.slug == "mysql-reporter":
|
||||
return spec.source_dir
|
||||
raise AssertionError("mysql-reporter builtin skill spec not found")
|
||||
|
||||
|
||||
def _load_script(script_name: str) -> ModuleType:
|
||||
script_path = _mysql_reporter_dir() / "scripts" / script_name
|
||||
spec = importlib.util.spec_from_file_location(f"mysql_reporter_{script_path.stem}", script_path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_mysql_reporter_query_security_validates_sql_and_timeout():
|
||||
query_script = _load_script("query.py")
|
||||
sql_cases = {
|
||||
"": False,
|
||||
"SELECT * FROM users": True,
|
||||
"show tables": True,
|
||||
"DESCRIBE users": True,
|
||||
"EXPLAIN SELECT * FROM users": True,
|
||||
"SELECT 1;": True,
|
||||
"DELETE FROM users": False,
|
||||
"SELECT * FROM users WHERE id = 1 OR 1=1": False,
|
||||
"SELECT * FROM users UNION SELECT password FROM admin": False,
|
||||
"SELECT 'DROP' AS keyword_text": True,
|
||||
"/* comment */ SELECT 1": True,
|
||||
"/* multi\nline */ SELECT 1": True,
|
||||
"SELECT * FROM users; DROP TABLE users": False,
|
||||
"SELECT * FROM users; CREATE TABLE audit_log(id INT)": False,
|
||||
"SELECT * FROM users; SET @unsafe = 1": False,
|
||||
}
|
||||
|
||||
for sql, expected in sql_cases.items():
|
||||
assert query_script.MySQLSecurityChecker.validate_sql(sql) is expected
|
||||
|
||||
timeout_cases = {
|
||||
None: False,
|
||||
0: False,
|
||||
1: True,
|
||||
60: True,
|
||||
600: True,
|
||||
601: False,
|
||||
"60": False,
|
||||
}
|
||||
|
||||
for timeout, expected in timeout_cases.items():
|
||||
assert query_script.MySQLSecurityChecker.validate_timeout(timeout) is expected
|
||||
|
||||
|
||||
def test_mysql_reporter_describe_table_name_security_validates_known_cases():
|
||||
describe_script = _load_script("describe_table.py")
|
||||
table_cases = {
|
||||
"": False,
|
||||
"users": True,
|
||||
"_audit_log": True,
|
||||
"user_2026": True,
|
||||
"1users": False,
|
||||
"user-name": False,
|
||||
"users;drop": False,
|
||||
}
|
||||
|
||||
for table_name, expected in table_cases.items():
|
||||
assert describe_script.MySQLSecurityChecker.validate_table_name(table_name) is expected
|
||||
@@ -0,0 +1,370 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.skills import remote_install as svc
|
||||
|
||||
|
||||
def test_parse_available_skills_from_cli_output() -> None:
|
||||
output = """
|
||||
\x1b[38;5;250m███████╗\x1b[0m
|
||||
◇ Available Skills
|
||||
Claude Api
|
||||
|
||||
claude-api
|
||||
|
||||
Build apps with the Claude API.
|
||||
|
||||
Example Skills
|
||||
|
||||
frontend-design
|
||||
|
||||
Create distinctive frontend interfaces.
|
||||
|
||||
└ Use --skill <name> to install specific skills
|
||||
"""
|
||||
|
||||
skills = svc._parse_available_skills(output)
|
||||
|
||||
assert skills == [
|
||||
{"name": "claude-api", "description": "Build apps with the Claude API."},
|
||||
{"name": "frontend-design", "description": "Create distinctive frontend interfaces."},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_remote_skills_uses_isolated_home(monkeypatch: pytest.MonkeyPatch):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_run_skills_cli(args: list[str], *, env: dict[str, str], cwd: str) -> str:
|
||||
captured["args"] = args
|
||||
captured["home"] = env["HOME"]
|
||||
captured["cwd"] = cwd
|
||||
return """
|
||||
◇ Available Skills
|
||||
|
||||
frontend-design
|
||||
|
||||
Create distinctive frontend interfaces.
|
||||
|
||||
└ Use --skill <name> to install specific skills
|
||||
"""
|
||||
|
||||
monkeypatch.setattr(svc, "_run_skills_cli", fake_run_skills_cli)
|
||||
|
||||
items = await svc.list_remote_skills("anthropics/skills")
|
||||
|
||||
assert items == [{"name": "frontend-design", "description": "Create distinctive frontend interfaces."}]
|
||||
assert captured["args"] == ["npx", "-y", "skills", "add", "anthropics/skills", "--list"]
|
||||
assert str(captured["cwd"]).startswith(str(captured["home"]))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_remote_skill_imports_from_cli_output_dir(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: list[tuple[list[str], str]] = []
|
||||
|
||||
async def fake_run_skills_cli(args: list[str], *, env: dict[str, str], cwd: str) -> str:
|
||||
calls.append((args, env["HOME"]))
|
||||
home = Path(env["HOME"])
|
||||
if "--list" in args:
|
||||
return """
|
||||
◇ Available Skills
|
||||
|
||||
frontend-design
|
||||
|
||||
Create distinctive frontend interfaces.
|
||||
|
||||
└ Use --skill <name> to install specific skills
|
||||
"""
|
||||
skill_dir = home / ".agents" / "skills" / "frontend-design"
|
||||
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir / "SKILL.md").write_text(
|
||||
"---\nname: frontend-design\ndescription: demo\n---\n# Demo\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return "installed"
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_import_skill_dir(_db, *, source_dir, created_by):
|
||||
captured["source_dir"] = Path(source_dir)
|
||||
captured["created_by"] = created_by
|
||||
return {"slug": "frontend-design"}
|
||||
|
||||
monkeypatch.setattr(svc, "_run_skills_cli", fake_run_skills_cli)
|
||||
monkeypatch.setattr(svc, "import_skill_dir", fake_import_skill_dir)
|
||||
|
||||
item = await svc.install_remote_skill(
|
||||
None,
|
||||
source="anthropics/skills",
|
||||
skill="frontend-design",
|
||||
created_by="root",
|
||||
)
|
||||
|
||||
assert item == {"slug": "frontend-design"}
|
||||
assert calls[0][0] == ["npx", "-y", "skills", "add", "anthropics/skills", "--list"]
|
||||
assert calls[1][0] == [
|
||||
"npx",
|
||||
"-y",
|
||||
"skills",
|
||||
"add",
|
||||
"anthropics/skills",
|
||||
"--skill",
|
||||
"frontend-design",
|
||||
"-g",
|
||||
"-y",
|
||||
"--copy",
|
||||
]
|
||||
assert captured["source_dir"] == Path(calls[1][1]) / ".agents" / "skills" / "frontend-design"
|
||||
assert captured["created_by"] == "root"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_remote_skill_rejects_missing_remote_skill(monkeypatch: pytest.MonkeyPatch):
|
||||
async def fake_run_skills_cli(args: list[str], *, env: dict[str, str], cwd: str) -> str:
|
||||
return """
|
||||
◇ Available Skills
|
||||
|
||||
other-skill
|
||||
|
||||
Description
|
||||
|
||||
└ Use --skill <name> to install specific skills
|
||||
"""
|
||||
|
||||
monkeypatch.setattr(svc, "_run_skills_cli", fake_run_skills_cli)
|
||||
|
||||
with pytest.raises(ValueError, match="不存在 skill"):
|
||||
await svc.install_remote_skill(
|
||||
None,
|
||||
source="anthropics/skills",
|
||||
skill="frontend-design",
|
||||
created_by="root",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_remote_skills_batch_installs_all(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: list[tuple[list[str], str]] = []
|
||||
imported_skills: list[str] = []
|
||||
|
||||
async def fake_run_skills_cli(args: list[str], *, env: dict[str, str], cwd: str) -> str:
|
||||
calls.append((args, env["HOME"]))
|
||||
home = Path(env["HOME"])
|
||||
skill_dir_base = home / ".agents" / "skills"
|
||||
for skill_name in ("frontend-design", "claude-api", "code-review"):
|
||||
(skill_dir_base / skill_name).mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir_base / skill_name / "SKILL.md").write_text(
|
||||
f"---\nname: {skill_name}\ndescription: demo\n---\n# {skill_name}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return "installed"
|
||||
|
||||
async def fake_import_skill_dir(_db, *, source_dir, created_by):
|
||||
imported_skills.append(source_dir.name)
|
||||
return SimpleNamespace(slug=source_dir.name)
|
||||
|
||||
monkeypatch.setattr(svc, "_run_skills_cli", fake_run_skills_cli)
|
||||
monkeypatch.setattr(svc, "import_skill_dir", fake_import_skill_dir)
|
||||
|
||||
results = await svc.install_remote_skills_batch(
|
||||
None,
|
||||
source="anthropics/skills",
|
||||
skills=["frontend-design", "claude-api", "code-review"],
|
||||
created_by="root",
|
||||
)
|
||||
|
||||
# Should only have 1 CLI call (no --list, direct batch install)
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0] == [
|
||||
"npx",
|
||||
"-y",
|
||||
"skills",
|
||||
"add",
|
||||
"anthropics/skills",
|
||||
"--skill",
|
||||
"frontend-design",
|
||||
"--skill",
|
||||
"claude-api",
|
||||
"--skill",
|
||||
"code-review",
|
||||
"-g",
|
||||
"-y",
|
||||
"--copy",
|
||||
]
|
||||
|
||||
assert len(results) == 3
|
||||
assert all(r["success"] for r in results)
|
||||
assert [r["slug"] for r in results] == ["frontend-design", "claude-api", "code-review"]
|
||||
assert imported_skills == ["frontend-design", "claude-api", "code-review"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_remote_skills_batch_skips_missing(monkeypatch: pytest.MonkeyPatch):
|
||||
async def fake_run_skills_cli(args: list[str], *, env: dict[str, str], cwd: str) -> str:
|
||||
home = Path(env["HOME"])
|
||||
skill_dir_base = home / ".agents" / "skills"
|
||||
(skill_dir_base / "frontend-design").mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir_base / "frontend-design" / "SKILL.md").write_text(
|
||||
"---\nname: frontend-design\ndescription: demo\n---\n# Demo\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return "installed"
|
||||
|
||||
async def fake_import_skill_dir(_db, *, source_dir, created_by):
|
||||
return SimpleNamespace(slug=source_dir.name)
|
||||
|
||||
monkeypatch.setattr(svc, "_run_skills_cli", fake_run_skills_cli)
|
||||
monkeypatch.setattr(svc, "import_skill_dir", fake_import_skill_dir)
|
||||
|
||||
results = await svc.install_remote_skills_batch(
|
||||
None,
|
||||
source="anthropics/skills",
|
||||
skills=["frontend-design", "nonexistent-skill"],
|
||||
created_by="root",
|
||||
)
|
||||
|
||||
assert len(results) == 2
|
||||
assert results[0] == {"slug": "frontend-design", "success": True}
|
||||
assert results[1] == {"slug": "nonexistent-skill", "success": False, "error": "skills CLI 未生成预期的技能目录"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_remote_skills_batch_partial_failure(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: list[tuple[list[str], str]] = []
|
||||
|
||||
async def fake_run_skills_cli(args: list[str], *, env: dict[str, str], cwd: str) -> str:
|
||||
calls.append((args, env["HOME"]))
|
||||
home = Path(env["HOME"])
|
||||
skill_dir_base = home / ".agents" / "skills"
|
||||
(skill_dir_base / "skill-a").mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir_base / "skill-a" / "SKILL.md").write_text(
|
||||
"---\nname: skill-a\ndescription: demo\n---\n# A\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
# skill-b directory missing (simulate install failure from CLI side)
|
||||
(skill_dir_base / "skill-c").mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir_base / "skill-c" / "SKILL.md").write_text(
|
||||
"---\nname: skill-c\ndescription: demo\n---\n# C\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return "installed"
|
||||
|
||||
async def fake_import_skill_dir(_db, *, source_dir, created_by):
|
||||
return SimpleNamespace(slug=source_dir.name)
|
||||
|
||||
monkeypatch.setattr(svc, "_run_skills_cli", fake_run_skills_cli)
|
||||
monkeypatch.setattr(svc, "import_skill_dir", fake_import_skill_dir)
|
||||
|
||||
results = await svc.install_remote_skills_batch(
|
||||
None,
|
||||
source="test/repo",
|
||||
skills=["skill-a", "skill-b", "skill-c"],
|
||||
created_by="root",
|
||||
)
|
||||
|
||||
assert len(results) == 3
|
||||
assert results[0] == {"slug": "skill-a", "success": True}
|
||||
assert results[1] == {"slug": "skill-b", "success": False, "error": "skills CLI 未生成预期的技能目录"}
|
||||
assert results[2] == {"slug": "skill-c", "success": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_remote_skills_batch_handles_invalid_names(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: list[tuple[list[str], str]] = []
|
||||
|
||||
async def fake_run_skills_cli(args: list[str], *, env: dict[str, str], cwd: str) -> str:
|
||||
calls.append((args, env["HOME"]))
|
||||
home = Path(env["HOME"])
|
||||
skill_dir_base = home / ".agents" / "skills"
|
||||
(skill_dir_base / "valid-skill").mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir_base / "valid-skill" / "SKILL.md").write_text(
|
||||
"---\nname: valid-skill\ndescription: demo\n---\n# Valid\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return "installed"
|
||||
|
||||
async def fake_import_skill_dir(_db, *, source_dir, created_by):
|
||||
return SimpleNamespace(slug=source_dir.name)
|
||||
|
||||
monkeypatch.setattr(svc, "_run_skills_cli", fake_run_skills_cli)
|
||||
monkeypatch.setattr(svc, "import_skill_dir", fake_import_skill_dir)
|
||||
|
||||
results = await svc.install_remote_skills_batch(
|
||||
None,
|
||||
source="test/repo",
|
||||
skills=["valid-skill", "Bad Name", "another-valid"],
|
||||
created_by="root",
|
||||
)
|
||||
|
||||
assert len(results) == 3
|
||||
assert results[0] == {"slug": "valid-skill", "success": True}
|
||||
assert results[1]["success"] is False
|
||||
assert "不合法" in results[1]["error"]
|
||||
assert results[2] == {"slug": "another-valid", "success": False, "error": "skills CLI 未生成预期的技能目录"}
|
||||
|
||||
# Only valid skills passed to the CLI
|
||||
assert len(calls) == 1
|
||||
assert "--skill" in str(calls[0][0])
|
||||
assert "valid-skill" in str(calls[0][0])
|
||||
assert "Bad" not in str(calls[0][0])
|
||||
|
||||
|
||||
def test_parse_search_skills() -> None:
|
||||
output = """
|
||||
Install with npx skills add <owner/repo@skill>
|
||||
|
||||
vercel-labs/agent-skills@web-design-guidelines 339.3K installs
|
||||
└ https://skills.sh/vercel-labs/agent-skills/web-design-guidelines
|
||||
|
||||
xixu-me/skills@secure-linux-web-hosting 158.6K installs
|
||||
└ https://skills.sh/xixu-me/skills/secure-linux-web-hosting
|
||||
|
||||
anthropics/skills@webapp-testing
|
||||
└ https://skills.sh/anthropics/skills/webapp-testing
|
||||
"""
|
||||
|
||||
results = svc._parse_search_skills(output)
|
||||
assert results == [
|
||||
{
|
||||
"source": "vercel-labs/agent-skills",
|
||||
"name": "web-design-guidelines",
|
||||
"installs": "339.3K installs",
|
||||
},
|
||||
{
|
||||
"source": "xixu-me/skills",
|
||||
"name": "secure-linux-web-hosting",
|
||||
"installs": "158.6K installs",
|
||||
},
|
||||
{
|
||||
"source": "anthropics/skills",
|
||||
"name": "webapp-testing",
|
||||
"installs": "",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_remote_skills(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_run_skills_cli(args: list[str], *, env: dict[str, str], cwd: str) -> str:
|
||||
captured["args"] = args
|
||||
return """
|
||||
vercel-labs/agent-skills@web-design-guidelines 339.3K installs
|
||||
"""
|
||||
|
||||
monkeypatch.setattr(svc, "_run_skills_cli", fake_run_skills_cli)
|
||||
|
||||
items = await svc.search_remote_skills("web")
|
||||
assert items == [
|
||||
{
|
||||
"source": "vercel-labs/agent-skills",
|
||||
"name": "web-design-guidelines",
|
||||
"installs": "339.3K installs",
|
||||
}
|
||||
]
|
||||
assert captured["args"] == ["npx", "-y", "skills", "find", "web"]
|
||||
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.types import Command
|
||||
|
||||
from yuxi.agents.base import _json_safe, _normalize_tool_event_data
|
||||
|
||||
|
||||
def _command_tool_finished(tool_call_id: str) -> dict:
|
||||
"""模拟 write_todos / task 这类返回 Command 的工具的 tool-finished 事件。"""
|
||||
tool_message = ToolMessage(
|
||||
content="Updated todo list to [{'content': '步骤一', 'status': 'in_progress'}]",
|
||||
tool_call_id=tool_call_id,
|
||||
)
|
||||
command = Command(update={"todos": [{"content": "步骤一", "status": "in_progress"}], "messages": [tool_message]})
|
||||
return {"event": "tool-finished", "tool_call_id": tool_call_id, "output": command}
|
||||
|
||||
|
||||
def test_command_tool_finished_extracts_tool_message_for_frontend_association():
|
||||
tool_call_id = "call_abc"
|
||||
data = _normalize_tool_event_data(_command_tool_finished(tool_call_id))
|
||||
safe = _json_safe(data)
|
||||
output = safe["output"]
|
||||
|
||||
# 前端按 tool_call_id 关联结果,并要求 output 是对象(dict),否则会被丢弃。
|
||||
assert isinstance(output, dict)
|
||||
assert output["tool_call_id"] == tool_call_id
|
||||
assert output["type"] == "tool"
|
||||
assert "步骤一" in output["content"]
|
||||
|
||||
|
||||
def test_command_tool_finished_prefers_message_matching_tool_call_id():
|
||||
other = ToolMessage(content="别的工具结果", tool_call_id="call_other")
|
||||
target = ToolMessage(content="目标结果", tool_call_id="call_target")
|
||||
data = {
|
||||
"event": "tool-finished",
|
||||
"tool_call_id": "call_target",
|
||||
"output": Command(update={"messages": [other, target]}),
|
||||
}
|
||||
|
||||
output = _normalize_tool_event_data(data)["output"]
|
||||
assert isinstance(output, ToolMessage)
|
||||
assert output.tool_call_id == "call_target"
|
||||
assert output.content == "目标结果"
|
||||
|
||||
|
||||
def test_regular_dict_output_is_left_untouched():
|
||||
data = {"event": "tool-finished", "tool_call_id": "call_x", "output": {"content": "plain", "type": "tool"}}
|
||||
assert _normalize_tool_event_data(data)["output"] == {"content": "plain", "type": "tool"}
|
||||
|
||||
|
||||
def test_tool_started_event_is_left_untouched():
|
||||
data = {"event": "tool-started", "tool_call_id": "call_x", "output": None}
|
||||
assert _normalize_tool_event_data(data) is data
|
||||
|
||||
|
||||
def test_command_without_tool_message_is_left_untouched():
|
||||
command = Command(update={"todos": [{"content": "无消息", "status": "pending"}]})
|
||||
data = {"event": "tool-finished", "tool_call_id": "call_x", "output": command}
|
||||
assert _normalize_tool_event_data(data)["output"] is command
|
||||
|
||||
@@ -0,0 +1,390 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import types
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _load_context_module():
|
||||
return importlib.import_module("yuxi.agents.context")
|
||||
|
||||
|
||||
context_module = _load_context_module()
|
||||
BaseContext = context_module.BaseContext
|
||||
filter_config_by_role = context_module.filter_config_by_role
|
||||
normalize_agent_context_config = context_module.normalize_agent_context_config
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class ChatBotContext(BaseContext):
|
||||
subagents: list[str] | None = field(default=None, metadata={"kind": "subagents"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class SuperAdminOnlyContext(BaseContext):
|
||||
secret_setting: str = field(default="hidden", metadata={"name": "Secret", "auth": "superadmin"})
|
||||
|
||||
|
||||
def test_get_configurable_items_filters_admin_fields_for_user():
|
||||
items = BaseContext.get_configurable_items(user_role="user")
|
||||
|
||||
assert "system_prompt" in items
|
||||
assert "summary_threshold" not in items
|
||||
assert "summary_keep_messages" not in items
|
||||
assert "summary_prompt" not in items
|
||||
assert "summary_tool_result_token_limit" not in items
|
||||
assert "max_execution_steps" not in items
|
||||
|
||||
|
||||
def test_get_configurable_items_allows_admin_and_superadmin_fields():
|
||||
admin_items = BaseContext.get_configurable_items(user_role="admin")
|
||||
superadmin_items = SuperAdminOnlyContext.get_configurable_items(user_role="superadmin")
|
||||
|
||||
assert "summary_threshold" in admin_items
|
||||
assert "summary_keep_messages" in admin_items
|
||||
assert "summary_prompt" in admin_items
|
||||
assert "summary_tool_result_token_limit" in admin_items
|
||||
assert "max_execution_steps" in admin_items
|
||||
assert "secret_setting" in superadmin_items
|
||||
|
||||
|
||||
def test_filter_config_by_role_removes_unauthorized_context_values():
|
||||
config_json = {
|
||||
"context": {
|
||||
"system_prompt": "visible",
|
||||
"summary_threshold": 10,
|
||||
"summary_keep_messages": 8,
|
||||
"summary_prompt": "custom summary",
|
||||
"summary_tool_result_token_limit": 500,
|
||||
"max_execution_steps": 50,
|
||||
"secret_setting": "nope",
|
||||
},
|
||||
"other": {"keep": True},
|
||||
}
|
||||
|
||||
filtered = filter_config_by_role(config_json, "user", context_schema=SuperAdminOnlyContext)
|
||||
|
||||
assert filtered == {"context": {"system_prompt": "visible"}, "other": {"keep": True}}
|
||||
assert config_json["context"]["summary_threshold"] == 10
|
||||
|
||||
|
||||
def test_filter_config_by_role_keeps_admin_context_values_for_admin():
|
||||
filtered = filter_config_by_role(
|
||||
{
|
||||
"context": {
|
||||
"summary_threshold": 10,
|
||||
"summary_keep_messages": 8,
|
||||
"summary_prompt": "custom summary",
|
||||
"summary_tool_result_token_limit": 500,
|
||||
"max_execution_steps": 50,
|
||||
"secret_setting": "nope",
|
||||
}
|
||||
},
|
||||
"admin",
|
||||
context_schema=SuperAdminOnlyContext,
|
||||
)
|
||||
|
||||
assert filtered == {
|
||||
"context": {
|
||||
"summary_threshold": 10,
|
||||
"summary_keep_messages": 8,
|
||||
"summary_prompt": "custom summary",
|
||||
"summary_tool_result_token_limit": 500,
|
||||
"max_execution_steps": 50,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_agent_resource_options_empty_fields_loads_nothing(monkeypatch):
|
||||
async def fail_if_loaded(*_args, **_kwargs):
|
||||
raise AssertionError("empty resource_fields should not load resources")
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.knowledge",
|
||||
types.SimpleNamespace(knowledge_base=types.SimpleNamespace(get_databases_by_user=fail_if_loaded)),
|
||||
)
|
||||
|
||||
assert await context_module.resolve_agent_resource_options(set(), db=object(), user=object()) == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalize_agent_context_config_expands_null_and_filters_explicit_lists(monkeypatch):
|
||||
async def fake_get_databases_by_user(_user):
|
||||
return {"databases": [{"kb_id": "kb-a"}, {"kb_id": "kb-b"}]}
|
||||
|
||||
async def fake_get_all_mcp_servers(_db):
|
||||
return [
|
||||
types.SimpleNamespace(slug="mcp-a", name="MCP A", description="", enabled=True),
|
||||
types.SimpleNamespace(slug="mcp-b", name="MCP B", description="", enabled=True),
|
||||
]
|
||||
|
||||
async def fake_list_skills(_db, _user):
|
||||
return [
|
||||
types.SimpleNamespace(slug="skill-a", name="Skill A", description=""),
|
||||
types.SimpleNamespace(slug="skill-b", name="Skill B", description=""),
|
||||
]
|
||||
|
||||
class FakeAgentRepository:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def list_visible_subagents(self, *, user):
|
||||
assert user.uid == "u1"
|
||||
return [
|
||||
types.SimpleNamespace(slug="research-agent", name="Research", description=""),
|
||||
types.SimpleNamespace(slug="critique-agent", name="Critique", description=""),
|
||||
]
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.toolkits.service",
|
||||
types.SimpleNamespace(
|
||||
get_tool_metadata=lambda category=None: [
|
||||
{"slug": "ask_user_question", "name": "Ask User", "description": ""},
|
||||
{"slug": "tavily_search", "name": "Tavily", "description": ""},
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.knowledge",
|
||||
types.SimpleNamespace(knowledge_base=types.SimpleNamespace(get_databases_by_user=fake_get_databases_by_user)),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.mcp.service",
|
||||
types.SimpleNamespace(get_all_mcp_servers=fake_get_all_mcp_servers),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.skills.service",
|
||||
types.SimpleNamespace(list_accessible_skills=fake_list_skills),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.repositories.agent_repository",
|
||||
types.SimpleNamespace(AgentRepository=FakeAgentRepository),
|
||||
)
|
||||
|
||||
normalized = await normalize_agent_context_config(
|
||||
{
|
||||
"tools": None,
|
||||
"knowledges": ["kb-b", "missing", "kb-b"],
|
||||
"mcps": None,
|
||||
"skills": [],
|
||||
"subagents": ["research-agent", "missing"],
|
||||
"summary_threshold": 10,
|
||||
"summary_keep_messages": 8,
|
||||
"summary_prompt": "custom summary",
|
||||
"summary_tool_result_token_limit": 500,
|
||||
"max_execution_steps": 50,
|
||||
},
|
||||
db=object(),
|
||||
user=types.SimpleNamespace(role="user", uid="u1", department_id=None),
|
||||
context_schema=ChatBotContext,
|
||||
)
|
||||
|
||||
assert normalized["tools"] == ["ask_user_question", "tavily_search"]
|
||||
assert normalized["knowledges"] == ["kb-b"]
|
||||
assert normalized["mcps"] == ["mcp-a", "mcp-b"]
|
||||
assert normalized["skills"] == []
|
||||
assert normalized["subagents"] == ["research-agent"]
|
||||
assert "summary_threshold" not in normalized
|
||||
assert "summary_keep_messages" not in normalized
|
||||
assert "summary_prompt" not in normalized
|
||||
assert "summary_tool_result_token_limit" not in normalized
|
||||
assert "max_execution_steps" not in normalized
|
||||
|
||||
empty_subagents_normalized = await normalize_agent_context_config(
|
||||
{"tools": [], "knowledges": [], "mcps": [], "skills": [], "subagents": []},
|
||||
db=object(),
|
||||
user=types.SimpleNamespace(role="user", uid="u1", department_id=None),
|
||||
context_schema=ChatBotContext,
|
||||
)
|
||||
|
||||
assert empty_subagents_normalized["subagents"] == ["research-agent", "critique-agent"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_agent_runtime_context_filters_resources_and_derives_runtime_scope(monkeypatch):
|
||||
async def fake_get_databases_by_user(_user):
|
||||
return {"databases": [{"kb_id": "kb-a"}, {"kb_id": "kb-b"}]}
|
||||
|
||||
async def fake_get_all_mcp_servers(_db):
|
||||
return [types.SimpleNamespace(slug="mcp-a", name="MCP A", description="", enabled=True)]
|
||||
|
||||
async def fake_list_skills(_db, _user):
|
||||
return [
|
||||
types.SimpleNamespace(slug="skill-a", name="Skill A", description=""),
|
||||
types.SimpleNamespace(slug="skill-b", name="Skill B", description=""),
|
||||
]
|
||||
|
||||
async def fake_resolve_visible_knowledge_bases(context):
|
||||
assert context.knowledges == ["kb-a"]
|
||||
context._visible_knowledge_bases = [{"slug": "kb-a", "name": "Docs A"}]
|
||||
return context._visible_knowledge_bases
|
||||
|
||||
async def fake_resolve_runtime_skills_for_context(context, *, db=None, user=None):
|
||||
del db
|
||||
assert user.uid == "u1"
|
||||
assert context.skills == ["skill-a"]
|
||||
return {
|
||||
"context_skills": ["skill-a"],
|
||||
"prompt_skills": ["skill-a", "skill-b"],
|
||||
"readable_skills": ["skill-a", "skill-b"],
|
||||
"runtime_skill_metadata": {"skill-a": {"name": "Skill A"}},
|
||||
"runtime_skill_dependency_map": {"skill-a": {"skills": ["skill-b"]}},
|
||||
}
|
||||
|
||||
class FakeSessionContext:
|
||||
async def __aenter__(self):
|
||||
return object()
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
class FakeUserRepository:
|
||||
async def get_by_uid_with_db(self, _db, uid):
|
||||
assert uid == "u1"
|
||||
return types.SimpleNamespace(role="user", uid="u1", department_id=None)
|
||||
|
||||
class FakeAgentRepository:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def list_visible_subagents(self, *, user):
|
||||
assert user.uid == "u1"
|
||||
return [types.SimpleNamespace(slug="research-agent", name="Research", description="")]
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.backends.knowledge_base_backend",
|
||||
types.SimpleNamespace(resolve_visible_knowledge_bases_for_context=fake_resolve_visible_knowledge_bases),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.middlewares.skills",
|
||||
types.SimpleNamespace(resolve_runtime_skills_for_context=fake_resolve_runtime_skills_for_context),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.repositories.user_repository",
|
||||
types.SimpleNamespace(UserRepository=FakeUserRepository),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.storage.postgres.manager",
|
||||
types.SimpleNamespace(pg_manager=types.SimpleNamespace(get_async_session_context=lambda: FakeSessionContext())),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.toolkits.service",
|
||||
types.SimpleNamespace(
|
||||
get_tool_metadata=lambda category=None: [
|
||||
{"slug": "ask_user_question", "name": "Ask User", "description": ""}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.knowledge",
|
||||
types.SimpleNamespace(knowledge_base=types.SimpleNamespace(get_databases_by_user=fake_get_databases_by_user)),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.mcp.service",
|
||||
types.SimpleNamespace(get_all_mcp_servers=fake_get_all_mcp_servers),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.skills.service",
|
||||
types.SimpleNamespace(list_accessible_skills=fake_list_skills),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.repositories.agent_repository",
|
||||
types.SimpleNamespace(AgentRepository=FakeAgentRepository),
|
||||
)
|
||||
context = ChatBotContext(
|
||||
uid="u1",
|
||||
tools=["ask_user_question", "missing"],
|
||||
knowledges=["kb-a", "missing"],
|
||||
mcps=None,
|
||||
skills=["skill-a", "missing"],
|
||||
subagents=[],
|
||||
)
|
||||
|
||||
prepared = await context_module.prepare_agent_runtime_context(context)
|
||||
|
||||
assert prepared.tools == ["ask_user_question"]
|
||||
assert prepared.knowledges == ["kb-a"]
|
||||
assert prepared.mcps == ["mcp-a"]
|
||||
assert prepared.skills == ["skill-a"]
|
||||
assert prepared.subagents == ["research-agent"]
|
||||
assert prepared._visible_knowledge_bases == [{"slug": "kb-a", "name": "Docs A"}]
|
||||
assert prepared._prompt_skills == ["skill-a", "skill-b"]
|
||||
assert prepared._readable_skills == ["skill-a", "skill-b"]
|
||||
assert prepared._runtime_skill_metadata == {"skill-a": {"name": "Skill A"}}
|
||||
assert prepared._runtime_skill_dependency_map == {"skill-a": {"skills": ["skill-b"]}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_agent_runtime_context_clears_resources_for_missing_user(monkeypatch):
|
||||
class FakeSessionContext:
|
||||
async def __aenter__(self):
|
||||
return object()
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
class FakeUserRepository:
|
||||
async def get_by_uid_with_db(self, _db, _uid):
|
||||
return None
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.backends.knowledge_base_backend",
|
||||
types.SimpleNamespace(resolve_visible_knowledge_bases_for_context=lambda _context: None),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.agents.middlewares.skills",
|
||||
types.SimpleNamespace(resolve_runtime_skills_for_context=lambda _context, db=None, user=None: None),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.repositories.user_repository",
|
||||
types.SimpleNamespace(UserRepository=FakeUserRepository),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"yuxi.storage.postgres.manager",
|
||||
types.SimpleNamespace(pg_manager=types.SimpleNamespace(get_async_session_context=lambda: FakeSessionContext())),
|
||||
)
|
||||
|
||||
context = ChatBotContext(
|
||||
uid="missing",
|
||||
tools=["tool"],
|
||||
knowledges=["kb"],
|
||||
mcps=["mcp"],
|
||||
skills=["skill"],
|
||||
subagents=["agent"],
|
||||
)
|
||||
|
||||
prepared = await context_module.prepare_agent_runtime_context(context)
|
||||
|
||||
assert prepared.tools == []
|
||||
assert prepared.knowledges == []
|
||||
assert prepared.mcps == []
|
||||
assert prepared.skills == []
|
||||
assert prepared.subagents == []
|
||||
assert prepared._visible_knowledge_bases == []
|
||||
assert prepared._prompt_skills == []
|
||||
assert prepared._readable_skills == []
|
||||
assert prepared._runtime_skill_metadata == {}
|
||||
assert prepared._runtime_skill_dependency_map == {}
|
||||
@@ -0,0 +1,112 @@
|
||||
"""回归测试:流式 tool_call 续片空串 name/id 归一化,规避 LangGraph v3 累积缺陷。
|
||||
|
||||
背景:v3 流式累积对 tool_call 字段是“后值覆盖”,部分 OpenAI 兼容提供商
|
||||
(siliconflow、阿里云百炼等)在续片里把 name/id 下发为空字符串 "",会覆盖首片的
|
||||
真实值(丢 name / 丢 id),导致工具结果无法按 tool_call_id 关联。
|
||||
`_normalize_tool_call_chunks` 把空串归一化为 None(对齐 OpenAI 官方)来规避。
|
||||
|
||||
本测试用 fake 流式模型确定性复现该缺陷(无需网络/API key),并验证修复有效。
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
from langchain_core.messages import AIMessageChunk, HumanMessage
|
||||
from langchain_core.messages.tool import tool_call_chunk
|
||||
from langchain_core.outputs import ChatGenerationChunk, ChatResult
|
||||
from langchain_core.tools import tool
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.errors import GraphRecursionError
|
||||
|
||||
from yuxi.agents.models import _normalize_tool_call_chunks
|
||||
|
||||
|
||||
@tool
|
||||
def get_weather(city: str) -> str:
|
||||
"""查询指定城市的天气。"""
|
||||
return f"{city} 晴 25℃"
|
||||
|
||||
|
||||
class _FakeSiliconFlowModel(BaseChatModel):
|
||||
"""模拟 SiliconFlow 流式:首片带 name+id,续片 name=''(空串)。
|
||||
|
||||
`apply_fix=True` 时在续片产出后调用 `_normalize_tool_call_chunks`,
|
||||
复刻 `_ToolCallChunkFixChatOpenAI` 的归一化行为。
|
||||
"""
|
||||
|
||||
apply_fix: bool = False
|
||||
call_count: int = 0
|
||||
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
return "fake-siliconflow"
|
||||
|
||||
def bind_tools(self, tools, **kwargs): # noqa: ARG002
|
||||
return self
|
||||
|
||||
async def _astream(self, messages, stop=None, run_manager=None, **kwargs): # noqa: ARG002
|
||||
self.call_count += 1
|
||||
call_id = f"call_{self.call_count}"
|
||||
deltas = [
|
||||
tool_call_chunk(name="get_weather", args="", id=call_id, index=0),
|
||||
tool_call_chunk(name="", args='{"city": ', id=None, index=0),
|
||||
tool_call_chunk(name="", args='"北京"}', id=None, index=0),
|
||||
]
|
||||
for delta in deltas:
|
||||
chunk = ChatGenerationChunk(message=AIMessageChunk(content="", tool_call_chunks=[delta]))
|
||||
if self.apply_fix:
|
||||
_normalize_tool_call_chunks(chunk.message)
|
||||
yield chunk
|
||||
|
||||
def _generate(self, messages, stop=None, run_manager=None, **kwargs): # noqa: ARG002
|
||||
raise NotImplementedError("仅用于流式测试")
|
||||
|
||||
|
||||
async def _run_and_get_tool_calls(model: BaseChatModel) -> list[dict]:
|
||||
agent = create_agent(model=model, tools=[get_weather], checkpointer=InMemorySaver())
|
||||
config = {"configurable": {"thread_id": "t"}, "recursion_limit": 4}
|
||||
graph_input = {"messages": [HumanMessage("北京天气?")]}
|
||||
try:
|
||||
run = await agent.astream_events(graph_input, config=config, version="v3")
|
||||
async for _ in run:
|
||||
pass
|
||||
except GraphRecursionError:
|
||||
pass # name 丢失会导致死循环,这里只取已落到 state 的 tool_call
|
||||
state = await agent.aget_state(config)
|
||||
tool_calls: list[dict] = []
|
||||
for msg in state.values.get("messages", []):
|
||||
if msg.type == "ai" and msg.tool_calls:
|
||||
tool_calls.extend(msg.tool_calls)
|
||||
return tool_calls
|
||||
|
||||
|
||||
def test_normalize_replaces_empty_string_with_none():
|
||||
msg = AIMessageChunk(
|
||||
content="",
|
||||
tool_call_chunks=[
|
||||
tool_call_chunk(name="", args="{}", id="", index=0),
|
||||
tool_call_chunk(name="foo", args="{}", id="abc", index=1),
|
||||
],
|
||||
)
|
||||
_normalize_tool_call_chunks(msg)
|
||||
assert msg.tool_call_chunks[0]["name"] is None
|
||||
assert msg.tool_call_chunks[0]["id"] is None
|
||||
# 非空值保持不变
|
||||
assert msg.tool_call_chunks[1]["name"] == "foo"
|
||||
assert msg.tool_call_chunks[1]["id"] == "abc"
|
||||
|
||||
|
||||
async def test_v3_loses_name_without_fix():
|
||||
"""对照组:复现上游缺陷——不归一化时 v3 累积出的 tool_call 真实 name 被空串覆盖。"""
|
||||
tool_calls = await _run_and_get_tool_calls(_FakeSiliconFlowModel(apply_fix=False))
|
||||
assert tool_calls, "应至少累积出一个 tool_call"
|
||||
assert tool_calls[0]["name"] == "", "未修复时首片真实 name 应被续片空串覆盖"
|
||||
|
||||
|
||||
async def test_v3_preserves_name_with_fix():
|
||||
"""修复组:归一化空串后 v3 累积出的 tool_call 保留完整 name/id 与参数。"""
|
||||
tool_calls = await _run_and_get_tool_calls(_FakeSiliconFlowModel(apply_fix=True))
|
||||
assert tool_calls, "应至少累积出一个 tool_call"
|
||||
assert all(tc["name"] == "get_weather" for tc in tool_calls)
|
||||
assert all(tc["id"] for tc in tool_calls)
|
||||
assert tool_calls[0]["args"] == {"city": "北京"}
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.buildin.subagent import graph as subagent_graph
|
||||
|
||||
|
||||
class _Request:
|
||||
def __init__(self, tools):
|
||||
self.tools = tools
|
||||
|
||||
def override(self, **kwargs):
|
||||
return _Request(kwargs.get("tools", self.tools))
|
||||
|
||||
|
||||
def test_filter_disabled_tools_keeps_allowed_tools_order():
|
||||
tools = [
|
||||
SimpleNamespace(name="search"),
|
||||
SimpleNamespace(name="present_artifacts"),
|
||||
{"name": "ask_user_question"},
|
||||
SimpleNamespace(name="install_skill"),
|
||||
SimpleNamespace(name="calculator"),
|
||||
]
|
||||
|
||||
filtered = subagent_graph._filter_disabled_tools(tools)
|
||||
|
||||
assert [subagent_graph._tool_name(tool) for tool in filtered] == ["search", "calculator"]
|
||||
|
||||
|
||||
def test_subagent_tool_filter_middleware_filters_before_handler():
|
||||
middleware = subagent_graph._SubAgentToolFilterMiddleware()
|
||||
seen = {}
|
||||
|
||||
def handler(request):
|
||||
seen["tools"] = request.tools
|
||||
return "ok"
|
||||
|
||||
result = middleware.wrap_model_call(
|
||||
_Request([
|
||||
SimpleNamespace(name="present_artifacts"),
|
||||
SimpleNamespace(name="allowed_tool"),
|
||||
]),
|
||||
handler,
|
||||
)
|
||||
|
||||
assert result == "ok"
|
||||
assert [tool.name for tool in seen["tools"]] == ["allowed_tool"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_tool_filter_middleware_filters_async_before_handler():
|
||||
middleware = subagent_graph._SubAgentToolFilterMiddleware()
|
||||
seen = {}
|
||||
|
||||
async def handler(request):
|
||||
seen["tools"] = request.tools
|
||||
return "ok"
|
||||
|
||||
result = await middleware.awrap_model_call(
|
||||
_Request([
|
||||
{"name": "ask_user_question"},
|
||||
SimpleNamespace(name="allowed_tool"),
|
||||
]),
|
||||
handler,
|
||||
)
|
||||
|
||||
assert result == "ok"
|
||||
assert [subagent_graph._tool_name(tool) for tool in seen["tools"]] == ["allowed_tool"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_get_info_hides_disabled_tool_options(monkeypatch):
|
||||
async def get_info(_self, **_kwargs):
|
||||
return {
|
||||
"metadata": {},
|
||||
"configurable_items": {
|
||||
"tools": {
|
||||
"options": [
|
||||
{"key": "present_artifacts", "name": "展示交付物"},
|
||||
{"key": "allowed_tool", "name": "Allowed"},
|
||||
{"key": "ask_user_question", "name": "向用户提问"},
|
||||
{"key": "install_skill", "name": "安装技能"},
|
||||
]
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(subagent_graph.BaseAgent, "get_info", get_info)
|
||||
|
||||
info = await subagent_graph.SubAgentBackend().get_info()
|
||||
|
||||
assert [option["key"] for option in info["configurable_items"]["tools"]["options"]] == ["allowed_tool"]
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.buildin.chatbot import graph as chatbot_graph
|
||||
from yuxi.agents.buildin.subagent import graph as subagent_graph
|
||||
|
||||
|
||||
def _context(summary_threshold: int = 123) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
model="",
|
||||
summary_threshold=summary_threshold,
|
||||
summary_keep_messages=7,
|
||||
summary_prompt="SUMMARY {messages}",
|
||||
summary_tool_result_token_limit=300,
|
||||
summary_l2_trigger_ratio=0.75,
|
||||
tool_token_limit=3,
|
||||
model_retry_times=1,
|
||||
)
|
||||
|
||||
|
||||
def _patch_common_graph_deps(monkeypatch: pytest.MonkeyPatch, graph_module, captured: dict) -> None:
|
||||
monkeypatch.setattr(graph_module, "load_chat_model", lambda fully_specified_name: object())
|
||||
monkeypatch.setattr(graph_module, "create_agent_filesystem_middleware", lambda *_args, **_kwargs: object())
|
||||
|
||||
def create_summary_middleware(**kwargs):
|
||||
captured["summary_kwargs"] = kwargs
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(graph_module, "create_summary_middleware", create_summary_middleware)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.asyncio
|
||||
async def test_chatbot_summary_trim_limit_matches_summary_threshold(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
captured: dict = {}
|
||||
_patch_common_graph_deps(monkeypatch, chatbot_graph, captured)
|
||||
|
||||
async def no_subagent_middleware(_context):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(chatbot_graph, "create_subagent_task_middleware", no_subagent_middleware)
|
||||
|
||||
await chatbot_graph._build_middlewares(_context(summary_threshold=123))
|
||||
|
||||
assert captured["summary_kwargs"]["trigger"] == ("tokens", 123 * 1024)
|
||||
assert captured["summary_kwargs"]["trim_tokens_to_summarize"] == 123 * 1024
|
||||
assert captured["summary_kwargs"]["l1_l2_trigger_ratio"] == 0.75
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_summary_trim_limit_matches_summary_threshold(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
captured: dict = {}
|
||||
_patch_common_graph_deps(monkeypatch, subagent_graph, captured)
|
||||
|
||||
await subagent_graph._build_middlewares(_context(summary_threshold=64))
|
||||
|
||||
assert captured["summary_kwargs"]["trigger"] == ("tokens", 64 * 1024)
|
||||
assert captured["summary_kwargs"]["trim_tokens_to_summarize"] == 64 * 1024
|
||||
assert captured["summary_kwargs"]["l1_l2_trigger_ratio"] == 0.75
|
||||
@@ -0,0 +1,85 @@
|
||||
"""测试内置 ask_user_question 工具的格式契约。"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.toolkits.buildin import tools
|
||||
|
||||
|
||||
def test_ask_user_question_interrupt_payload_and_result_format(monkeypatch):
|
||||
captured_payloads = []
|
||||
expected_answer = {"style": "simple"}
|
||||
|
||||
def fake_interrupt(payload):
|
||||
captured_payloads.append(payload)
|
||||
return expected_answer
|
||||
|
||||
monkeypatch.setattr(tools, "interrupt", fake_interrupt)
|
||||
|
||||
result = tools.ask_user_question.func(
|
||||
questions=[
|
||||
{
|
||||
"question_id": "style",
|
||||
"question": "选择界面风格",
|
||||
"options": [
|
||||
{"label": "简洁 (Recommended)", "value": "simple"},
|
||||
{"label": "详细", "value": "detailed"},
|
||||
],
|
||||
"multi_select": False,
|
||||
"allow_other": False,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
expected_questions = [
|
||||
{
|
||||
"question_id": "style",
|
||||
"question": "选择界面风格",
|
||||
"options": [
|
||||
{"label": "简洁 (Recommended)", "value": "simple"},
|
||||
{"label": "详细", "value": "detailed"},
|
||||
],
|
||||
"multi_select": False,
|
||||
"allow_other": False,
|
||||
}
|
||||
]
|
||||
|
||||
assert captured_payloads == [{"questions": expected_questions, "source": "ask_user_question"}]
|
||||
assert result == {"questions": expected_questions, "answer": expected_answer}
|
||||
|
||||
|
||||
def test_ask_user_question_accepts_json_string_questions(monkeypatch):
|
||||
captured_payloads = []
|
||||
|
||||
monkeypatch.setattr(tools, "interrupt", lambda payload: captured_payloads.append(payload) or {"q-1": "A"})
|
||||
|
||||
result = tools.ask_user_question.func(
|
||||
questions=json.dumps(
|
||||
[
|
||||
{
|
||||
"question": "选择一个选项",
|
||||
"options": ["A", "B"],
|
||||
"allow_other": False,
|
||||
}
|
||||
],
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert captured_payloads[0]["source"] == "ask_user_question"
|
||||
assert captured_payloads[0]["questions"] == [
|
||||
{
|
||||
"question_id": "q-1",
|
||||
"question": "选择一个选项",
|
||||
"options": [{"label": "A", "value": "A"}, {"label": "B", "value": "B"}],
|
||||
"multi_select": False,
|
||||
"allow_other": False,
|
||||
}
|
||||
]
|
||||
assert result["answer"] == {"q-1": "A"}
|
||||
|
||||
|
||||
def test_ask_user_question_rejects_empty_questions():
|
||||
with pytest.raises(ValueError, match="questions 至少需要包含一个有效问题"):
|
||||
tools.ask_user_question.func(questions=[])
|
||||
@@ -0,0 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
import yuxi.agents.backends.knowledge_base_backend as knowledge_base_backend
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_visible_knowledge_bases_requires_slug(monkeypatch):
|
||||
async def fake_get_databases_by_uid(_uid):
|
||||
return {"databases": [{"id": "legacy-id", "name": "Legacy"}]}
|
||||
|
||||
monkeypatch.setattr(knowledge_base_backend.knowledge_base, "get_databases_by_uid", fake_get_databases_by_uid)
|
||||
|
||||
context = SimpleNamespace(uid="u1", knowledges=["legacy-id"])
|
||||
|
||||
databases = await knowledge_base_backend.resolve_visible_knowledge_bases_for_context(context)
|
||||
|
||||
assert databases == []
|
||||
@@ -0,0 +1,661 @@
|
||||
"""Tests for sandbox backend components."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import threading
|
||||
from types import MethodType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from deepagents.backends.protocol import GlobResult, ReadResult
|
||||
from deepagents.backends.sandbox import MAX_BINARY_BYTES
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.prebuilt.tool_node import ToolRuntime
|
||||
from yuxi.agents.backends.composite import (
|
||||
CustomCompositeBackend,
|
||||
create_agent_composite_backend,
|
||||
create_agent_filesystem_middleware,
|
||||
)
|
||||
from yuxi.agents.backends.sandbox import resolve_virtual_path, sandbox_id_for_thread
|
||||
from yuxi.agents.backends.sandbox.backend import ProvisionerSandboxBackend
|
||||
from yuxi.agents.middlewares.skills import SkillsMiddleware
|
||||
from yuxi.utils.paths import VIRTUAL_PATH_CONVERSATION_HISTORY, VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
|
||||
|
||||
def _runtime(
|
||||
*,
|
||||
thread_id: str | None = "thread-1",
|
||||
uid: str | None = "user-1",
|
||||
skills: list[str] | None = None,
|
||||
readable_skills: list[str] | None = None,
|
||||
visible_kbs: list[dict] | None = None,
|
||||
):
|
||||
configurable = {"thread_id": thread_id, "uid": uid} if thread_id and uid else {}
|
||||
return SimpleNamespace(
|
||||
config={"configurable": configurable},
|
||||
context=SimpleNamespace(
|
||||
skills=skills or [],
|
||||
_readable_skills=readable_skills,
|
||||
_visible_knowledge_bases=visible_kbs or [],
|
||||
uid=uid,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_create_agent_composite_backend_uses_prepared_readable_skills(monkeypatch):
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
|
||||
backend = create_agent_composite_backend(
|
||||
_runtime(readable_skills=["reporter"], visible_kbs=[{"slug": "db-1", "name": "Docs"}])
|
||||
)
|
||||
|
||||
assert isinstance(backend.default, ProvisionerSandboxBackend)
|
||||
assert backend.default._readable_skills == ["reporter"]
|
||||
assert backend.artifacts_root == "/home/gem/user-data/outputs"
|
||||
assert "/skills/" in backend.routes
|
||||
assert "/home/gem/kbs/" not in backend.routes
|
||||
|
||||
|
||||
def test_create_agent_composite_backend_requires_thread_id():
|
||||
with pytest.raises(ValueError, match="thread_id is required"):
|
||||
create_agent_composite_backend(_runtime(thread_id=None))
|
||||
|
||||
|
||||
def test_create_agent_composite_backend_ignores_unprepared_context_skills(monkeypatch):
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
|
||||
backend = create_agent_composite_backend(_runtime(skills=["configured"], readable_skills=None))
|
||||
|
||||
assert backend.default._readable_skills == []
|
||||
|
||||
|
||||
def test_create_agent_composite_backend_uses_split_thread_scopes(monkeypatch):
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
runtime = _runtime(thread_id="child-thread", uid="user-1", readable_skills=["worker-skill"])
|
||||
runtime.config["configurable"].update(
|
||||
{"file_thread_id": "parent-thread", "skills_thread_id": "child-skills-thread"}
|
||||
)
|
||||
|
||||
backend = create_agent_composite_backend(runtime)
|
||||
|
||||
assert backend.default._thread_id == "child-thread"
|
||||
assert backend.default._file_thread_id == "parent-thread"
|
||||
assert backend.default._skills_thread_id == "child-skills-thread"
|
||||
assert backend.default._readable_skills == ["worker-skill"]
|
||||
|
||||
|
||||
def test_create_agent_composite_backend_uses_split_thread_scopes_from_state(monkeypatch):
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
runtime = _runtime(thread_id="child-thread", uid="user-1", readable_skills=["worker-skill"])
|
||||
runtime.state = {"file_thread_id": "parent-thread", "skills_thread_id": "child-skills-thread"}
|
||||
|
||||
backend = create_agent_composite_backend(runtime)
|
||||
|
||||
assert backend.default._thread_id == "child-thread"
|
||||
assert backend.default._file_thread_id == "parent-thread"
|
||||
assert backend.default._skills_thread_id == "child-skills-thread"
|
||||
|
||||
|
||||
def test_create_agent_filesystem_middleware_uses_context_scope(monkeypatch):
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
context = SimpleNamespace(
|
||||
thread_id="child-thread",
|
||||
uid="user-1",
|
||||
file_thread_id="parent-thread",
|
||||
skills_thread_id="child-skills-thread",
|
||||
_readable_skills=["worker-skill"],
|
||||
)
|
||||
|
||||
middleware = create_agent_filesystem_middleware(context=context)
|
||||
backend = middleware.backend
|
||||
|
||||
assert backend.default._thread_id == "child-thread"
|
||||
assert backend.default._file_thread_id == "parent-thread"
|
||||
assert backend.default._skills_thread_id == "child-skills-thread"
|
||||
assert backend.default._readable_skills == ["worker-skill"]
|
||||
|
||||
|
||||
def test_create_agent_filesystem_middleware_uses_outputs_for_internal_artifacts() -> None:
|
||||
middleware = create_agent_filesystem_middleware(tool_token_limit_before_evict=500)
|
||||
|
||||
assert middleware._tool_token_limit_before_evict == 500
|
||||
assert middleware._large_tool_results_prefix == VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
assert middleware._conversation_history_prefix == VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
|
||||
|
||||
def test_filesystem_middleware_evicts_large_non_read_file_tool_result() -> None:
|
||||
class _Backend:
|
||||
artifacts_root = "/"
|
||||
|
||||
def __init__(self):
|
||||
self.writes: list[tuple[str, str]] = []
|
||||
|
||||
def write(self, path: str, content: str):
|
||||
self.writes.append((path, content))
|
||||
return SimpleNamespace(error=None)
|
||||
|
||||
backend = _Backend()
|
||||
middleware = create_agent_filesystem_middleware(tool_token_limit_before_evict=1)
|
||||
middleware.backend = backend
|
||||
request = SimpleNamespace(tool_call={"name": "grep"}, runtime=SimpleNamespace())
|
||||
content = "BEGIN\n" + ("middle\n" * 5000) + "END"
|
||||
|
||||
result = middleware.wrap_tool_call(
|
||||
request,
|
||||
lambda _: ToolMessage(content=content, name="grep", tool_call_id="call-grep"),
|
||||
)
|
||||
|
||||
assert backend.writes == [(f"{VIRTUAL_PATH_LARGE_TOOL_RESULTS}/call-grep", content)]
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert len(result.content) < len(content)
|
||||
assert f"{VIRTUAL_PATH_LARGE_TOOL_RESULTS}/call-grep" in result.content
|
||||
|
||||
|
||||
def test_filesystem_middleware_keeps_read_file_result_inline_to_avoid_evict_loop() -> None:
|
||||
class _Backend:
|
||||
artifacts_root = "/"
|
||||
|
||||
def __init__(self):
|
||||
self.writes: list[tuple[str, str]] = []
|
||||
|
||||
def write(self, path: str, content: str):
|
||||
self.writes.append((path, content))
|
||||
return SimpleNamespace(error=None)
|
||||
|
||||
backend = _Backend()
|
||||
middleware = create_agent_filesystem_middleware(tool_token_limit_before_evict=1)
|
||||
middleware.backend = backend
|
||||
request = SimpleNamespace(tool_call={"name": "read_file"}, runtime=SimpleNamespace())
|
||||
content = "x" * 100
|
||||
|
||||
result = middleware.wrap_tool_call(
|
||||
request,
|
||||
lambda _: ToolMessage(content=content, name="read_file", tool_call_id="call-read"),
|
||||
)
|
||||
|
||||
assert backend.writes == []
|
||||
assert result.content == content
|
||||
|
||||
|
||||
def test_custom_composite_glob_only_searches_routes_from_root() -> None:
|
||||
class _Backend:
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.calls: list[tuple[str, str]] = []
|
||||
|
||||
def glob(self, pattern: str, path: str = "/") -> GlobResult:
|
||||
self.calls.append((pattern, path))
|
||||
return GlobResult(matches=[{"path": f"{path.rstrip('/')}/{self.name}.md"}])
|
||||
|
||||
default = _Backend("default")
|
||||
routed = _Backend("skill")
|
||||
backend = CustomCompositeBackend(default=default, routes={"/skills/": routed})
|
||||
|
||||
result = backend.glob("**/*.md", path="/home/gem/user-data")
|
||||
|
||||
assert result.error is None
|
||||
assert default.calls == [("**/*.md", "/home/gem/user-data")]
|
||||
assert routed.calls == []
|
||||
|
||||
|
||||
def test_skills_middleware_extracts_slug_for_new_paths() -> None:
|
||||
middleware = SkillsMiddleware()
|
||||
assert middleware.skills_sources_for_prompt == ["/home/gem/skills/"]
|
||||
assert middleware._extract_skill_slug_from_skill_md_path("/home/gem/skills/demo-skill/SKILL.md") == "demo-skill"
|
||||
|
||||
|
||||
def test_resolve_virtual_path_rejects_outside_prefix():
|
||||
with pytest.raises(ValueError, match="path must start with"):
|
||||
resolve_virtual_path("thread-1", "/etc/passwd", uid="user-1")
|
||||
|
||||
|
||||
def test_resolve_virtual_path_rejects_path_traversal():
|
||||
with pytest.raises(ValueError, match="path traversal"):
|
||||
resolve_virtual_path("thread-1", "/home/gem/user-data/../secrets", uid="user-1")
|
||||
|
||||
|
||||
def test_sandbox_id_for_thread_is_stable():
|
||||
sid1 = sandbox_id_for_thread("thread-1")
|
||||
sid2 = sandbox_id_for_thread("thread-1")
|
||||
sid3 = sandbox_id_for_thread("thread-2")
|
||||
assert sid1 == sid2
|
||||
assert sid1 != sid3
|
||||
assert len(sid1) == 12
|
||||
|
||||
|
||||
def test_sandbox_id_for_thread_includes_skills_scope():
|
||||
parent_only = sandbox_id_for_thread("parent-thread")
|
||||
split_scope = sandbox_id_for_thread("parent-thread", "child-skills-thread")
|
||||
|
||||
assert split_scope == sandbox_id_for_thread("parent-thread", "child-skills-thread")
|
||||
assert split_scope != parent_only
|
||||
assert sandbox_id_for_thread("parent-thread", "parent-thread") == parent_only
|
||||
|
||||
|
||||
def test_provider_uses_distinct_sandbox_scope_for_different_uid(monkeypatch) -> None:
|
||||
from yuxi.agents.backends.sandbox.provider import ProvisionerSandboxProvider
|
||||
|
||||
created = []
|
||||
|
||||
class FakeClient:
|
||||
def create(self, sandbox_id, thread_id, uid, env, *, file_thread_id=None, skills_thread_id=None):
|
||||
created.append((sandbox_id, thread_id, uid, env, file_thread_id, skills_thread_id))
|
||||
return SimpleNamespace(sandbox_id=sandbox_id, sandbox_url=f"http://sandbox/{uid}")
|
||||
|
||||
def touch(self, _sandbox_id):
|
||||
return True
|
||||
|
||||
provider = ProvisionerSandboxProvider.__new__(ProvisionerSandboxProvider)
|
||||
provider._client = FakeClient()
|
||||
provider._lock = threading.Lock()
|
||||
provider._thread_locks = {}
|
||||
provider._connections = {}
|
||||
provider._last_touch_at = {}
|
||||
provider._touch_interval_seconds = 30
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.provider.load_user_agent_env", lambda uid: {"A": uid})
|
||||
|
||||
sandbox_1 = provider.acquire(
|
||||
"child-thread",
|
||||
uid="user-1",
|
||||
file_thread_id="parent-thread",
|
||||
skills_thread_id="child-skills-thread",
|
||||
)
|
||||
sandbox_2 = provider.acquire(
|
||||
"child-thread",
|
||||
uid="user-2",
|
||||
file_thread_id="parent-thread",
|
||||
skills_thread_id="child-skills-thread",
|
||||
)
|
||||
|
||||
assert sandbox_1 != sandbox_2
|
||||
assert created[0][2] == "user-1"
|
||||
assert created[1][2] == "user-2"
|
||||
|
||||
|
||||
def test_provider_get_create_if_missing_ensures_expected_split_scope(monkeypatch) -> None:
|
||||
from yuxi.agents.backends.sandbox.provider import ProvisionerSandboxProvider
|
||||
|
||||
calls = []
|
||||
|
||||
class FakeClient:
|
||||
def create(self, sandbox_id, thread_id, uid, env, *, file_thread_id=None, skills_thread_id=None):
|
||||
calls.append((sandbox_id, thread_id, uid, env, file_thread_id, skills_thread_id))
|
||||
return SimpleNamespace(sandbox_id=sandbox_id, sandbox_url="http://sandbox")
|
||||
|
||||
def discover(self, _sandbox_id):
|
||||
raise AssertionError("create_if_missing should ensure sandbox through provisioner create")
|
||||
|
||||
provider = ProvisionerSandboxProvider.__new__(ProvisionerSandboxProvider)
|
||||
provider._client = FakeClient()
|
||||
provider._lock = threading.Lock()
|
||||
provider._thread_locks = {}
|
||||
provider._connections = {}
|
||||
provider._last_touch_at = {}
|
||||
provider._touch_interval_seconds = 30
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.provider.load_user_agent_env", lambda uid: {"A": uid})
|
||||
|
||||
connection = provider.get(
|
||||
"child-thread",
|
||||
uid="user-1",
|
||||
create_if_missing=True,
|
||||
file_thread_id="parent-thread",
|
||||
skills_thread_id="child-skills-thread",
|
||||
)
|
||||
|
||||
sandbox_id = sandbox_id_for_thread("parent-thread", "child-skills-thread", uid="user-1")
|
||||
assert connection.sandbox_id == sandbox_id
|
||||
assert connection.file_thread_id == "parent-thread"
|
||||
assert connection.skills_thread_id == "child-skills-thread"
|
||||
assert calls == [
|
||||
(
|
||||
sandbox_id,
|
||||
"child-thread",
|
||||
"user-1",
|
||||
{"A": "user-1"},
|
||||
"parent-thread",
|
||||
"child-skills-thread",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_provisioner_uses_file_and_skills_thread_ids(monkeypatch) -> None:
|
||||
provider_calls = []
|
||||
synced = []
|
||||
|
||||
class FakeProvider:
|
||||
def get(self, thread_id, **kwargs):
|
||||
provider_calls.append((thread_id, kwargs))
|
||||
return SimpleNamespace(sandbox_url="http://sandbox")
|
||||
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: FakeProvider())
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.sandbox.backend.sync_thread_readable_skills",
|
||||
lambda thread_id, skills: synced.append((thread_id, skills)),
|
||||
)
|
||||
|
||||
backend = ProvisionerSandboxBackend(
|
||||
thread_id="child-thread",
|
||||
uid="user-1",
|
||||
readable_skills=["worker-skill"],
|
||||
file_thread_id="parent-thread",
|
||||
skills_thread_id="child-skills-thread",
|
||||
)
|
||||
backend._build_client = MethodType(lambda self, sandbox_url: SimpleNamespace(url=sandbox_url), backend)
|
||||
|
||||
client = backend._get_client()
|
||||
|
||||
assert client.url == "http://sandbox"
|
||||
assert synced == [("child-skills-thread", ["worker-skill"])]
|
||||
assert provider_calls == [
|
||||
(
|
||||
"child-thread",
|
||||
{
|
||||
"uid": "user-1",
|
||||
"create_if_missing": True,
|
||||
"file_thread_id": "parent-thread",
|
||||
"skills_thread_id": "child-skills-thread",
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_provisioner_denies_reads_outside_allowed_roots(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
result = backend.read("/etc/passwd")
|
||||
|
||||
assert result.error == "permission denied for read on '/etc/passwd'"
|
||||
|
||||
|
||||
def test_provisioner_denies_upload_writes(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
write_result = backend.write("/home/gem/user-data/uploads/blocked.txt", "blocked")
|
||||
upload_result = backend.upload_files([("/home/gem/user-data/uploads/blocked.bin", b"blocked")])
|
||||
|
||||
assert write_result.error and "permission denied" in write_result.error
|
||||
assert upload_result[0].error == "permission_denied"
|
||||
|
||||
|
||||
def test_provisioner_allows_outputs_writes(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
def _missing_file(path, offset=0, limit=None):
|
||||
raise FileNotFoundError
|
||||
|
||||
monkeypatch.setattr(backend, "_read_binary", _missing_file)
|
||||
|
||||
calls = []
|
||||
|
||||
def _write_file(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return SimpleNamespace(success=True, message="")
|
||||
|
||||
fake_client = SimpleNamespace(file=SimpleNamespace(write_file=_write_file))
|
||||
backend._get_client = MethodType(lambda self: fake_client, backend)
|
||||
|
||||
result = backend.write("/home/gem/user-data/outputs/report.md", "ok")
|
||||
|
||||
assert result.error is None
|
||||
assert result.path == "/home/gem/user-data/outputs/report.md"
|
||||
assert calls[0]["file"] == "/home/gem/user-data/outputs/report.md"
|
||||
|
||||
|
||||
def test_provisioner_glob_root_searches_readable_roots(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
calls = []
|
||||
|
||||
def _find_files(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return SimpleNamespace(data=SimpleNamespace(files=[f"{kwargs['path']}/match.md"]))
|
||||
|
||||
fake_client = SimpleNamespace(file=SimpleNamespace(find_files=_find_files))
|
||||
backend._get_client = MethodType(lambda self: fake_client, backend)
|
||||
|
||||
result = backend.glob("**/*.md")
|
||||
|
||||
assert result.error is None
|
||||
assert [call["path"] for call in calls] == ["/home/gem/user-data", "/home/gem/skills"]
|
||||
assert [item["path"] for item in result.matches] == [
|
||||
"/home/gem/skills/match.md",
|
||||
"/home/gem/user-data/match.md",
|
||||
]
|
||||
|
||||
|
||||
def test_provisioner_read_preserves_base64_like_plain_text(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
fake_client = SimpleNamespace(
|
||||
file=SimpleNamespace(read_file=lambda **_kwargs: SimpleNamespace(data=SimpleNamespace(content="SGVsbG8=")))
|
||||
)
|
||||
backend._get_client = MethodType(lambda self: fake_client, backend)
|
||||
|
||||
result = backend.read("/home/gem/user-data/outputs/base64-looking.txt")
|
||||
|
||||
assert result.error is None
|
||||
assert result.file_data == {"content": "SGVsbG8=", "encoding": "utf-8"}
|
||||
|
||||
|
||||
def test_provisioner_read_decodes_explicit_base64(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
fake_client = SimpleNamespace(
|
||||
file=SimpleNamespace(
|
||||
read_file=lambda **_kwargs: SimpleNamespace(data=SimpleNamespace(content="SGVsbG8=", encoding="base64"))
|
||||
)
|
||||
)
|
||||
backend._get_client = MethodType(lambda self: fake_client, backend)
|
||||
|
||||
assert backend._read_binary("/home/gem/user-data/outputs/file.bin") == b"Hello"
|
||||
|
||||
|
||||
def test_provisioner_read_file_base64_reads_temp_file_not_shell_output(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
expected = base64.b64encode(b"\x89PNG\r\n\x1a\nimage-bytes").decode("ascii")
|
||||
shell_calls = []
|
||||
|
||||
def _exec_command(**kwargs):
|
||||
shell_calls.append(kwargs)
|
||||
return SimpleNamespace(
|
||||
data=SimpleNamespace(
|
||||
exit_code=0,
|
||||
output="broken\n[... Observation truncated due to length ...]\nbase64",
|
||||
)
|
||||
)
|
||||
|
||||
def _read_file(**kwargs):
|
||||
assert kwargs["file"].startswith("/tmp/yuxi-read-file-")
|
||||
return SimpleNamespace(data=SimpleNamespace(content=expected))
|
||||
|
||||
fake_client = SimpleNamespace(
|
||||
shell=SimpleNamespace(exec_command=_exec_command),
|
||||
file=SimpleNamespace(read_file=_read_file),
|
||||
)
|
||||
backend._get_client = MethodType(lambda self: fake_client, backend)
|
||||
|
||||
result = backend._read_file_base64("/home/gem/user-data/workspace/image.png")
|
||||
|
||||
assert result == expected
|
||||
assert len(shell_calls) == 2
|
||||
assert shell_calls[0]["command"].startswith("python3 -c")
|
||||
assert shell_calls[1]["command"].startswith("rm -f /tmp/yuxi-read-file-")
|
||||
|
||||
|
||||
def test_provisioner_read_reports_binary_files(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
monkeypatch.setattr(backend, "_file_size_bytes", lambda _path: 8)
|
||||
monkeypatch.setattr(backend, "_read_file_base64", lambda _path: "iVBORw0KGgo=")
|
||||
|
||||
result = backend.read("/home/gem/user-data/image.png")
|
||||
|
||||
assert result.error is None
|
||||
assert result.file_data is not None
|
||||
assert result.file_data["encoding"] == "base64"
|
||||
|
||||
|
||||
def test_provisioner_read_treats_known_non_text_extension_as_base64(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
monkeypatch.setattr(backend, "_file_size_bytes", lambda _path: 6)
|
||||
monkeypatch.setattr(backend, "_read_binary", lambda path, offset=0, limit=None: pytest.fail("file API used"))
|
||||
monkeypatch.setattr(backend, "_read_file_base64", lambda _path: "R0lGODlh")
|
||||
|
||||
result = backend.read("/home/gem/user-data/image.gif")
|
||||
|
||||
assert result.error is None
|
||||
assert result.file_data == {"content": "R0lGODlh", "encoding": "base64"}
|
||||
|
||||
|
||||
def test_provisioner_read_rejects_large_known_binary_before_read(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
read_calls: list[tuple[str, int, int | None]] = []
|
||||
monkeypatch.setattr(backend, "_file_size_bytes", lambda _path: MAX_BINARY_BYTES + 1)
|
||||
|
||||
def _read_binary(path, offset=0, limit=None):
|
||||
read_calls.append((path, offset, limit))
|
||||
return b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
monkeypatch.setattr(backend, "_read_binary", _read_binary)
|
||||
monkeypatch.setattr(backend, "_read_file_base64", lambda _path: pytest.fail("binary file was read"))
|
||||
|
||||
result = backend.read("/home/gem/user-data/large.png")
|
||||
|
||||
assert result.file_data is None
|
||||
assert result.error == f"Binary file exceeds maximum preview size of {MAX_BINARY_BYTES} bytes"
|
||||
assert read_calls == []
|
||||
|
||||
|
||||
def test_provisioner_read_rejects_large_unknown_binary_before_full_read(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
read_calls: list[tuple[str, int, int | None]] = []
|
||||
monkeypatch.setattr(backend, "_file_size_bytes", lambda _path: MAX_BINARY_BYTES + 1)
|
||||
|
||||
def _read_binary(path, offset=0, limit=None):
|
||||
read_calls.append((path, offset, limit))
|
||||
return b"\x00binary prefix"
|
||||
|
||||
monkeypatch.setattr(backend, "_read_binary", _read_binary)
|
||||
|
||||
result = backend.read("/home/gem/user-data/large.unknown")
|
||||
|
||||
assert result.file_data is None
|
||||
assert result.error == f"Binary file exceeds maximum preview size of {MAX_BINARY_BYTES} bytes"
|
||||
assert read_calls == [("/home/gem/user-data/large.unknown", 0, 2000)]
|
||||
|
||||
|
||||
def test_provisioner_read_falls_back_to_base64_on_sandbox_utf8_decode_failure(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
monkeypatch.setattr(backend, "_file_size_bytes", lambda _path: 6)
|
||||
|
||||
def _read_binary_raises(path, offset=0, limit=None):
|
||||
raise RuntimeError("'utf-8' codec can't decode byte 0x89 in position 0")
|
||||
|
||||
monkeypatch.setattr(backend, "_read_binary", _read_binary_raises)
|
||||
monkeypatch.setattr(backend, "_read_file_base64", lambda _path: "R0lGODlh")
|
||||
|
||||
result = backend.read("/home/gem/user-data/workspace/uploaded.bin")
|
||||
|
||||
assert result.error is None
|
||||
assert result.file_data == {"content": "R0lGODlh", "encoding": "base64"}
|
||||
|
||||
|
||||
def test_read_file_tool_returns_multimodal_block_for_small_binary() -> None:
|
||||
class _Backend:
|
||||
def read(self, path: str, offset: int = 0, limit: int = 100):
|
||||
return ReadResult(file_data={"content": "R0lGODlh", "encoding": "base64"})
|
||||
|
||||
middleware = create_agent_filesystem_middleware(tool_token_limit_before_evict=None)
|
||||
middleware.backend = _Backend()
|
||||
read_tool = next(tool for tool in middleware.tools if tool.name == "read_file")
|
||||
runtime = ToolRuntime(
|
||||
state={},
|
||||
context=None,
|
||||
config={},
|
||||
stream_writer=lambda _: None,
|
||||
tool_call_id="call-read",
|
||||
store=None,
|
||||
)
|
||||
|
||||
result = read_tool.func(file_path="/home/gem/user-data/uploads/image.gif", runtime=runtime)
|
||||
|
||||
assert result.status == "success"
|
||||
assert result.content_blocks == [{"type": "image", "base64": "R0lGODlh", "mime_type": "image/gif"}]
|
||||
assert result.additional_kwargs == {
|
||||
"read_file_path": "/home/gem/user-data/uploads/image.gif",
|
||||
"read_file_media_type": "image/gif",
|
||||
}
|
||||
|
||||
|
||||
def test_provisioner_read_reports_invalid_path(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
result = backend.read("secret.txt")
|
||||
|
||||
assert result.error == "Invalid path 'secret.txt': path must start with /"
|
||||
|
||||
|
||||
def test_provisioner_read_reports_path_traversal(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
result = backend.read("/home/gem/user-data/../secret.txt")
|
||||
|
||||
assert result.error == "Invalid path '/home/gem/user-data/../secret.txt': path traversal is not allowed"
|
||||
|
||||
|
||||
def test_provisioner_download_files_distinguishes_invalid_path_from_read_failure(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
def _fake_read_binary(path, offset=0, limit=None):
|
||||
raise RuntimeError("sandbox read timeout")
|
||||
|
||||
monkeypatch.setattr(backend, "_read_binary", _fake_read_binary)
|
||||
|
||||
responses = backend.download_files(["bad-path", "/home/gem/user-data/read-failed"])
|
||||
|
||||
assert responses[0].error == "invalid_path"
|
||||
assert responses[1].error.startswith("read_failed")
|
||||
|
||||
|
||||
def test_provisioner_download_files_treats_sandbox_404_as_missing(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
def _missing_from_sandbox(path, offset=0, limit=None):
|
||||
raise RuntimeError("status_code: 404, body: {'message': 'File does not exist'}")
|
||||
|
||||
monkeypatch.setattr(backend, "_read_binary", _missing_from_sandbox)
|
||||
|
||||
responses = backend.download_files(["/home/gem/user-data/outputs/missing.md"])
|
||||
|
||||
assert responses[0].error == "file_not_found"
|
||||
|
||||
|
||||
def test_provisioner_execute_returns_error_response_on_client_failure(monkeypatch) -> None:
|
||||
monkeypatch.setattr("yuxi.agents.backends.sandbox.backend.get_sandbox_provider", lambda: object())
|
||||
backend = ProvisionerSandboxBackend(thread_id="thread-1", uid="user-1")
|
||||
|
||||
class _FakeClient:
|
||||
class shell:
|
||||
@staticmethod
|
||||
def exec_command(**kwargs):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
backend._get_client = MethodType(lambda self: _FakeClient(), backend)
|
||||
result = backend.execute("echo hi")
|
||||
|
||||
assert result.exit_code == 1
|
||||
assert "Error:" in result.output
|
||||
@@ -0,0 +1,169 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
MODULE_NAME = "sandbox_provisioner_app_for_test"
|
||||
|
||||
|
||||
def _find_module_path() -> Path:
|
||||
current = Path(__file__).resolve()
|
||||
for parent in current.parents:
|
||||
candidate = parent / "docker" / "sandbox_provisioner" / "app.py"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
raise FileNotFoundError("docker/sandbox_provisioner/app.py not found from test path")
|
||||
|
||||
|
||||
MODULE_PATH = _find_module_path()
|
||||
|
||||
|
||||
def _load_module():
|
||||
existing = sys.modules.get(MODULE_NAME)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
spec = importlib.util.spec_from_file_location(MODULE_NAME, MODULE_PATH)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[MODULE_NAME] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_canonical_backend_name(monkeypatch):
|
||||
monkeypatch.setenv("PROVISIONER_BACKEND", "memory")
|
||||
module = _load_module()
|
||||
|
||||
assert module.canonical_backend_name("docker") == "docker"
|
||||
assert module.canonical_backend_name("kubernetes") == "kubernetes"
|
||||
|
||||
|
||||
def test_merged_sandbox_env_user_values_override_global(monkeypatch):
|
||||
monkeypatch.setenv("PROVISIONER_BACKEND", "memory")
|
||||
module = _load_module()
|
||||
|
||||
assert module.merged_sandbox_env(
|
||||
{"SHARED": "global", "GLOBAL_ONLY": "value"},
|
||||
{"SHARED": "user", "USER_ONLY": "value"},
|
||||
) == {
|
||||
"SHARED": "user",
|
||||
"GLOBAL_ONLY": "value",
|
||||
"USER_ONLY": "value",
|
||||
}
|
||||
|
||||
|
||||
def test_normalize_env_converts_values_to_strings(monkeypatch):
|
||||
monkeypatch.setenv("PROVISIONER_BACKEND", "memory")
|
||||
module = _load_module()
|
||||
|
||||
assert module.normalize_env({"A": 1, "B": None, "": "ignored"}) == {"A": "1", "B": ""}
|
||||
|
||||
|
||||
def test_local_container_identity_validation_rejects_unsafe_path_segments(monkeypatch):
|
||||
monkeypatch.setenv("PROVISIONER_BACKEND", "memory")
|
||||
module = _load_module()
|
||||
backend_cls = module.LocalContainerProvisionerBackend
|
||||
|
||||
assert backend_cls._validate_thread_id("thread-1_2") == "thread-1_2"
|
||||
assert backend_cls._validate_uid("user-1_2") == "user-1_2"
|
||||
|
||||
for value in ["../escape", "thread/name", "thread name", "thread;rm", "thread.name"]:
|
||||
with pytest.raises(ValueError):
|
||||
backend_cls._validate_thread_id(value)
|
||||
|
||||
for value in ["../user", "user/name", "user name", "user;rm", "user.name"]:
|
||||
with pytest.raises(ValueError):
|
||||
backend_cls._validate_uid(value)
|
||||
|
||||
|
||||
def test_memory_backend_accepts_split_thread_ids(monkeypatch):
|
||||
monkeypatch.setenv("PROVISIONER_BACKEND", "memory")
|
||||
module = _load_module()
|
||||
backend = module.MemoryProvisionerBackend()
|
||||
|
||||
record = backend.create(
|
||||
"sandbox-1",
|
||||
"child-thread",
|
||||
"user-1",
|
||||
file_thread_id="parent-thread",
|
||||
skills_thread_id="child-skills-thread",
|
||||
)
|
||||
|
||||
assert record.sandbox_id == "sandbox-1"
|
||||
assert backend.discover("sandbox-1") is record
|
||||
|
||||
|
||||
def test_docker_mount_checks_use_file_and_skills_thread_ids(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("PROVISIONER_BACKEND", "memory")
|
||||
module = _load_module()
|
||||
backend = object.__new__(module.LocalContainerProvisionerBackend)
|
||||
backend._threads_host_path = str(tmp_path)
|
||||
|
||||
workspace = tmp_path / "shared" / "user-1" / "workspace"
|
||||
uploads = tmp_path / "parent-thread" / "user-data" / "uploads"
|
||||
outputs = tmp_path / "parent-thread" / "user-data" / "outputs"
|
||||
skills = tmp_path / "child-skills-thread" / "skills"
|
||||
container = SimpleNamespace(
|
||||
attrs={
|
||||
"Mounts": [
|
||||
{"Destination": "/home/gem/user-data/workspace", "Source": str(workspace)},
|
||||
{"Destination": "/home/gem/user-data/uploads", "Source": str(uploads)},
|
||||
{"Destination": "/home/gem/user-data/outputs", "Source": str(outputs)},
|
||||
{"Destination": "/home/gem/skills", "Source": str(skills)},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert backend._has_expected_user_data_mounts(container, "parent-thread", "user-1") is True
|
||||
assert backend._is_expected_skills_mount(container, "child-skills-thread") is True
|
||||
assert backend._has_expected_user_data_mounts(container, "child-thread", "user-1") is False
|
||||
assert backend._is_expected_skills_mount(container, "parent-thread") is False
|
||||
|
||||
|
||||
def test_kubernetes_mount_check_uses_file_and_skills_thread_ids(monkeypatch):
|
||||
monkeypatch.setenv("PROVISIONER_BACKEND", "memory")
|
||||
module = _load_module()
|
||||
pod = SimpleNamespace(
|
||||
spec=SimpleNamespace(
|
||||
containers=[
|
||||
SimpleNamespace(
|
||||
name="sandbox",
|
||||
volume_mounts=[
|
||||
SimpleNamespace(
|
||||
mount_path="/home/gem/user-data/workspace",
|
||||
sub_path="threads/shared/user-1/workspace",
|
||||
),
|
||||
SimpleNamespace(
|
||||
mount_path="/home/gem/user-data/uploads",
|
||||
sub_path="threads/parent-thread/user-data/uploads",
|
||||
),
|
||||
SimpleNamespace(
|
||||
mount_path="/home/gem/user-data/outputs",
|
||||
sub_path="threads/parent-thread/user-data/outputs",
|
||||
),
|
||||
SimpleNamespace(mount_path="/home/gem/skills", sub_path="threads/child-skills-thread/skills"),
|
||||
],
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
assert module.KubernetesProvisionerBackend._pod_has_expected_mounts(
|
||||
pod,
|
||||
file_thread_id="parent-thread",
|
||||
skills_thread_id="child-skills-thread",
|
||||
uid="user-1",
|
||||
)
|
||||
assert not module.KubernetesProvisionerBackend._pod_has_expected_mounts(
|
||||
pod,
|
||||
file_thread_id="child-thread",
|
||||
skills_thread_id="child-skills-thread",
|
||||
uid="user-1",
|
||||
)
|
||||
@@ -0,0 +1,319 @@
|
||||
import pytest
|
||||
import numpy as np
|
||||
|
||||
# 现在可以安全地导入了,因为顶层不再有重型依赖
|
||||
from yuxi.knowledge.chunking.ragflow_like.parsers import semantic
|
||||
from yuxi.models import select_embedding_model
|
||||
|
||||
@pytest.fixture
|
||||
def embed_fn():
|
||||
"""使用真实的嵌入函数 (从 SiliconFlow 获取)"""
|
||||
model_id = "siliconflow/Qwen/Qwen3-Embedding-0.6B"
|
||||
try:
|
||||
model = select_embedding_model(model_id)
|
||||
def encode(sentences):
|
||||
if isinstance(sentences, str):
|
||||
sentences = [sentences]
|
||||
return model.encode(sentences)
|
||||
return encode
|
||||
except Exception as e:
|
||||
pytest.skip(f"无法初始化真实嵌入模型 {model_id}: {e}")
|
||||
|
||||
@pytest.fixture
|
||||
def sample_markdown():
|
||||
"""提供一个包含多章节、公式符号和复杂结构的临时 Markdown 样本"""
|
||||
return """
|
||||
## 痛点与挑战
|
||||
|
||||
在传统的RAG知识库前置处理过程中,我们面临着诸多挑战:
|
||||
|
||||
- **格式繁杂兼容难**:企业文档格式多样,传统工具(例如python-docx和pymupdf)难以同时高质量处理 Office 文档和复杂的 PDF 扫描件。
|
||||
- **语义割裂检索差**:简单的按字符切分导致上下文丢失,检索匹配度低,无法满足业务需求。
|
||||
- **解析效率瓶颈**:面对海量存量文档,缺乏高并发处理能力,响应速度慢,难以支撑大规模应用。
|
||||
|
||||
## 技术方案概览
|
||||
|
||||
为了解决上述问题,我们构建了一套高性能、可扩展的文档处理架构:
|
||||
|
||||
- **全格式支持**:全面覆盖 **Word、Excel、PDF、PPT和图片**等主流办公文档格式。
|
||||
- **高精度解析**:通过引入MinerU VLM模型实现准确的布局解析 and 元素提取。
|
||||
- **跨平台支持**:同时支持**英伟达CUDA和华为CANN框架。**
|
||||
- **知识库应用优化**:为了更好支持上层应用,解析后增加了智能切分和实体识别功能,增强后续的检索效果。
|
||||
- **高并发架构设计**:
|
||||
- 采用 **FastAPI** 作为服务入口,保障请求的高效接收与响应。
|
||||
- 引入 **Celery** 分布式任务队列,实现请求响应与处理过程的完全解耦,有效应对流量洪峰,保障系统稳定性和可用性。
|
||||
- **资源利用最大化**:通过创建多组 **Celery Worker**(执行CPU密集任务) + **vLLM Server**(执行GPU密集推理任务)的Pod单元,实现对多核CPU+多卡GPU的并行调用,大幅提升单机处理吞吐量。
|
||||
- 使用Docker Compose进行服务编排,不同功能划分为不同的进程,保证服务的并发量和性能。
|
||||
|
||||

|
||||
|
||||
## 架构设计
|
||||
|
||||
| 服务名称 | 容器名称 | 职责描述 |
|
||||
| --- | --- | --- |
|
||||
| **mineru-api** | `mineru-api` | **API 网关:**基于 FastAPI,提供对外 HTTP 接口 。负责接收请求、鉴权及任务分发。 |
|
||||
| **mineru-worker** | `mineru-worker` | **异步任务处理器:**基于 Celery,负责 PDF 解析、OCR、文档转换等耗时任务。 |
|
||||
| **vllm** | `mineru-vllm` | **推理后端:**运行 MinerU 视觉大模型 (VLM),提供高并发的文档理解与提取能力。 |
|
||||
| **redis** | `mineru-redis` | **消息中间件:**作为 Celery 的 Broker 和 Backend,同时缓存部分运行时数据。 |
|
||||
|
||||
## 核心技术亮点
|
||||
|
||||
### 1. 多模态融合的高精度解析
|
||||
|
||||
针对不同类型的文档,我们采用了差异化的解析策略,以确保最佳效果:
|
||||
* **Office 文档**:利用 **Docling** ,快速、精准地从文件中提取文本与格式信息。
|
||||
* **复杂 PDF 文档**:引入 **MinerU 视觉大模型(VLM)**,该模型具备类似人类的视觉能力,能够深度理解文档排版,完美还原文档结构和准确提取页面上的公式和表格元素,并统一输出为标准的 Markdown 格式。
|
||||
|
||||

|
||||
|
||||
原始文件
|
||||
|
||||

|
||||
|
||||
解析后效果
|
||||
|
||||
### 2. 智能表格处理
|
||||
|
||||
针对文档中的表格数据,我们提供了两种灵活的转换方式,满足不同应用场景的需求:
|
||||
* **键值对转换**:将表格行列关系转换为自然语义表达,便于大模型理解和问答。并且**支持合并单元格的处理**,避免生成关系错乱的表格数据。例如,将”产品名称 | 价格 | 库存”的表格转换为”产品名称:A;价格:100元,库存:50件;产品名称:B;价格:150元,库存:30件”等键值对形式。
|
||||
* **Markdown表格转换**:保持表格的原始结构,转换为标准的Markdown表格格式,便于上层应用中可视化展示。
|
||||
|
||||

|
||||
|
||||
原始表格
|
||||
|
||||

|
||||
|
||||
markdown格式表格
|
||||
|
||||

|
||||
|
||||
key-value格式表格
|
||||
|
||||
此外,**针对过长的表格,支持按照长度切分,在不丢失信息的前提下**适配向量数据库的存储,保证后续的检索效果。
|
||||
|
||||

|
||||
|
||||
原始表格
|
||||
|
||||

|
||||
|
||||
表格切分后
|
||||
|
||||
### 3. 基于语义的智能切分
|
||||
|
||||
摒弃了传统机械的字数切分方式,我们引入了 **BGE Embedding 模型**。通过计算文本向量并利用相似度聚类算法,将含义紧密相关的段落自动聚合。这种方式确保了每一个数据切片都是一个完整的”语义单元”,大幅提升了后续检索的准确性。与此同时,模型规模为24M,资源占用率极小,对整体解析流程的影响微乎其微。
|
||||
|
||||

|
||||
|
||||
语义切分前1
|
||||
|
||||

|
||||
|
||||
语义切分后2
|
||||
|
||||

|
||||
|
||||
语义切分前2
|
||||
|
||||

|
||||
|
||||
语义切分后2
|
||||
|
||||
### 4. 上下文感知的标题聚合
|
||||
|
||||
针对企业长文档层级复杂的特点,我们开发了**标题聚合功能**。系统会自动将多级标题信息聚合到对应的段落中。无论文档如何切分,每个切片都能保留”父级标题-子标题”的完整路径,有效解决了碎片化导致的上下文丢失问题,从而提升了后续检索的准确性和相关性。
|
||||
|
||||

|
||||
|
||||
多级标题聚合
|
||||
|
||||
### 5. 关键信息自动提取
|
||||
|
||||
为了进一步提升检索效率,我们集成了中英文的 **NER(命名实体识别)模型**。在解析过程中,系统会自动抽取文档中的组织机构、人名、专有名词等关键实体,为文档打上”智能标签”,实现多维度的精准检索。
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
### 6. 内容检索
|
||||
|
||||

|
||||
|
||||
原始PDF页面
|
||||
|
||||

|
||||
|
||||
搜索关键字“叶片描述”
|
||||
|
||||
返回结果如下:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "success",
|
||||
"message": "搜索成功",
|
||||
"data": {
|
||||
"result": [
|
||||
{
|
||||
"page_idx": 14,
|
||||
"span_range": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"bbox": [
|
||||
70,
|
||||
586,
|
||||
525,
|
||||
614
|
||||
]
|
||||
},
|
||||
{
|
||||
"page_idx": 14,
|
||||
"span_range": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"bbox": [
|
||||
69,
|
||||
625,
|
||||
147,
|
||||
638
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
应用展示
|
||||
|
||||

|
||||
|
||||
### 7. 性能优化
|
||||
|
||||
通过将CPU密集型任务和GPU密集型任务切分为相互独立的进程,更好利用了高性能服务器的硬件资源,不仅可以进行多路解析,在单任务的处理上也有显著提升。在附加了实体识别和智能分段等附加功能,处理时间和使用MinerU CLI所需的时间几乎没有差别。在多路服务器上可以通过简单配置,使不同的任务使用服务器上独立硬件资源,成倍提速。
|
||||
|
||||
| 解析样例 | 原生MinerU | 本方案 |
|
||||
| --- | --- | --- |
|
||||
| PDF测试1(158页英文内容) | 79.26s | 79.661 |
|
||||
| PDF测试2(731页英文内容) | 402s | 393s |
|
||||
| PDF测试2(670页中文内容) | 163 | 116.53 |
|
||||
|
||||
测试环境:
|
||||
|
||||
- GPU: NVIDIA A100 80GB
|
||||
- CPU: Intel(R) Xeon(R) Gold 6348 CPU @ 2.60GHz
|
||||
|
||||
## 核心组件
|
||||
|
||||
| 依赖名称 | 版本范围 | 功能描述 |
|
||||
| --- | --- | --- |
|
||||
| **fastapi** | `>=0.115.12` | **Web 框架**:提供高性能的 API 服务接口,支持异步编程。 |
|
||||
| **mineru** | `>=2.5.0` | **核心引擎**:MinerU 核心库,提供 PDF 解析、布局分析和公式提取能力。 |
|
||||
| **celery** | `>=5.5.3` | **异步任务队列**:用于处理耗时的 PDF 解析和 NLP任务。 |
|
||||
| **vllm** | `>=0.11.0` | **大模型推理**:高性能 LLM/VLM 推理引擎,用于加速VLM视觉大模型的推理。 |
|
||||
| **transformers** | `>=4.53.0` | **NLP 模型库**:用于加载和运行各种预训练模型(如 NER、Embedding)。 |
|
||||
| **minio** | `>=7.2.15` | **对象存储客户端**:用于上传和下载 PDF 源文件及解析后的结果文件。 |
|
||||
| **sentence-transformers** | `>=5.1.0` | **文本向量化**:用于生成文本 Embedding,实现基于文本语义的切分。 |
|
||||
| **sqlalchemy** | `>=2.0.41` | **ORM 框架**:用于数据库操作 and 连接管理。 |
|
||||
| **redis** | (Implied) | **缓存与消息中间件**:作为 Celery 的 Broker 和应用缓存(通过 `redislite` 或外部服务)。 |
|
||||
|
||||
## 应用价值
|
||||
|
||||
- **提质**:将非结构化文档转化为高质量的 Markdown 数据,为大模型知识库建设提供了坚实的基础。
|
||||
- **增效**:通过自动化、并行化的处理流程,结合软件架构的优化,将文档处理效率提升数倍,实现了从“小时级”到“分钟级”的跨越。
|
||||
- **赋能**:该平台可广泛应用于政策文件检索、合同智能审核、技术文档问答等多种业务场景,切实助力企业提效减负。
|
||||
|
||||
| 对比维度 | 原生 MinerU | 本方案 | 优势 |
|
||||
| --- | --- | --- | --- |
|
||||
| **系统架构** | 单体应用,命令行工具 | 分布式微服务架构(FastAPI + Celery + vLLM + Redis) | 支持高并发、可水平扩展,适合企业级部署 |
|
||||
| **服务化能力** | 无服务接口,需本地调用 | 提供 RESTful API,支持 HTTP 调用 | 易于集成到现有系统,支持多语言调用 |
|
||||
| **并发处理** | 单任务串行处理 | 分布式任务队列,支持多 Worker 并行 | 处理效率提升数倍,支持海量文档批量处理 |
|
||||
| **资源利用** | CPU/GPU 资源利用率低 | CPU 密集任务与 GPU 推理任务分离,多卡并行 | 单机吞吐量大幅提升,资源利用最大化 |
|
||||
| **文档格式支持** | 主要支持 PDF | 全格式支持(Word、Excel、PDF、PPT、图片) | 一站式解决企业多格式文档处理需求 |
|
||||
| **表格处理** | 基础表格识别 | 智能表格处理(键值对转换 + Markdown 表格) | 满足问答 and 可视化展示双重场景需求 |
|
||||
| **文本切分** | 无切分功能 | 基于语义的智能切分(BGE Embedding + 聚类) | 保证语义完整性,提升检索准确性 |
|
||||
| **上下文管理** | 无标题聚合功能 | 上下文感知的标题聚合(多级标题路径保留) | 解决碎片化导致的上下文丢失问题 |
|
||||
| **信息提取** | 无实体识别功能 | 集成 NER 模型,自动提取关键实体 | 支持多维度精准检索,为文档打智能标签 |
|
||||
|
||||
"""
|
||||
def test_semantic_chunking_basic(embed_fn, sample_markdown):
|
||||
"""测试基本的语义切分逻辑 (使用真实嵌入模型)"""
|
||||
|
||||
# 配置切分参数
|
||||
parser_config = {
|
||||
"chunk_token_num": 1000, # 针对标准文档调整 token 数
|
||||
"overlapped_percent": 0.1
|
||||
}
|
||||
|
||||
# 执行语义切分
|
||||
chunks = semantic.chunk_markdown(
|
||||
sample_markdown,
|
||||
parser_config=parser_config,
|
||||
embed_fn=embed_fn
|
||||
)
|
||||
|
||||
# 1. 基础验证
|
||||
assert isinstance(chunks, list)
|
||||
assert len(chunks) >= 5, f"预期至少切分为 5 个片段,实际仅有 {len(chunks)} 个"
|
||||
|
||||
# 2. 验证上下文感知与标题聚合 (检查 RAGFlow 风格的层级路径)
|
||||
# 验证子章节片段是否保留了父级标题路径
|
||||
# 检查 "Docling" 相关的片段 (属于 "核心技术亮点" -> "多模态融合的高精度解析")
|
||||
docling_chunks = [c for c in chunks if "Docling" in c]
|
||||
assert docling_chunks, "未找到包含 'Docling' 的片段"
|
||||
for chunk in docling_chunks:
|
||||
assert "核心技术亮点" in chunk, "子章节片段丢失了父级标题 '核心技术亮点'"
|
||||
assert "多模态融合的高精度解析" in chunk, "子章节片段丢失了自身标题 '多模态融合的高精度解析'"
|
||||
# 验证层级分隔符
|
||||
path_part = chunk.split("\n")[0] if "\n" in chunk else chunk
|
||||
assert "|" in path_part, f"标题路径中缺失层级分隔符 '|': {path_part}"
|
||||
assert path_part.count("|") >= 1, f"标题路径层级深度不足: {path_part}"
|
||||
|
||||
# 检查顶级章节 (Level 2) 是否保持独立 (不应包含其他不相关的顶级标题)
|
||||
pain_point_chunks = [c for c in chunks if "格式繁杂兼容难" in c]
|
||||
for chunk in pain_point_chunks:
|
||||
path_part = chunk.split("\n")[0]
|
||||
assert "痛点与挑战" in path_part
|
||||
assert "技术方案概览" not in path_part, "顶级标题路径污染:包含了不相关的顶级标题"
|
||||
|
||||
# 3. 验证关键技术词汇识别
|
||||
# 重点检查文档中出现的核心组件词汇
|
||||
keywords_to_check = ["FastAPI", "Celery", "vLLM", "MinerU"]
|
||||
found_keywords = [s for s in keywords_to_check if any(s in chunk for chunk in chunks)]
|
||||
assert len(found_keywords) > 0, f"在切分结果中未找到关键技术词汇: {keywords_to_check}"
|
||||
|
||||
# 4. 验证核心章节内容是否存在
|
||||
has_arch_section = any("架构设计" in chunk for chunk in chunks)
|
||||
has_highlight_section = any("核心技术亮点" in chunk for chunk in chunks)
|
||||
assert has_arch_section, "未找到 '架构设计' 相关内容"
|
||||
assert has_highlight_section, "未找到 '核心技术亮点' 相关内容"
|
||||
|
||||
# 5. 验证语义聚类是否将不同主题分开
|
||||
# 架构设计和核心技术亮点是两个独立的大章节,语义聚类应该尽量避免将它们的核心内容混在一个 chunk 中
|
||||
arch_start_chunks = [i for i, c in enumerate(chunks) if "## 架构设计" in c]
|
||||
highlight_start_chunks = [i for i, c in enumerate(chunks) if "## 核心技术亮点" in c]
|
||||
|
||||
if arch_start_chunks and highlight_start_chunks:
|
||||
# 确保起始片段不重合
|
||||
assert not set(arch_start_chunks).intersection(set(highlight_start_chunks)), \
|
||||
"语义聚类错误:架构设计与核心技术亮点的章节头部被挤在了同一个 chunk 中"
|
||||
|
||||
print(f"\n[测试成功] 文档成功切分为 {len(chunks)} 个片段")
|
||||
print(f"识别到的关键技术词汇: {found_keywords}")
|
||||
|
||||
# 直接输出到控制台预览切分结果,避免文件副作用
|
||||
print("\n--- 语义切分结果开始 ---")
|
||||
for idx, chunk in enumerate(chunks, 1):
|
||||
print(f"\n[Chunk {idx}]\n{chunk}")
|
||||
print("\n--- 语义切分结果结束 ---")
|
||||
|
||||
def test_heading_inference():
|
||||
"""测试标题层级推断工具类"""
|
||||
from yuxi.knowledge.chunking.ragflow_like.utils.md_parser_utils import infer_heading_level
|
||||
|
||||
assert infer_heading_level("1. 简介") == 1
|
||||
assert infer_heading_level("1.1 详细设计") == 2
|
||||
assert infer_heading_level("1.2.3 核心逻辑") == 3
|
||||
assert infer_heading_level("一、 背景") == 1
|
||||
assert infer_heading_level("普通文本") == 1
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from yuxi.agents.backends import skills_backend
|
||||
|
||||
|
||||
def _prepare_skills_dir(root: Path) -> None:
|
||||
(root / "alpha").mkdir(parents=True, exist_ok=True)
|
||||
(root / "alpha" / "SKILL.md").write_text(
|
||||
"---\nname: alpha\ndescription: alpha\n---\n# alpha\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "beta").mkdir(parents=True, exist_ok=True)
|
||||
(root / "beta" / "SKILL.md").write_text(
|
||||
"---\nname: beta\ndescription: beta\n---\n# beta\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_selected_skills_backend_none_exposes_no_skills(tmp_path, monkeypatch):
|
||||
_prepare_skills_dir(tmp_path)
|
||||
monkeypatch.setattr(skills_backend, "get_skills_root_dir", lambda: tmp_path)
|
||||
|
||||
backend = skills_backend.SelectedSkillsReadonlyBackend(selected_slugs=None)
|
||||
|
||||
assert backend.ls("/").entries == []
|
||||
assert backend.read("/alpha/SKILL.md").error == "Access denied: file is outside selected skills."
|
||||
|
||||
|
||||
def test_selected_skills_backend_readonly_and_visible_only_selected(tmp_path, monkeypatch):
|
||||
_prepare_skills_dir(tmp_path)
|
||||
monkeypatch.setattr(skills_backend, "get_skills_root_dir", lambda: tmp_path)
|
||||
|
||||
backend = skills_backend.SelectedSkillsReadonlyBackend(selected_slugs=["alpha"])
|
||||
|
||||
root_result = backend.ls("/")
|
||||
assert root_result.error is None
|
||||
paths = sorted(entry.get("path") for entry in (root_result.entries or []))
|
||||
assert paths == ["/alpha/"]
|
||||
|
||||
ok_read = backend.read("/alpha/SKILL.md")
|
||||
assert ok_read.file_data is not None
|
||||
assert "alpha" in ok_read.file_data["content"]
|
||||
|
||||
denied_read = backend.read("/beta/SKILL.md")
|
||||
assert denied_read.error and "Access denied" in denied_read.error
|
||||
|
||||
write_result = backend.write("/alpha/new.md", "x")
|
||||
assert write_result.error and "read-only" in write_result.error
|
||||
|
||||
edit_result = backend.edit("/alpha/SKILL.md", "alpha", "changed")
|
||||
assert edit_result.error and "read-only" in edit_result.error
|
||||
|
||||
upload_result = backend.upload_files([("/alpha/a.txt", b"a")])
|
||||
assert len(upload_result) == 1
|
||||
assert upload_result[0].error == "permission_denied"
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from deepagents.backends import FilesystemBackend
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.backends.skills_backend import SelectedSkillsReadonlyBackend
|
||||
|
||||
|
||||
def test_skills_backend_read_outside_root_returns_error_message(monkeypatch) -> None:
|
||||
def _fake_read(self, file_path: str, offset: int = 0, limit: int = 2000):
|
||||
raise ValueError(
|
||||
"Path:/app/package/yuxi/agents/skills/buildin/reporter outside root directory: /app/saves/skills"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(FilesystemBackend, "read", _fake_read)
|
||||
|
||||
backend = SelectedSkillsReadonlyBackend(selected_slugs=["reporter"])
|
||||
with pytest.raises(ValueError, match="outside root directory"):
|
||||
backend.read("/reporter/SKILL.md")
|
||||
|
||||
|
||||
def test_skills_backend_ls_outside_root_raises(monkeypatch) -> None:
|
||||
def _fake_ls(self, path: str):
|
||||
raise ValueError("Path outside root directory")
|
||||
|
||||
monkeypatch.setattr(FilesystemBackend, "ls", _fake_ls)
|
||||
|
||||
backend = SelectedSkillsReadonlyBackend(selected_slugs=["reporter"])
|
||||
with pytest.raises(ValueError, match="outside root directory"):
|
||||
backend.ls("/reporter")
|
||||
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import tomli
|
||||
from yuxi.config.app import Config
|
||||
from yuxi.config.cache import RUNTIME_CONFIG_REDIS_KEY
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
config_cache = importlib.import_module("yuxi.config.cache")
|
||||
|
||||
|
||||
class _FakeRedis:
|
||||
def __init__(
|
||||
self, raw: str | None = None, *, get_error: Exception | None = None, set_error: Exception | None = None
|
||||
):
|
||||
self.raw = raw
|
||||
self.get_error = get_error
|
||||
self.set_error = set_error
|
||||
self.data: dict[str, str] = {}
|
||||
self.get_keys: list[str] = []
|
||||
|
||||
def get(self, key: str) -> str | None:
|
||||
self.get_keys.append(key)
|
||||
if self.get_error:
|
||||
raise self.get_error
|
||||
return self.raw
|
||||
|
||||
def set(self, key: str, value: str) -> bool:
|
||||
if self.set_error:
|
||||
raise self.set_error
|
||||
self.data[key] = value
|
||||
return True
|
||||
|
||||
|
||||
def _patch_runtime_redis(monkeypatch: pytest.MonkeyPatch, redis: _FakeRedis) -> None:
|
||||
@contextmanager
|
||||
def fake_sync_redis_client(*args, **kwargs):
|
||||
del args, kwargs
|
||||
yield redis
|
||||
|
||||
monkeypatch.setattr(config_cache, "sync_redis_client", fake_sync_redis_client)
|
||||
|
||||
|
||||
def test_save_writes_runtime_snapshot_after_base_toml(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
redis = _FakeRedis()
|
||||
_patch_runtime_redis(monkeypatch, redis)
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
|
||||
cfg.default_model = "test-provider:new-chat"
|
||||
cfg.enable_content_guard = True
|
||||
cfg.save()
|
||||
|
||||
base_config = tomli.loads((tmp_path / "config" / "base.toml").read_text())
|
||||
assert base_config["default_model"] == "test-provider:new-chat"
|
||||
|
||||
payload = json.loads(redis.data[RUNTIME_CONFIG_REDIS_KEY])
|
||||
assert payload["default_model"] == "test-provider:new-chat"
|
||||
assert payload["enable_content_guard"] is True
|
||||
assert payload["default_ocr_engine"] == "rapid_ocr"
|
||||
assert "save_dir" not in payload
|
||||
assert payload["sandbox_provider"] == "provisioner"
|
||||
assert "enable_reranker" not in payload
|
||||
assert "default_agent_id" not in payload
|
||||
# 快照只含公开配置字段,不夹带元数据
|
||||
assert "schema_version" not in payload
|
||||
assert "saved_at" not in payload
|
||||
assert all(not key.startswith("_") for key in payload)
|
||||
|
||||
|
||||
def test_unknown_config_fields_are_removed_on_save(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
_patch_runtime_redis(monkeypatch, _FakeRedis())
|
||||
config_dir = tmp_path / "config"
|
||||
config_dir.mkdir(parents=True)
|
||||
(config_dir / "base.toml").write_text(
|
||||
'default_model = "test-provider:file-chat"\nenable_reranker = true\ndefault_agent_id = "ChatbotAgent"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
|
||||
assert cfg.default_model == "test-provider:file-chat"
|
||||
assert not hasattr(cfg, "enable_reranker")
|
||||
assert not hasattr(cfg, "default_agent_id")
|
||||
|
||||
cfg.save()
|
||||
|
||||
base_config = tomli.loads((config_dir / "base.toml").read_text())
|
||||
assert base_config == {"default_model": "test-provider:file-chat"}
|
||||
|
||||
|
||||
def test_save_dir_from_base_toml_is_ignored(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
_patch_runtime_redis(monkeypatch, _FakeRedis())
|
||||
config_dir = tmp_path / "config"
|
||||
config_dir.mkdir(parents=True)
|
||||
(config_dir / "base.toml").write_text(
|
||||
f'save_dir = "{tmp_path / "from-file"}"\ndefault_model = "test-provider:file-chat"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
|
||||
assert cfg.save_dir == str(tmp_path)
|
||||
assert cfg.default_model == "test-provider:file-chat"
|
||||
|
||||
cfg.save()
|
||||
|
||||
base_config = tomli.loads((config_dir / "base.toml").read_text())
|
||||
assert base_config == {"default_model": "test-provider:file-chat"}
|
||||
|
||||
|
||||
def test_refresh_loads_public_config_from_redis(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
redis_save_dir = str(tmp_path / "redis-save")
|
||||
payload = {
|
||||
"default_model": "test-provider:worker-chat",
|
||||
"save_dir": redis_save_dir,
|
||||
"sandbox_virtual_path_prefix": "/redis/user-data",
|
||||
"default_ocr_engine": "mineru_ocr",
|
||||
"enable_reranker": True,
|
||||
}
|
||||
redis = _FakeRedis(raw=json.dumps(payload))
|
||||
_patch_runtime_redis(monkeypatch, redis)
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
cfg.default_model = "test-provider:old-chat"
|
||||
|
||||
cfg.refresh()
|
||||
|
||||
assert cfg.default_model == "test-provider:worker-chat"
|
||||
assert cfg.default_ocr_engine == "mineru_ocr"
|
||||
assert cfg.save_dir == str(tmp_path)
|
||||
assert cfg.sandbox_virtual_path_prefix == "/redis/user-data"
|
||||
assert str(cfg._config_file) == str(tmp_path / "config" / "base.toml")
|
||||
# 快照里的非运行时字段和未知键不会被写回
|
||||
assert not hasattr(cfg, "enable_reranker")
|
||||
assert redis.get_keys == [RUNTIME_CONFIG_REDIS_KEY]
|
||||
|
||||
|
||||
def test_refresh_keeps_memory_value_when_redis_unavailable(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
redis = _FakeRedis(get_error=RuntimeError("redis unavailable"))
|
||||
_patch_runtime_redis(monkeypatch, redis)
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
cfg.default_model = "test-provider:local-chat"
|
||||
|
||||
cfg.refresh()
|
||||
|
||||
assert cfg.default_model == "test-provider:local-chat"
|
||||
assert redis.get_keys == [RUNTIME_CONFIG_REDIS_KEY]
|
||||
|
||||
|
||||
def test_save_keeps_base_toml_when_runtime_snapshot_write_fails(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
redis = _FakeRedis(set_error=RuntimeError("redis unavailable"))
|
||||
_patch_runtime_redis(monkeypatch, redis)
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
cfg.default_model = "test-provider:file-chat"
|
||||
|
||||
cfg.save()
|
||||
|
||||
base_config = tomli.loads((tmp_path / "config" / "base.toml").read_text())
|
||||
assert base_config["default_model"] == "test-provider:file-chat"
|
||||
|
||||
|
||||
def test_start_runtime_sync_is_idempotent(tmp_path):
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
|
||||
cfg.start_runtime_sync(interval=3600)
|
||||
thread = cfg._runtime_sync_thread
|
||||
cfg.start_runtime_sync(interval=3600)
|
||||
|
||||
assert cfg._runtime_sync_thread is thread
|
||||
assert thread.daemon is True
|
||||
|
||||
|
||||
def test_update_ignores_readonly_save_dir(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
_patch_runtime_redis(monkeypatch, _FakeRedis())
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
|
||||
cfg.update(
|
||||
{
|
||||
"save_dir": str(tmp_path / "other-save"),
|
||||
"default_model": "test-provider:updated-chat",
|
||||
"default_ocr_engine": "mineru_ocr",
|
||||
}
|
||||
)
|
||||
|
||||
assert cfg.save_dir == str(tmp_path)
|
||||
assert cfg.default_model == "test-provider:updated-chat"
|
||||
assert cfg.default_ocr_engine == "mineru_ocr"
|
||||
|
||||
|
||||
def test_update_rejects_unknown_default_ocr_engine(tmp_path, monkeypatch: pytest.MonkeyPatch):
|
||||
_patch_runtime_redis(monkeypatch, _FakeRedis())
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
|
||||
with pytest.raises(ValueError, match="不支持的默认 OCR 引擎"):
|
||||
cfg.update({"default_ocr_engine": "not_an_ocr_engine"})
|
||||
|
||||
|
||||
def test_dump_config_hides_save_dir(tmp_path):
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
|
||||
dumped = cfg.dump_config()
|
||||
|
||||
assert "save_dir" not in dumped
|
||||
assert "save_dir" not in dumped["_config_items"]
|
||||
|
||||
|
||||
def test_resolve_chat_model_spec_reads_runtime_refreshed_default(tmp_path, monkeypatch):
|
||||
from yuxi.agents import models
|
||||
|
||||
payload = {"default_model": "test-provider:resolved-chat"}
|
||||
_patch_runtime_redis(monkeypatch, _FakeRedis(raw=json.dumps(payload)))
|
||||
cfg = Config(save_dir=str(tmp_path))
|
||||
cfg.default_model = "test-provider:old-chat"
|
||||
cfg.refresh()
|
||||
monkeypatch.setattr(models, "sys_config", cfg)
|
||||
|
||||
assert models.resolve_chat_model_spec(None) == "test-provider:resolved-chat"
|
||||
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from yuxi.config import UserConfig, UserConfigSchema
|
||||
from yuxi.storage.postgres.models_business import Base, Department, User
|
||||
from yuxi.storage.postgres.models_business import UserConfig as UserConfigRecord
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture()
|
||||
async def session():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
department = Department(name="Config Dept")
|
||||
user = User(
|
||||
username="Config User",
|
||||
uid="config_user",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="user",
|
||||
department=department,
|
||||
)
|
||||
db.add_all([department, user])
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
yield db, user
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_user_config_load_returns_defaults_without_creating_row(session):
|
||||
db, user = session
|
||||
|
||||
user_config = await UserConfig.load(db, user.uid)
|
||||
dumped = user_config.dump_config()
|
||||
|
||||
assert dumped["uid"] == user.uid
|
||||
assert dumped["enable_memory"] is False
|
||||
|
||||
result = await db.execute(select(UserConfigRecord).filter(UserConfigRecord.uid == user.uid))
|
||||
assert result.scalar_one_or_none() is None
|
||||
|
||||
|
||||
async def test_user_config_save_persists_user_specific_values(session):
|
||||
db, user = session
|
||||
|
||||
saved = await UserConfig(uid=user.uid, schema=UserConfigSchema(enable_memory=True)).save(db)
|
||||
loaded = await UserConfig.load(db, user.uid)
|
||||
|
||||
assert saved.dump_config()["enable_memory"] is True
|
||||
assert loaded.dump_config()["enable_memory"] is True
|
||||
result = await db.execute(select(UserConfigRecord).filter(UserConfigRecord.uid == user.uid))
|
||||
assert result.scalar_one().enable_memory is True
|
||||
@@ -0,0 +1,328 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.knowledge.graphs.extractors import (
|
||||
GraphExtractorFactory,
|
||||
LLMGraphExtractor,
|
||||
normalize_extraction_result,
|
||||
)
|
||||
from yuxi.knowledge.graphs.milvus_graph_service import MilvusGraphService
|
||||
|
||||
|
||||
def _raw_graph_node(node_id: str, *, labels: list[str] | None = None, name: str | None = None) -> dict:
|
||||
return {
|
||||
"id": node_id,
|
||||
"labels": labels or ["MilvusKB", "Entity"],
|
||||
"properties": {"name": name or node_id, "kb_id": "kb_test"},
|
||||
}
|
||||
|
||||
|
||||
def _raw_graph_edge(edge_id: str, source_id: str, target_id: str) -> dict:
|
||||
return {
|
||||
"id": edge_id,
|
||||
"type": "RELATED_TO",
|
||||
"source_id": source_id,
|
||||
"target_id": target_id,
|
||||
"properties": {},
|
||||
}
|
||||
|
||||
|
||||
def test_normalize_extraction_result_defaults_and_validates_refs():
|
||||
result = normalize_extraction_result(
|
||||
{
|
||||
"entities": [{"text": "张三"}, {"text": "公司"}],
|
||||
"relations": [{"source": "张三", "target": "公司", "text": "任职于"}],
|
||||
},
|
||||
"llm",
|
||||
)
|
||||
|
||||
assert result["entities"][0]["label"] == "Entity"
|
||||
assert result["relations"][0]["label"] == "RELATED_TO"
|
||||
assert result["relations"][0]["source"] == {"text": "张三", "label": "Entity", "attributes": []}
|
||||
assert result["metadata"] == {"extractor_type": "llm", "schema_version": 1}
|
||||
|
||||
|
||||
def test_normalize_extraction_result_accepts_llm_nested_relation_entities():
|
||||
result = normalize_extraction_result(
|
||||
{
|
||||
"relations": [
|
||||
{
|
||||
"source": {
|
||||
"text": "张三",
|
||||
"label": "Person",
|
||||
"attributes": [{"text": "工程师", "label": "Occupation"}],
|
||||
},
|
||||
"target": {"text": "公司", "label": "Organization"},
|
||||
"text": "任职于",
|
||||
"label": "WORKS_AT",
|
||||
}
|
||||
]
|
||||
},
|
||||
"llm",
|
||||
)
|
||||
|
||||
assert result["entities"] == [
|
||||
{"text": "张三", "label": "Person", "attributes": [{"text": "工程师", "label": "Occupation"}]},
|
||||
{"text": "公司", "label": "Organization", "attributes": []},
|
||||
]
|
||||
assert result["relations"][0]["source"]["attributes"] == [{"text": "工程师", "label": "Occupation"}]
|
||||
assert result["relations"][0]["target"] == {"text": "公司", "label": "Organization", "attributes": []}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
{"entities": [{"text": "张三"}], "relations": [{"source": "张三", "target": "不存在", "text": "关系"}]},
|
||||
{"entities": [{"text": ""}], "relations": []},
|
||||
],
|
||||
)
|
||||
def test_normalize_extraction_result_rejects_invalid_payload(payload):
|
||||
with pytest.raises(ValueError):
|
||||
normalize_extraction_result(payload, "llm")
|
||||
|
||||
|
||||
def test_llm_graph_extractor_rejects_custom_prompt():
|
||||
extractor = LLMGraphExtractor({"model_spec": "test/model", "prompt": "custom"})
|
||||
|
||||
with pytest.raises(ValueError, match="不支持自定义完整 Prompt"):
|
||||
extractor.validate_options()
|
||||
|
||||
|
||||
def test_llm_graph_extractor_appends_schema_to_fixed_prompt():
|
||||
extractor = LLMGraphExtractor(
|
||||
{
|
||||
"model_spec": "test/model",
|
||||
"schema": "实体类型只能是 Person 或 Organization",
|
||||
"concurrency_count": 5,
|
||||
"model_params": {"temperature": 0.1},
|
||||
}
|
||||
)
|
||||
|
||||
prompt = extractor._build_prompt("张三任职于公司")
|
||||
|
||||
assert "请从下面文本中抽取实体和实体关系" in prompt
|
||||
assert "抽取 Schema 约束" in prompt
|
||||
assert "实体类型只能是 Person 或 Organization" in prompt
|
||||
assert "文本:\n张三任职于公司" in prompt
|
||||
|
||||
|
||||
def test_graph_extractor_factory_supports_only_llm():
|
||||
assert GraphExtractorFactory.supported_types() == ["llm"]
|
||||
|
||||
|
||||
def test_graph_extractor_factory_rejects_spacy():
|
||||
with pytest.raises(ValueError, match="spacy"):
|
||||
GraphExtractorFactory.create("spacy", {"model": "zh_core_web_sm"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_milvus_graph_service_configure_rejects_spacy():
|
||||
kb = SimpleNamespace(kb_type="milvus", additional_params={})
|
||||
|
||||
class Repo:
|
||||
async def get_by_kb_id(self, kb_id):
|
||||
return kb
|
||||
|
||||
async def update(self, kb_id, data):
|
||||
raise AssertionError("unsupported extractor should not be persisted")
|
||||
|
||||
service = MilvusGraphService(kb_repo=Repo())
|
||||
|
||||
with pytest.raises(ValueError, match="不支持的图谱抽取器类型"):
|
||||
await service.configure(
|
||||
"kb_test",
|
||||
extractor_type="spacy",
|
||||
extractor_options={"model": "zh_core_web_sm"},
|
||||
created_by="user_1",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_milvus_graph_service_configure_persists_updated_concurrency():
|
||||
kb = SimpleNamespace(
|
||||
kb_type="milvus",
|
||||
additional_params={
|
||||
"graph_build_config": {
|
||||
"locked": True,
|
||||
"extractor_type": "llm",
|
||||
"extractor_options": {"model_spec": "test/model", "concurrency_count": 5},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
class Repo:
|
||||
async def get_by_kb_id(self, kb_id):
|
||||
return kb
|
||||
|
||||
async def update(self, kb_id, data):
|
||||
kb.additional_params = data["additional_params"]
|
||||
return kb
|
||||
|
||||
chunk_repo = SimpleNamespace(
|
||||
count_by_kb_id=AsyncMock(return_value=0),
|
||||
count_graph_pending_by_kb_id=AsyncMock(return_value=0),
|
||||
count_graph_indexed_by_kb_id=AsyncMock(return_value=0),
|
||||
)
|
||||
graph_repo = SimpleNamespace(count_by_kb_id=AsyncMock(return_value=(3, 2)))
|
||||
service = MilvusGraphService(kb_repo=Repo(), chunk_repo=chunk_repo, graph_repo=graph_repo)
|
||||
|
||||
await service.configure(
|
||||
"kb_test",
|
||||
extractor_type="llm",
|
||||
extractor_options={"model_spec": "test/model", "concurrency_count": 9},
|
||||
created_by="user_1",
|
||||
)
|
||||
status = await service.get_status("kb_test")
|
||||
|
||||
assert status["config"]["extractor_options"]["concurrency_count"] == 9
|
||||
assert status["entity_count"] == 3
|
||||
assert status["relationship_count"] == 2
|
||||
|
||||
|
||||
def test_milvus_graph_service_writes_chunk_entity_and_relation():
|
||||
tx = MagicMock()
|
||||
session = MagicMock()
|
||||
session.__enter__.return_value = session
|
||||
session.execute_write.side_effect = lambda func: func(tx)
|
||||
driver = MagicMock()
|
||||
driver.session.return_value = session
|
||||
connection = SimpleNamespace(driver=driver)
|
||||
service = MilvusGraphService(neo4j_connection=connection)
|
||||
chunk = SimpleNamespace(
|
||||
chunk_id="chunk_1",
|
||||
file_id="file_1",
|
||||
kb_id="kb_test",
|
||||
chunk_index=1,
|
||||
content="张三任职于公司",
|
||||
start_char_pos=0,
|
||||
end_char_pos=8,
|
||||
)
|
||||
|
||||
entities, triples = service.write_chunk_graph(
|
||||
"kb_test",
|
||||
chunk,
|
||||
normalize_extraction_result(
|
||||
{
|
||||
"relations": [
|
||||
{
|
||||
"source": {
|
||||
"text": "张三",
|
||||
"label": "Person",
|
||||
"attributes": [{"text": "工程师", "label": "Occupation"}],
|
||||
},
|
||||
"target": {"text": "公司", "label": "Organization"},
|
||||
"text": "任职于",
|
||||
"label": "WORKS_AT",
|
||||
}
|
||||
],
|
||||
},
|
||||
"llm",
|
||||
),
|
||||
)
|
||||
|
||||
assert [entity["name"] for entity in entities] == ["张三", "公司"]
|
||||
assert {entity["label"] for entity in entities} == {"Person", "Organization"}
|
||||
assert triples[0]["relation_type"] == "WORKS_AT"
|
||||
queries = [call.args[0] for call in tx.run.call_args_list]
|
||||
assert any("MERGE (c:Chunk:MilvusKB:`kb_test`" in query for query in queries)
|
||||
assert any("MERGE (e:Entity:MilvusKB:`kb_test`" in query for query in queries)
|
||||
assert any("MERGE (source)-[r:RELATION" in query for query in queries)
|
||||
entity_call = next(call for call in tx.run.call_args_list if "MERGE (e:Entity" in call.args[0])
|
||||
assert entity_call.kwargs["attributes"] == '[{"text": "工程师", "label": "Occupation"}]'
|
||||
|
||||
|
||||
def test_milvus_graph_service_process_query_result_keeps_complete_edges():
|
||||
service = MilvusGraphService()
|
||||
result = service._process_query_result(
|
||||
[
|
||||
{
|
||||
"h": _raw_graph_node("node-a"),
|
||||
"t": _raw_graph_node("node-b"),
|
||||
"r": _raw_graph_edge("edge-a-b", "node-a", "node-b"),
|
||||
}
|
||||
],
|
||||
limit=2,
|
||||
kb_id="kb_test",
|
||||
)
|
||||
|
||||
assert [node["id"] for node in result["nodes"]] == ["node-a", "node-b"]
|
||||
assert [edge["id"] for edge in result["edges"]] == ["edge-a-b"]
|
||||
|
||||
|
||||
def test_milvus_graph_service_process_query_result_filters_edges_after_node_limit():
|
||||
service = MilvusGraphService()
|
||||
result = service._process_query_result(
|
||||
[
|
||||
{
|
||||
"h": _raw_graph_node("node-a"),
|
||||
"t": _raw_graph_node("node-b"),
|
||||
"r": _raw_graph_edge("edge-a-b", "node-a", "node-b"),
|
||||
}
|
||||
],
|
||||
limit=1,
|
||||
kb_id="kb_test",
|
||||
)
|
||||
|
||||
assert [node["id"] for node in result["nodes"]] == ["node-a"]
|
||||
assert result["edges"] == []
|
||||
|
||||
|
||||
def test_milvus_graph_service_process_query_result_filters_edges_to_excluded_chunk_nodes():
|
||||
service = MilvusGraphService()
|
||||
result = service._process_query_result(
|
||||
[
|
||||
{
|
||||
"h": _raw_graph_node("entity-a"),
|
||||
"t": _raw_graph_node("chunk-a", labels=["MilvusKB", "Chunk"]),
|
||||
"r": _raw_graph_edge("edge-entity-chunk", "entity-a", "chunk-a"),
|
||||
}
|
||||
],
|
||||
limit=2,
|
||||
kb_id="kb_test",
|
||||
exclude_chunk=True,
|
||||
)
|
||||
|
||||
assert [node["id"] for node in result["nodes"]] == ["entity-a"]
|
||||
assert result["edges"] == []
|
||||
|
||||
|
||||
def test_milvus_graph_service_process_query_result_clamps_negative_limit():
|
||||
service = MilvusGraphService()
|
||||
result = service._process_query_result(
|
||||
[
|
||||
{
|
||||
"h": _raw_graph_node("node-a"),
|
||||
"t": _raw_graph_node("node-b"),
|
||||
"r": _raw_graph_edge("edge-a-b", "node-a", "node-b"),
|
||||
}
|
||||
],
|
||||
limit=-1,
|
||||
kb_id="kb_test",
|
||||
)
|
||||
|
||||
assert result == {"nodes": [], "edges": []}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_milvus_graph_service_query_nodes_empty_kb_id():
|
||||
service = MilvusGraphService()
|
||||
result = await service.query_nodes(kb_id=None, keyword="test")
|
||||
assert result == {"nodes": [], "edges": []}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_milvus_graph_service_get_labels_empty_kb_id():
|
||||
service = MilvusGraphService()
|
||||
result = await service.get_labels(kb_id=None)
|
||||
assert result == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_milvus_graph_service_get_stats_empty_kb_id():
|
||||
service = MilvusGraphService()
|
||||
result = await service.get_stats(kb_id=None)
|
||||
assert result == {"total_nodes": 0, "total_edges": 0, "entity_types": []}
|
||||
@@ -0,0 +1,413 @@
|
||||
import asyncio
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("OPENAI_API_KEY", "test-key")
|
||||
|
||||
from yuxi.knowledge.eval import benchmark_generation
|
||||
from yuxi.knowledge.eval.benchmark_generation import (
|
||||
build_benchmark_generation_prompt,
|
||||
clamp_neighbors_count,
|
||||
collect_kb_chunks,
|
||||
iter_generated_benchmark_items,
|
||||
normalize_generation_concurrency_count,
|
||||
select_graph_enhanced_chunks,
|
||||
select_neighbor_chunks_by_kb_query,
|
||||
)
|
||||
|
||||
|
||||
class FakeKnowledgeBase:
|
||||
pass
|
||||
|
||||
|
||||
class FakeGenerationKnowledgeBase:
|
||||
def __init__(self, query_results=None):
|
||||
self.query_results = query_results or []
|
||||
self.query_calls = []
|
||||
|
||||
async def aquery(self, query_text, kb_id, **kwargs):
|
||||
self.query_calls.append({"query_text": query_text, "kb_id": kb_id, **kwargs})
|
||||
return self.query_results
|
||||
|
||||
|
||||
class FakeLlm:
|
||||
def __init__(self, gold_chunk_id="anchor_chunk"):
|
||||
self.gold_chunk_id = gold_chunk_id
|
||||
self.prompts = []
|
||||
|
||||
async def call(self, prompt, stream):
|
||||
self.prompts.append(prompt)
|
||||
return SimpleNamespace(
|
||||
content=('{"query":"问题","gold_answer":"答案","gold_chunk_ids":["' + self.gold_chunk_id + '"]}')
|
||||
)
|
||||
|
||||
|
||||
class NoQueryKnowledgeBase(FakeGenerationKnowledgeBase):
|
||||
async def aquery(self, query_text, kb_id, **kwargs):
|
||||
raise AssertionError("neighbors_count=1 时不应调用 aquery")
|
||||
|
||||
|
||||
class TrackingLlm:
|
||||
def __init__(self, content=None, delay=0):
|
||||
self.content = content or '{"query":"问题","gold_answer":"答案","gold_chunk_ids":["anchor_chunk"]}'
|
||||
self.delay = delay
|
||||
self.active_calls = 0
|
||||
self.max_active_calls = 0
|
||||
self.calls = 0
|
||||
|
||||
async def call(self, prompt, stream):
|
||||
self.calls += 1
|
||||
self.active_calls += 1
|
||||
self.max_active_calls = max(self.max_active_calls, self.active_calls)
|
||||
try:
|
||||
if self.delay:
|
||||
await asyncio.sleep(self.delay)
|
||||
return SimpleNamespace(content=self.content)
|
||||
finally:
|
||||
self.active_calls -= 1
|
||||
|
||||
|
||||
class FakeGraphGenerationKnowledgeBase(FakeGenerationKnowledgeBase):
|
||||
pass
|
||||
|
||||
|
||||
def make_chunk(
|
||||
chunk_id: str,
|
||||
*,
|
||||
kb_id: str = "db_1",
|
||||
file_id: str = "file_a",
|
||||
content: str = "anchor content",
|
||||
chunk_index: int = 0,
|
||||
graph_indexed: bool = False,
|
||||
ent_ids: list[str] | None = None,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
chunk_id=chunk_id,
|
||||
kb_id=kb_id,
|
||||
file_id=file_id,
|
||||
content=content,
|
||||
chunk_index=chunk_index,
|
||||
graph_indexed=graph_indexed,
|
||||
ent_ids=ent_ids,
|
||||
tags=None,
|
||||
extraction_result=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fake_chunk_repository(monkeypatch):
|
||||
class FakeChunkRepository:
|
||||
chunks = [make_chunk("anchor_chunk")]
|
||||
|
||||
async def list_by_kb_id(self, kb_id):
|
||||
return [chunk for chunk in self.chunks if chunk.kb_id == kb_id]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_chunk_repository.KnowledgeChunkRepository",
|
||||
FakeChunkRepository,
|
||||
)
|
||||
return FakeChunkRepository
|
||||
|
||||
|
||||
def test_clamp_neighbors_count():
|
||||
assert clamp_neighbors_count(-1) == 0
|
||||
assert clamp_neighbors_count(3) == 3
|
||||
assert clamp_neighbors_count(11) == 10
|
||||
|
||||
|
||||
def test_normalize_generation_concurrency_count():
|
||||
assert normalize_generation_concurrency_count(None) == 10
|
||||
assert normalize_generation_concurrency_count("") == 10
|
||||
assert normalize_generation_concurrency_count(0) == 1
|
||||
assert normalize_generation_concurrency_count(-5) == 1
|
||||
assert normalize_generation_concurrency_count(10000) == 20
|
||||
|
||||
|
||||
def test_build_benchmark_generation_prompt_contains_required_schema():
|
||||
prompt = build_benchmark_generation_prompt([("chunk_1", "片段内容")])
|
||||
|
||||
assert "片段ID=chunk_1" in prompt
|
||||
assert "query、gold_answer、gold_chunk_ids" in prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_kb_chunks_filters_kb_id(fake_chunk_repository):
|
||||
fake_chunk_repository.chunks = [
|
||||
make_chunk("file_a_chunk", content="内容"),
|
||||
make_chunk("file_b_chunk", kb_id="db_2", file_id="file_b", content="其他"),
|
||||
]
|
||||
|
||||
chunks = await collect_kb_chunks(FakeKnowledgeBase(), "db_1")
|
||||
|
||||
assert chunks == [
|
||||
{
|
||||
"id": "file_a_chunk",
|
||||
"content": "内容",
|
||||
"file_id": "file_a",
|
||||
"chunk_index": 0,
|
||||
"graph_indexed": False,
|
||||
"ent_ids": [],
|
||||
"tags": [],
|
||||
"extraction_result": None,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_with_one_chunk_does_not_query(monkeypatch):
|
||||
fake_llm = FakeLlm()
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=NoQueryKnowledgeBase(),
|
||||
kb_id="db_1",
|
||||
count=1,
|
||||
neighbors_count=1,
|
||||
llm_model_spec="test-provider:test-model",
|
||||
)
|
||||
]
|
||||
|
||||
assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}]
|
||||
assert "片段ID=anchor_chunk" in fake_llm.prompts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_select_neighbor_chunks_by_kb_query_filters_anchor():
|
||||
kb = FakeGenerationKnowledgeBase(
|
||||
query_results=[
|
||||
{
|
||||
"content": "anchor content",
|
||||
"metadata": {"chunk_id": "anchor_chunk", "file_id": "file_a", "chunk_index": 0},
|
||||
},
|
||||
{
|
||||
"content": "neighbor content",
|
||||
"metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
chunks = await select_neighbor_chunks_by_kb_query(
|
||||
kb_instance=kb,
|
||||
kb_id="db_1",
|
||||
anchor_chunk={"id": "anchor_chunk", "content": "anchor content", "file_id": "file_a", "chunk_index": 0},
|
||||
neighbors_count=1,
|
||||
)
|
||||
|
||||
assert chunks == [{"id": "neighbor_chunk", "content": "neighbor content", "file_id": "file_a", "chunk_index": 1}]
|
||||
assert kb.query_calls == [
|
||||
{
|
||||
"query_text": "anchor content",
|
||||
"kb_id": "db_1",
|
||||
"search_mode": "vector",
|
||||
"final_top_k": 4,
|
||||
"use_reranker": False,
|
||||
"similarity_threshold": 0.0,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_select_graph_enhanced_chunks_expands_by_ppr_with_anchor_bias(monkeypatch):
|
||||
calls = []
|
||||
|
||||
async def fake_rank(self, kb_id, seed_weights, *, max_nodes, top_k, damping):
|
||||
calls.append(dict(seed_weights))
|
||||
if len(calls) == 1:
|
||||
return [("anchor", 0.9), ("neighbor_1", 0.8)]
|
||||
return [("anchor", 0.9), ("neighbor_1", 0.8), ("neighbor_2", 0.7)]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.knowledge.graphs.milvus_graph_service.MilvusGraphService.query_and_rank_chunks_by_ppr",
|
||||
fake_rank,
|
||||
)
|
||||
chunks_by_id = {
|
||||
"anchor": {"id": "anchor", "content": "anchor", "ent_ids": ["anchor_entity"]},
|
||||
"neighbor_1": {"id": "neighbor_1", "content": "neighbor 1", "ent_ids": ["entity_1"]},
|
||||
"neighbor_2": {"id": "neighbor_2", "content": "neighbor 2", "ent_ids": ["entity_2"]},
|
||||
}
|
||||
|
||||
chunks = await select_graph_enhanced_chunks(
|
||||
kb_id="db_1",
|
||||
anchor_chunk=chunks_by_id["anchor"],
|
||||
chunks_by_id=chunks_by_id,
|
||||
context_count=3,
|
||||
graph_expand_top_k=1,
|
||||
)
|
||||
|
||||
assert [chunk["id"] for chunk in chunks] == ["anchor", "neighbor_1", "neighbor_2"]
|
||||
assert calls[0] == {"anchor_entity": 1.0}
|
||||
assert calls[1]["anchor_entity"] == 1.0
|
||||
assert calls[1]["entity_1"] == 0.9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_graph_mode_uses_graph_indexed_anchor(monkeypatch, fake_chunk_repository):
|
||||
fake_chunk_repository.chunks = [
|
||||
make_chunk(
|
||||
"vector_anchor",
|
||||
content="vector content",
|
||||
chunk_index=0,
|
||||
graph_indexed=False,
|
||||
ent_ids=["vector_entity"],
|
||||
),
|
||||
make_chunk(
|
||||
"graph_anchor",
|
||||
content="graph anchor content",
|
||||
chunk_index=1,
|
||||
graph_indexed=True,
|
||||
ent_ids=["anchor_entity"],
|
||||
),
|
||||
make_chunk(
|
||||
"graph_neighbor",
|
||||
content="graph neighbor content",
|
||||
chunk_index=2,
|
||||
graph_indexed=False,
|
||||
ent_ids=["neighbor_entity"],
|
||||
),
|
||||
]
|
||||
|
||||
async def fake_rank(self, kb_id, seed_weights, *, max_nodes, top_k, damping):
|
||||
assert seed_weights["anchor_entity"] == 1.0
|
||||
return [("graph_anchor", 0.9), ("graph_neighbor", 0.8)]
|
||||
|
||||
fake_llm = FakeLlm(gold_chunk_id="graph_neighbor")
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.knowledge.graphs.milvus_graph_service.MilvusGraphService.query_and_rank_chunks_by_ppr",
|
||||
fake_rank,
|
||||
)
|
||||
kb = FakeGraphGenerationKnowledgeBase()
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=kb,
|
||||
kb_id="db_1",
|
||||
count=1,
|
||||
neighbors_count=2,
|
||||
llm_model_spec="test-provider:test-model",
|
||||
generation_mode="graph_enhanced",
|
||||
)
|
||||
]
|
||||
|
||||
assert items == [{"query": "问题", "gold_chunk_ids": ["graph_neighbor"], "gold_answer": "答案"}]
|
||||
assert kb.query_calls == []
|
||||
assert "片段ID=graph_anchor" in fake_llm.prompts[0]
|
||||
assert "片段ID=graph_neighbor" in fake_llm.prompts[0]
|
||||
assert "片段ID=vector_anchor" not in fake_llm.prompts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_uses_query_neighbor(monkeypatch):
|
||||
fake_llm = FakeLlm(gold_chunk_id="neighbor_chunk")
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
kb = FakeGenerationKnowledgeBase(
|
||||
query_results=[
|
||||
{
|
||||
"content": "neighbor content",
|
||||
"metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=kb,
|
||||
kb_id="db_1",
|
||||
count=1,
|
||||
neighbors_count=2,
|
||||
llm_model_spec="test-provider:test-model",
|
||||
)
|
||||
]
|
||||
|
||||
assert items == [{"query": "问题", "gold_chunk_ids": ["neighbor_chunk"], "gold_answer": "答案"}]
|
||||
assert kb.query_calls[0]["query_text"] == "anchor content"
|
||||
assert kb.query_calls[0]["search_mode"] == "vector"
|
||||
assert "片段ID=neighbor_chunk" in fake_llm.prompts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_falls_back_to_anchor_when_query_empty(monkeypatch):
|
||||
fake_llm = FakeLlm()
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=FakeGenerationKnowledgeBase(query_results=[]),
|
||||
kb_id="db_1",
|
||||
count=1,
|
||||
neighbors_count=2,
|
||||
llm_model_spec="test-provider:test-model",
|
||||
)
|
||||
]
|
||||
|
||||
assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}]
|
||||
assert "片段ID=anchor_chunk" in fake_llm.prompts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_respects_concurrency_count(monkeypatch):
|
||||
fake_llm = TrackingLlm(delay=0.01)
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=NoQueryKnowledgeBase(),
|
||||
kb_id="db_1",
|
||||
count=4,
|
||||
neighbors_count=1,
|
||||
concurrency_count=2,
|
||||
llm_model_spec="test-provider:test-model",
|
||||
)
|
||||
]
|
||||
|
||||
assert len(items) == 4
|
||||
assert fake_llm.max_active_calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_returns_at_most_count(monkeypatch):
|
||||
fake_llm = TrackingLlm(delay=0.01)
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=NoQueryKnowledgeBase(),
|
||||
kb_id="db_1",
|
||||
count=3,
|
||||
neighbors_count=1,
|
||||
concurrency_count=10,
|
||||
llm_model_spec="test-provider:test-model",
|
||||
)
|
||||
]
|
||||
|
||||
assert len(items) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_stops_at_max_attempts(monkeypatch):
|
||||
fake_llm = TrackingLlm(content='{"query":"","gold_answer":"答案","gold_chunk_ids":["anchor_chunk"]}')
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=NoQueryKnowledgeBase(),
|
||||
kb_id="db_1",
|
||||
count=2,
|
||||
neighbors_count=1,
|
||||
concurrency_count=10,
|
||||
llm_model_spec="test-provider:test-model",
|
||||
)
|
||||
]
|
||||
|
||||
assert items == []
|
||||
assert fake_llm.calls == 50
|
||||
@@ -0,0 +1,39 @@
|
||||
import os
|
||||
|
||||
os.environ.setdefault("OPENAI_API_KEY", "test-key")
|
||||
|
||||
from yuxi.knowledge.eval.evaluator import aggregate_metrics, build_answer_prompt, normalize_query_result
|
||||
|
||||
|
||||
def test_normalize_query_result_supports_dict_and_list():
|
||||
answer, chunks = normalize_query_result({"answer": "A", "retrieved_chunks": [{"content": "C"}]})
|
||||
assert answer == "A"
|
||||
assert chunks == [{"content": "C"}]
|
||||
|
||||
answer, chunks = normalize_query_result([{"content": "C"}])
|
||||
assert answer == ""
|
||||
assert chunks == [{"content": "C"}]
|
||||
|
||||
|
||||
def test_build_answer_prompt_uses_first_five_non_empty_chunks():
|
||||
chunks = [{"content": f"内容{i}"} for i in range(6)] + [{"content": ""}]
|
||||
|
||||
prompt = build_answer_prompt("问题", chunks)
|
||||
|
||||
assert "用户问题:问题" in prompt
|
||||
assert "内容0" in prompt
|
||||
assert "内容4" in prompt
|
||||
assert "内容5" not in prompt
|
||||
|
||||
|
||||
def test_aggregate_metrics_matches_service_output_shape():
|
||||
metrics, overall_score = aggregate_metrics(
|
||||
[{"recall@1": 1.0, "f1@1": 0.0}, {"recall@1": 0.0, "f1@1": 1.0}],
|
||||
[{"score": 1.0}, {"score": 0.0}],
|
||||
include_overall_score=True,
|
||||
)
|
||||
|
||||
assert metrics["recall@1"] == 0.5
|
||||
assert metrics["f1@1"] == 0.5
|
||||
assert metrics["answer_correctness"] == 0.5
|
||||
assert metrics["overall_score"] == overall_score
|
||||
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("OPENAI_API_KEY", "test-key")
|
||||
|
||||
from yuxi.knowledge.eval.metrics import EvaluationMetricsCalculator, RetrievalMetrics
|
||||
|
||||
|
||||
def test_retrieval_metrics_use_metadata_chunk_id():
|
||||
retrieved_chunks = [
|
||||
{"metadata": {"chunk_id": "chunk_a"}},
|
||||
{"metadata": {"chunk_id": "chunk_b"}},
|
||||
]
|
||||
|
||||
metrics = EvaluationMetricsCalculator.calculate_retrieval_metrics(
|
||||
retrieved_chunks, ["chunk_b", "chunk_c"], k_values=[1, 3]
|
||||
)
|
||||
|
||||
assert metrics["recall@1"] == 0.0
|
||||
assert metrics["recall@3"] == 0.5
|
||||
assert metrics["f1@3"] == RetrievalMetrics.f1_score_at_k(["chunk_a", "chunk_b"], ["chunk_b", "chunk_c"], 3)
|
||||
|
||||
|
||||
def test_overall_score_uses_answer_accuracy_when_available():
|
||||
# 有答案准确率时,综合得分取各题 score 的平均,且与检索指标无关
|
||||
retrieval = [{"recall@10": 1.0, "f1@10": 0.2}, {"recall@10": 0.0, "f1@10": 0.0}]
|
||||
answers = [{"score": 1.0}, {"score": 0.0}, {"score": 1.0}, {"score": 1.0}]
|
||||
|
||||
score = EvaluationMetricsCalculator.calculate_overall_score(retrieval, answers)
|
||||
|
||||
assert score == 0.75
|
||||
|
||||
|
||||
def test_overall_score_uses_recall_at_10_without_answers():
|
||||
# 无答案准确率时,综合得分取各题 recall@10 的平均,不受 f1/其它 k 影响
|
||||
retrieval = [
|
||||
{"recall@1": 0.0, "recall@5": 0.5, "recall@10": 0.8, "f1@10": 0.1},
|
||||
{"recall@1": 1.0, "recall@5": 1.0, "recall@10": 0.4, "f1@10": 0.9},
|
||||
]
|
||||
|
||||
score = EvaluationMetricsCalculator.calculate_overall_score(retrieval, [])
|
||||
|
||||
assert score == pytest.approx(0.6)
|
||||
|
||||
|
||||
def test_overall_score_returns_none_without_any_metrics():
|
||||
score = EvaluationMetricsCalculator.calculate_overall_score([], [])
|
||||
|
||||
assert score is None
|
||||
@@ -0,0 +1,133 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.knowledge.eval import service as eval_service_module
|
||||
from yuxi.knowledge.eval.service import EvaluationService, build_evaluation_run_name
|
||||
|
||||
|
||||
class FakeEvaluationRepository:
|
||||
def __init__(self):
|
||||
self.created_dataset = None
|
||||
self.updated_dataset = None
|
||||
self.dataset = None
|
||||
self.created_run = None
|
||||
|
||||
async def create_dataset(self, payload):
|
||||
self.created_dataset = payload
|
||||
|
||||
async def update_dataset(self, dataset_id, payload):
|
||||
self.updated_dataset = (dataset_id, payload)
|
||||
|
||||
async def get_dataset(self, dataset_id):
|
||||
return self.dataset
|
||||
|
||||
async def create_run(self, payload):
|
||||
self.created_run = payload
|
||||
|
||||
|
||||
class FakeChunkRepository:
|
||||
def __init__(self, indexed_count):
|
||||
self.indexed_count = indexed_count
|
||||
|
||||
async def count_graph_indexed_by_kb_id(self, kb_id):
|
||||
return self.indexed_count
|
||||
|
||||
|
||||
class FakeKnowledgeBaseRepository:
|
||||
async def get_by_kb_id(self, kb_id):
|
||||
return SimpleNamespace(query_params={"options": {"top_k": 3}})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_dataset_saves_generation_params(monkeypatch):
|
||||
async def fake_enqueue(**kwargs):
|
||||
return SimpleNamespace(id="task_1")
|
||||
|
||||
monkeypatch.setattr(eval_service_module.tasker, "enqueue", fake_enqueue)
|
||||
service = EvaluationService()
|
||||
service.eval_repo = FakeEvaluationRepository()
|
||||
service.chunk_repo = FakeChunkRepository(indexed_count=1)
|
||||
|
||||
result = await service.generate_dataset(
|
||||
kb_id="db_1",
|
||||
name="dataset",
|
||||
description="desc",
|
||||
count=2,
|
||||
neighbors_count=3,
|
||||
concurrency_count=4,
|
||||
llm_model_spec="test:model",
|
||||
generation_mode="graph_enhanced",
|
||||
graph_expand_top_k=2,
|
||||
created_by="user_1",
|
||||
)
|
||||
|
||||
assert result["task_id"] == "task_1"
|
||||
params = service.eval_repo.created_dataset["build_metadata"]["params"]
|
||||
assert params["generation_mode"] == "graph_enhanced"
|
||||
assert params["graph_expand_top_k"] == 2
|
||||
updated_metadata = service.eval_repo.updated_dataset[1]["build_metadata"]
|
||||
assert updated_metadata["params"] == params
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_dataset_rejects_graph_mode_without_indexed_chunks():
|
||||
service = EvaluationService()
|
||||
service.eval_repo = FakeEvaluationRepository()
|
||||
service.chunk_repo = FakeChunkRepository(indexed_count=0)
|
||||
|
||||
with pytest.raises(ValueError, match="尚未完成图索引"):
|
||||
await service.generate_dataset(
|
||||
kb_id="db_1",
|
||||
name="dataset",
|
||||
description="desc",
|
||||
count=2,
|
||||
neighbors_count=3,
|
||||
concurrency_count=4,
|
||||
llm_model_spec="test:model",
|
||||
generation_mode="graph_enhanced",
|
||||
graph_expand_top_k=1,
|
||||
created_by="user_1",
|
||||
)
|
||||
|
||||
assert service.eval_repo.created_dataset is None
|
||||
|
||||
|
||||
def test_build_evaluation_run_name_uses_eval_date_hash_format():
|
||||
name = build_evaluation_run_name(hash_value="abcdef12")
|
||||
|
||||
assert name.startswith("eval-")
|
||||
assert name.endswith("-abcdef")
|
||||
assert len(name.split("-")[1]) == 8
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_evaluation_saves_custom_name(monkeypatch):
|
||||
async def fake_enqueue(**kwargs):
|
||||
return SimpleNamespace(id="task_1")
|
||||
|
||||
monkeypatch.setattr(eval_service_module.tasker, "enqueue", fake_enqueue)
|
||||
repo = FakeEvaluationRepository()
|
||||
repo.dataset = SimpleNamespace(
|
||||
dataset_id="dataset_1",
|
||||
kb_id="db_1",
|
||||
name="dataset",
|
||||
item_count=2,
|
||||
build_metadata={"status": "completed"},
|
||||
)
|
||||
service = EvaluationService()
|
||||
service.eval_repo = repo
|
||||
service.kb_repo = FakeKnowledgeBaseRepository()
|
||||
|
||||
run_id = await service.run_evaluation(
|
||||
kb_id="db_1",
|
||||
dataset_id="dataset_1",
|
||||
name=" 回归评估 ",
|
||||
model_config={"answer_llm": "test:model"},
|
||||
created_by="user_1",
|
||||
)
|
||||
|
||||
assert run_id.startswith("run_")
|
||||
assert repo.created_run["name"] == "回归评估"
|
||||
assert repo.created_run["retrieval_config"]["top_k"] == 3
|
||||
assert repo.created_run["retrieval_config"]["answer_llm"] == "test:model"
|
||||
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from yuxi.repositories import knowledge_file_repository as repository_module
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
from yuxi.storage.postgres.models_knowledge import KnowledgeBase, KnowledgeFile
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
|
||||
|
||||
|
||||
class _AsyncSessionContext:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def __aenter__(self):
|
||||
return self.db
|
||||
|
||||
async def __aexit__(self, exc_type, *_args):
|
||||
if exc_type is None:
|
||||
await self.db.commit()
|
||||
else:
|
||||
await self.db.rollback()
|
||||
return False
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def knowledge_session(monkeypatch):
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(KnowledgeBase.__table__.create)
|
||||
await conn.run_sync(KnowledgeFile.__table__.create)
|
||||
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with session_factory() as session:
|
||||
monkeypatch.setattr(
|
||||
repository_module.pg_manager,
|
||||
"get_async_session_context",
|
||||
lambda: _AsyncSessionContext(session),
|
||||
)
|
||||
yield session
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_exists_by_filename_matches_active_file_exactly(knowledge_session):
|
||||
existing_name = (
|
||||
"google_drive/shared_drives/engineering/serving-runtime/dsid_e4ff04ebc2a14c1982abc4987753790c__playbook.txt"
|
||||
)
|
||||
knowledge_session.add_all(
|
||||
[
|
||||
KnowledgeBase(kb_id="kb_1", name="KB 1", description="", kb_type="milvus"),
|
||||
KnowledgeBase(kb_id="kb_2", name="KB 2", description="", kb_type="milvus"),
|
||||
KnowledgeFile(
|
||||
file_id="file_active",
|
||||
kb_id="kb_1",
|
||||
filename=existing_name,
|
||||
status="uploaded",
|
||||
is_folder=False,
|
||||
),
|
||||
KnowledgeFile(
|
||||
file_id="file_other_kb",
|
||||
kb_id="kb_2",
|
||||
filename=existing_name,
|
||||
status="uploaded",
|
||||
is_folder=False,
|
||||
),
|
||||
KnowledgeFile(
|
||||
file_id="file_failed",
|
||||
kb_id="kb_1",
|
||||
filename="failed.txt",
|
||||
status="failed",
|
||||
is_folder=False,
|
||||
),
|
||||
KnowledgeFile(
|
||||
file_id="folder_same_name",
|
||||
kb_id="kb_1",
|
||||
filename="folder",
|
||||
status="done",
|
||||
is_folder=True,
|
||||
),
|
||||
KnowledgeFile(
|
||||
file_id="legacy_file",
|
||||
kb_id="kb_1",
|
||||
filename="legacy.txt",
|
||||
status="indexed",
|
||||
is_folder=None,
|
||||
),
|
||||
]
|
||||
)
|
||||
await knowledge_session.commit()
|
||||
|
||||
repo = KnowledgeFileRepository()
|
||||
|
||||
assert await repo.exists_by_filename(kb_id="kb_1", filename=existing_name) is True
|
||||
assert await repo.exists_by_filename(kb_id="kb_1", filename=existing_name.upper()) is False
|
||||
assert await repo.exists_by_filename(kb_id="kb_1", filename="failed.txt") is False
|
||||
assert await repo.exists_by_filename(kb_id="kb_1", filename="folder") is False
|
||||
assert await repo.exists_by_filename(kb_id="kb_1", filename="legacy.txt") is True
|
||||
assert await repo.exists_by_filename(kb_id="missing", filename=existing_name) is False
|
||||
@@ -0,0 +1,279 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.knowledge.manager import KnowledgeBaseManager
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
class FakeKnowledgeBaseClass:
|
||||
@classmethod
|
||||
def normalize_additional_params(cls, additional_params):
|
||||
return dict(additional_params or {})
|
||||
|
||||
|
||||
class FakeKnowledgeBaseRepository:
|
||||
async def get_by_kb_id(self, kb_id):
|
||||
if kb_id != "kb_1":
|
||||
return None
|
||||
return SimpleNamespace(
|
||||
kb_id="kb_1",
|
||||
name="知识库",
|
||||
description="desc",
|
||||
kb_type="milvus",
|
||||
embedding_model_spec="embedding:model",
|
||||
llm_model_spec="llm:model",
|
||||
query_params={"options": {}},
|
||||
additional_params={"chunk_preset_id": "general"},
|
||||
share_config=None,
|
||||
mindmap=None,
|
||||
sample_questions=[],
|
||||
created_at=None,
|
||||
)
|
||||
|
||||
|
||||
class FakeKnowledgeFileRepository:
|
||||
list_calls = []
|
||||
exists_calls = []
|
||||
action_id_calls = []
|
||||
|
||||
def __init__(self):
|
||||
self.records = [
|
||||
SimpleNamespace(
|
||||
file_id="folder_1",
|
||||
kb_id="kb_1",
|
||||
parent_id=None,
|
||||
filename="资料",
|
||||
file_type=None,
|
||||
status="done",
|
||||
is_folder=True,
|
||||
path=None,
|
||||
minio_url=None,
|
||||
markdown_file=None,
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=0,
|
||||
),
|
||||
SimpleNamespace(
|
||||
file_id="file_1",
|
||||
kb_id="kb_1",
|
||||
parent_id=None,
|
||||
filename="alpha.pdf",
|
||||
file_type="pdf",
|
||||
status="indexed",
|
||||
is_folder=False,
|
||||
path="minio://bucket/file",
|
||||
minio_url="minio://bucket/file",
|
||||
markdown_file="minio://bucket/parsed",
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=1024,
|
||||
),
|
||||
]
|
||||
|
||||
async def get_kb_file_stats(self, kb_id):
|
||||
return {
|
||||
"row_count": 3,
|
||||
"file_count": 2,
|
||||
"folder_count": 1,
|
||||
"total_size": 1024,
|
||||
"chunk_count": 9,
|
||||
"token_count": 128,
|
||||
"pending_parse_count": 1,
|
||||
"pending_index_count": 0,
|
||||
"processing_count": 0,
|
||||
}
|
||||
|
||||
async def get_by_file_id(self, file_id):
|
||||
return next((record for record in self.records if record.file_id == file_id), None)
|
||||
|
||||
async def list_documents(self, **kwargs):
|
||||
self.__class__.list_calls.append(kwargs)
|
||||
return self.records, 2
|
||||
|
||||
async def list_file_ids_by_exact_statuses(self, **kwargs):
|
||||
self.__class__.action_id_calls.append(kwargs)
|
||||
return ["file_2"]
|
||||
|
||||
async def exists_by_filename(self, *, kb_id, filename):
|
||||
self.__class__.exists_calls.append({"kb_id": kb_id, "filename": filename})
|
||||
return filename == "docs/Guide.md"
|
||||
|
||||
async def count_children_by_parent_ids(self, *, kb_id, parent_ids):
|
||||
return {"folder_1": 1}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def patch_repositories(monkeypatch):
|
||||
FakeKnowledgeFileRepository.list_calls = []
|
||||
FakeKnowledgeFileRepository.exists_calls = []
|
||||
FakeKnowledgeFileRepository.action_id_calls = []
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_base_repository.KnowledgeBaseRepository",
|
||||
FakeKnowledgeBaseRepository,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
|
||||
FakeKnowledgeFileRepository,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.knowledge.manager.KnowledgeBaseFactory.is_type_supported",
|
||||
staticmethod(lambda _kb_type: True),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.knowledge.manager.KnowledgeBaseFactory.get_kb_class",
|
||||
staticmethod(lambda _kb_type: FakeKnowledgeBaseClass),
|
||||
)
|
||||
|
||||
|
||||
async def test_get_database_info_omits_files_by_default():
|
||||
manager = KnowledgeBaseManager("/tmp/yuxi-test")
|
||||
|
||||
result = await manager.get_database_info("kb_1")
|
||||
|
||||
assert result["kb_id"] == "kb_1"
|
||||
assert "files" not in result
|
||||
assert result["stats"]["file_count"] == 2
|
||||
assert result["stats"]["total_size"] == 1024
|
||||
|
||||
|
||||
async def test_list_document_files_returns_lightweight_paginated_items():
|
||||
manager = KnowledgeBaseManager("/tmp/yuxi-test")
|
||||
|
||||
result = await manager.list_document_files(
|
||||
"kb_1",
|
||||
parent_id="folder_1",
|
||||
status="indexed",
|
||||
page=2,
|
||||
page_size=50,
|
||||
)
|
||||
|
||||
assert result["page"] == 2
|
||||
assert result["page_size"] == 50
|
||||
assert result["total"] == 2
|
||||
assert FakeKnowledgeFileRepository.list_calls == [
|
||||
{
|
||||
"kb_id": "kb_1",
|
||||
"parent_id": "folder_1",
|
||||
"path_prefix": None,
|
||||
"status": "indexed",
|
||||
"page": 2,
|
||||
"page_size": 50,
|
||||
"recursive": False,
|
||||
"files_only": False,
|
||||
}
|
||||
]
|
||||
assert result["items"][0]["has_children"] is True
|
||||
assert result["items"][1]["file_size"] == 1024
|
||||
assert result["items"][1]["has_original_file"] is True
|
||||
assert result["items"][1]["has_parsed_markdown"] is True
|
||||
|
||||
returned_keys = set(result["items"][1])
|
||||
assert "path" not in returned_keys
|
||||
assert "markdown_file" not in returned_keys
|
||||
assert "chunk_count" not in returned_keys
|
||||
assert "token_count" not in returned_keys
|
||||
assert "processing_params" not in returned_keys
|
||||
|
||||
|
||||
async def test_list_document_files_keeps_virtual_folder_contract():
|
||||
manager = KnowledgeBaseManager("/tmp/yuxi-test")
|
||||
virtual_record = SimpleNamespace(
|
||||
file_id="__virtual_folder__:root:资料/",
|
||||
kb_id="kb_1",
|
||||
parent_id=None,
|
||||
filename="资料",
|
||||
file_type="folder",
|
||||
status="done",
|
||||
is_folder=True,
|
||||
is_virtual_folder=True,
|
||||
path_prefix="资料/",
|
||||
virtual_children_count=3,
|
||||
path=None,
|
||||
minio_url=None,
|
||||
markdown_file=None,
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=0,
|
||||
)
|
||||
|
||||
item = manager._file_record_list_item(virtual_record)
|
||||
|
||||
assert item["is_folder"] is True
|
||||
assert item["is_virtual_folder"] is True
|
||||
assert item["path_prefix"] == "资料/"
|
||||
assert item["has_children"] is True
|
||||
assert item["children_count"] == 3
|
||||
|
||||
|
||||
async def test_list_document_files_passes_files_only_and_can_omit_stats():
|
||||
manager = KnowledgeBaseManager("/tmp/yuxi-test")
|
||||
|
||||
result = await manager.list_document_files("kb_1", files_only=True, include_stats=False)
|
||||
|
||||
assert "stats" not in result
|
||||
assert FakeKnowledgeFileRepository.list_calls == [
|
||||
{
|
||||
"kb_id": "kb_1",
|
||||
"parent_id": None,
|
||||
"path_prefix": None,
|
||||
"status": None,
|
||||
"page": 1,
|
||||
"page_size": 100,
|
||||
"recursive": False,
|
||||
"files_only": True,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def test_list_document_files_ignores_recursive_without_status_filter():
|
||||
manager = KnowledgeBaseManager("/tmp/yuxi-test")
|
||||
|
||||
result = await manager.list_document_files("kb_1", recursive=True, include_stats=False)
|
||||
|
||||
assert result["recursive"] is False
|
||||
assert FakeKnowledgeFileRepository.list_calls == [
|
||||
{
|
||||
"kb_id": "kb_1",
|
||||
"parent_id": None,
|
||||
"path_prefix": None,
|
||||
"status": None,
|
||||
"page": 1,
|
||||
"page_size": 100,
|
||||
"recursive": False,
|
||||
"files_only": False,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def test_document_file_exists_delegates_exact_filename_to_repository():
|
||||
manager = KnowledgeBaseManager("/tmp/yuxi-test")
|
||||
|
||||
assert await manager.document_file_exists("kb_1", " docs/Guide.md ") is True
|
||||
assert await manager.document_file_exists("kb_1", "docs/guide.md") is False
|
||||
assert FakeKnowledgeFileRepository.exists_calls == [
|
||||
{"kb_id": "kb_1", "filename": "docs/Guide.md"},
|
||||
{"kb_id": "kb_1", "filename": "docs/guide.md"},
|
||||
]
|
||||
|
||||
|
||||
async def test_list_document_file_ids_by_statuses_delegates_to_repository():
|
||||
manager = KnowledgeBaseManager("/tmp/yuxi-test")
|
||||
|
||||
result = await manager.list_document_file_ids_by_statuses(
|
||||
"kb_1",
|
||||
statuses=["parsed", "error_indexing"],
|
||||
after_file_id="file_1",
|
||||
limit=500,
|
||||
)
|
||||
|
||||
assert result == ["file_2"]
|
||||
assert FakeKnowledgeFileRepository.action_id_calls == [
|
||||
{
|
||||
"kb_id": "kb_1",
|
||||
"statuses": ["parsed", "error_indexing"],
|
||||
"after_file_id": "file_1",
|
||||
"limit": 500,
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,244 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestMinIOClientStatFile:
|
||||
"""Test MinIOClient.stat_file and astat_file methods."""
|
||||
|
||||
def test_stat_file_returns_size(self):
|
||||
from yuxi.storage.minio.client import MinIOClient
|
||||
|
||||
client = MinIOClient()
|
||||
mock_stat = MagicMock()
|
||||
mock_stat.size = 1024
|
||||
client._client = MagicMock()
|
||||
client._client.stat_object.return_value = mock_stat
|
||||
|
||||
result = client.stat_file("knowledgebases", "db/upload/test.pdf")
|
||||
assert result == 1024
|
||||
client._client.stat_object.assert_called_once_with(
|
||||
bucket_name="knowledgebases", object_name="db/upload/test.pdf"
|
||||
)
|
||||
|
||||
def test_stat_file_returns_none_when_not_found(self):
|
||||
from io import BytesIO
|
||||
|
||||
from minio.error import S3Error
|
||||
from urllib3 import HTTPResponse
|
||||
|
||||
from yuxi.storage.minio.client import MinIOClient
|
||||
|
||||
client = MinIOClient()
|
||||
client._client = MagicMock()
|
||||
resp = HTTPResponse(BytesIO(b""), status=404)
|
||||
client._client.stat_object.side_effect = S3Error(
|
||||
resp, "NoSuchKey", "Not found", "resource", "request_id", "host_id"
|
||||
)
|
||||
|
||||
result = client.stat_file("knowledgebases", "db/upload/missing.pdf")
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_astat_file_returns_size(self):
|
||||
from yuxi.storage.minio.client import MinIOClient
|
||||
|
||||
client = MinIOClient()
|
||||
mock_stat = MagicMock()
|
||||
mock_stat.size = 2048
|
||||
client._client = MagicMock()
|
||||
client._client.stat_object.return_value = mock_stat
|
||||
|
||||
result = await client.astat_file("knowledgebases", "db/upload/test.pdf")
|
||||
assert result == 2048
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestAddFileRecordSizeFallback:
|
||||
"""Test that add_file_record fills size from MinIO when not provided."""
|
||||
|
||||
def _make_test_kb(self, work_dir="/tmp/test_kb"):
|
||||
from yuxi.knowledge.base import KnowledgeBase
|
||||
|
||||
class TestKB(KnowledgeBase):
|
||||
@property
|
||||
def kb_type(self):
|
||||
return "test"
|
||||
|
||||
async def _create_kb_instance(self, kb_id, config):
|
||||
pass
|
||||
|
||||
async def _initialize_kb_instance(self, instance):
|
||||
pass
|
||||
|
||||
async def _persist_file_meta(self, file_id, meta):
|
||||
pass
|
||||
|
||||
async def _persist_kb(self, kb_id):
|
||||
pass
|
||||
|
||||
async def _save_metadata(self):
|
||||
pass
|
||||
|
||||
async def refresh_database_stats(self, kb_id):
|
||||
return {}
|
||||
|
||||
async def index_file(self, kb_id, file_id, operator_id=None):
|
||||
return {}
|
||||
|
||||
async def aquery(self, query_text, kb_id, **kwargs):
|
||||
return []
|
||||
|
||||
async def delete_file(self, kb_id, file_id):
|
||||
pass
|
||||
|
||||
async def get_file_basic_info(self, kb_id, file_id):
|
||||
return {}
|
||||
|
||||
async def get_file_content(self, kb_id, file_id):
|
||||
return {}
|
||||
|
||||
async def get_file_info(self, kb_id, file_id):
|
||||
return {}
|
||||
|
||||
async def update_content(self, kb_id, file_ids, params=None):
|
||||
return []
|
||||
|
||||
async def get_query_params_config(self, kb_id, **kwargs):
|
||||
return {"type": "test", "options": []}
|
||||
|
||||
kb = TestKB(work_dir=work_dir)
|
||||
kb.databases_meta["db1"] = {"metadata": {}}
|
||||
return kb
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_file_record_fetches_size_from_minio_when_missing(self):
|
||||
kb = self._make_test_kb()
|
||||
|
||||
item = "minio://knowledgebases/db1/upload/test_1234567890123.pdf"
|
||||
params = {
|
||||
"content_type": "file",
|
||||
"content_hashes": {item: "abc123"},
|
||||
}
|
||||
|
||||
mock_minio = AsyncMock()
|
||||
mock_minio.astat_file.return_value = 9999
|
||||
|
||||
with patch("yuxi.storage.minio.get_minio_client", return_value=mock_minio):
|
||||
metadata = await kb.add_file_record("db1", item, params=params)
|
||||
|
||||
assert metadata["size"] == 9999
|
||||
mock_minio.astat_file.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_file_record_keeps_provided_size(self):
|
||||
kb = self._make_test_kb()
|
||||
|
||||
item = "minio://knowledgebases/db1/upload/test_1234567890123.pdf"
|
||||
params = {
|
||||
"content_type": "file",
|
||||
"content_hashes": {item: "abc123"},
|
||||
"file_sizes": {item: 5555},
|
||||
}
|
||||
|
||||
mock_minio = AsyncMock()
|
||||
|
||||
with patch("yuxi.storage.minio.get_minio_client", return_value=mock_minio):
|
||||
metadata = await kb.add_file_record("db1", item, params=params)
|
||||
|
||||
assert metadata["size"] == 5555
|
||||
mock_minio.astat_file.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_file_record_size_fallback_handles_error_gracefully(self):
|
||||
kb = self._make_test_kb()
|
||||
|
||||
item = "minio://knowledgebases/db1/upload/test_1234567890123.pdf"
|
||||
params = {
|
||||
"content_type": "file",
|
||||
"content_hashes": {item: "abc123"},
|
||||
}
|
||||
|
||||
mock_minio = AsyncMock()
|
||||
mock_minio.astat_file.side_effect = Exception("MinIO connection error")
|
||||
|
||||
with patch("yuxi.storage.minio.get_minio_client", return_value=mock_minio):
|
||||
metadata = await kb.add_file_record("db1", item, params=params)
|
||||
|
||||
assert metadata.get("size") is None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_metadata_does_not_overwrite_existing_kb_config():
|
||||
from yuxi.knowledge.base import KnowledgeBase
|
||||
|
||||
class TestKB(KnowledgeBase):
|
||||
@property
|
||||
def kb_type(self):
|
||||
return "test"
|
||||
|
||||
async def _create_kb_instance(self, kb_id, config):
|
||||
pass
|
||||
|
||||
async def _initialize_kb_instance(self, instance):
|
||||
pass
|
||||
|
||||
async def index_file(self, kb_id, file_id, operator_id=None):
|
||||
return {}
|
||||
|
||||
async def update_content(self, kb_id, file_ids, params=None):
|
||||
return []
|
||||
|
||||
async def aquery(self, query_text, kb_id, **kwargs):
|
||||
return []
|
||||
|
||||
def get_query_params_config(self, kb_id, **kwargs):
|
||||
return {"type": "test", "options": []}
|
||||
|
||||
async def delete_file(self, kb_id, file_id):
|
||||
pass
|
||||
|
||||
async def get_file_basic_info(self, kb_id, file_id):
|
||||
return {}
|
||||
|
||||
async def get_file_content(self, kb_id, file_id):
|
||||
return {}
|
||||
|
||||
async def get_file_info(self, kb_id, file_id):
|
||||
return {}
|
||||
|
||||
class ExistingKbRepo:
|
||||
def __init__(self):
|
||||
self.created = []
|
||||
self.updated = []
|
||||
|
||||
async def get_by_kb_id(self, kb_id):
|
||||
return SimpleNamespace(kb_id=kb_id)
|
||||
|
||||
async def create(self, payload):
|
||||
self.created.append(payload)
|
||||
|
||||
async def update(self, kb_id, data):
|
||||
self.updated.append((kb_id, data))
|
||||
|
||||
kb_repo = ExistingKbRepo()
|
||||
kb = TestKB(work_dir="/tmp/test_kb")
|
||||
kb.databases_meta["db1"] = {
|
||||
"name": "Runtime name",
|
||||
"description": "Runtime description",
|
||||
"kb_type": "test",
|
||||
"metadata": {"graph_build_config": {"extractor_options": {"concurrency_count": 5}}},
|
||||
}
|
||||
|
||||
with (
|
||||
patch("yuxi.repositories.knowledge_base_repository.KnowledgeBaseRepository", return_value=kb_repo),
|
||||
patch("yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository", return_value=SimpleNamespace()),
|
||||
patch("yuxi.repositories.evaluation_repository.EvaluationRepository", return_value=SimpleNamespace()),
|
||||
):
|
||||
await kb._save_metadata()
|
||||
|
||||
assert kb_repo.created == []
|
||||
assert kb_repo.updated == []
|
||||
@@ -0,0 +1,54 @@
|
||||
import pytest
|
||||
|
||||
from yuxi.knowledge.utils.kb_utils import prepare_item_metadata
|
||||
|
||||
|
||||
async def test_prepare_item_metadata_preserves_uploaded_file_size():
|
||||
item = "minio://knowledgebases/db/upload/demo.txt"
|
||||
params = {
|
||||
"content_hashes": {item: "hash"},
|
||||
"file_sizes": {item: 1234},
|
||||
}
|
||||
|
||||
metadata = await prepare_item_metadata(item, "file", "db", params=params)
|
||||
|
||||
assert metadata["size"] == 1234
|
||||
assert "file_sizes" not in (metadata.get("processing_params") or {})
|
||||
|
||||
|
||||
async def test_prepare_item_metadata_uses_source_path_as_display_filename():
|
||||
item = "minio://knowledgebases/db/upload/intro_1710000000000.md"
|
||||
params = {
|
||||
"content_hashes": {item: "hash"},
|
||||
"source_path": "guides/setup/Intro.MD",
|
||||
}
|
||||
|
||||
metadata = await prepare_item_metadata(item, "file", "db", params=params)
|
||||
|
||||
assert metadata["filename"] == "guides/setup/Intro.MD"
|
||||
assert metadata["file_type"] == "md"
|
||||
assert metadata["path"] == item
|
||||
|
||||
|
||||
async def test_prepare_item_metadata_preserves_preprocessed_file_size():
|
||||
item = "minio://knowledgebases/db/upload/page.html"
|
||||
params = {
|
||||
"_preprocessed_map": {
|
||||
item: {
|
||||
"path": item,
|
||||
"content_hash": "hash",
|
||||
"filename": "https://example.com",
|
||||
"file_size": 5678,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
metadata = await prepare_item_metadata(item, "file", "db", params=params)
|
||||
|
||||
assert metadata["size"] == 5678
|
||||
assert "_preprocessed_map" not in (metadata.get("processing_params") or {})
|
||||
|
||||
|
||||
async def test_prepare_item_metadata_rejects_direct_url_content_type():
|
||||
with pytest.raises(ValueError, match="Unsupported content_type"):
|
||||
await prepare_item_metadata("https://example.com", "url", "db")
|
||||
@@ -0,0 +1,391 @@
|
||||
import asyncio
|
||||
import types
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like.nlp import count_tokens
|
||||
from yuxi.knowledge.base import KnowledgeBase
|
||||
|
||||
|
||||
class FakeKnowledgeBase(KnowledgeBase):
|
||||
@property
|
||||
def kb_type(self) -> str:
|
||||
return "fake"
|
||||
|
||||
async def _create_kb_instance(self, slug: str, config: dict):
|
||||
return None
|
||||
|
||||
async def _initialize_kb_instance(self, instance) -> None:
|
||||
pass
|
||||
|
||||
async def index_file(self, slug: str, file_id: str, operator_id: str | None = None) -> dict:
|
||||
return {}
|
||||
|
||||
async def update_content(self, slug: str, file_ids: list[str], params: dict | None = None) -> list[dict]:
|
||||
return []
|
||||
|
||||
async def aquery(self, query_text: str, slug: str, **kwargs) -> list[dict]:
|
||||
return []
|
||||
|
||||
def get_query_params_config(self, slug: str, **kwargs) -> dict:
|
||||
return {"options": []}
|
||||
|
||||
async def delete_file(self, slug: str, file_id: str) -> None:
|
||||
pass
|
||||
|
||||
async def get_file_basic_info(self, slug: str, file_id: str) -> dict:
|
||||
return {}
|
||||
|
||||
async def get_file_content(self, slug: str, file_id: str) -> dict:
|
||||
return {}
|
||||
|
||||
async def get_file_info(self, slug: str, file_id: str) -> dict:
|
||||
return {}
|
||||
|
||||
async def _save_metadata(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def make_kb(tmp_path):
|
||||
kb = FakeKnowledgeBase(str(tmp_path))
|
||||
kb.databases_meta = {
|
||||
"db": {
|
||||
"name": "Old name",
|
||||
"description": "Old description",
|
||||
"kb_type": "fake",
|
||||
"llm_model_spec": "provider:model-a",
|
||||
}
|
||||
}
|
||||
return kb
|
||||
|
||||
|
||||
def make_file_record(file_id: str, meta: dict):
|
||||
return types.SimpleNamespace(
|
||||
file_id=file_id,
|
||||
kb_id=meta.get("kb_id"),
|
||||
parent_id=meta.get("parent_id"),
|
||||
filename=meta.get("filename", ""),
|
||||
file_type=meta.get("file_type"),
|
||||
path=meta.get("path"),
|
||||
minio_url=meta.get("minio_url"),
|
||||
markdown_file=meta.get("markdown_file"),
|
||||
status=meta.get("status"),
|
||||
content_hash=meta.get("content_hash"),
|
||||
file_size=meta.get("size", meta.get("file_size")),
|
||||
chunk_count=meta.get("chunk_count", 0),
|
||||
token_count=meta.get("token_count", 0),
|
||||
content_type=meta.get("content_type"),
|
||||
processing_params=meta.get("processing_params"),
|
||||
is_folder=meta.get("is_folder", False),
|
||||
error_message=meta.get("error"),
|
||||
created_by=meta.get("created_by"),
|
||||
updated_by=meta.get("updated_by"),
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
original_filename=meta.get("original_filename"),
|
||||
)
|
||||
|
||||
|
||||
class FakeFileRepository:
|
||||
def __init__(self, records: dict[str, types.SimpleNamespace]):
|
||||
self.records = records
|
||||
self.update_calls = []
|
||||
|
||||
async def list_by_kb_id(self, kb_id: str):
|
||||
return [record for record in self.records.values() if record.kb_id == kb_id]
|
||||
|
||||
async def list_by_kb_id_after(
|
||||
self,
|
||||
kb_id: str,
|
||||
*,
|
||||
after_file_id: str | None = None,
|
||||
limit: int = 500,
|
||||
files_only: bool = False,
|
||||
):
|
||||
records = [
|
||||
record
|
||||
for record in self.records.values()
|
||||
if record.kb_id == kb_id
|
||||
and (not after_file_id or record.file_id > after_file_id)
|
||||
and (not files_only or not record.is_folder)
|
||||
]
|
||||
records.sort(key=lambda record: record.file_id)
|
||||
return records[:limit]
|
||||
|
||||
async def update_fields(self, *, file_id: str, data: dict, kb_id: str | None = None):
|
||||
record = self.records.get(file_id)
|
||||
if record is None or (kb_id and record.kb_id != kb_id):
|
||||
return None
|
||||
for key, value in data.items():
|
||||
setattr(record, key, value)
|
||||
self.update_calls.append((file_id, kb_id, dict(data)))
|
||||
return record
|
||||
|
||||
async def get_kb_file_stats(self, kb_id: str):
|
||||
records = [record for record in self.records.values() if record.kb_id == kb_id]
|
||||
files = [record for record in records if not record.is_folder]
|
||||
return {
|
||||
"row_count": len(records),
|
||||
"file_count": len(files),
|
||||
"folder_count": len(records) - len(files),
|
||||
"total_size": sum(int(record.file_size or 0) for record in files),
|
||||
"chunk_count": sum(int(record.chunk_count or 0) for record in files),
|
||||
"token_count": sum(int(record.token_count or 0) for record in files),
|
||||
"pending_parse_count": sum(1 for record in files if record.status == "uploaded"),
|
||||
"pending_index_count": sum(1 for record in files if record.status in {"parsed", "error_indexing"}),
|
||||
"processing_count": sum(1 for record in files if record.status in {"processing", "waiting", "parsing", "indexing"}),
|
||||
}
|
||||
|
||||
|
||||
def make_file_records(files: dict[str, dict]) -> dict[str, types.SimpleNamespace]:
|
||||
return {file_id: make_file_record(file_id, meta) for file_id, meta in files.items()}
|
||||
|
||||
|
||||
async def test_create_database_persists_allowed_record_fields(tmp_path, monkeypatch):
|
||||
created_payloads = []
|
||||
|
||||
class FakeKnowledgeBaseRepository:
|
||||
async def get_by_kb_id(self, kb_id):
|
||||
return None
|
||||
|
||||
async def create(self, payload):
|
||||
created_payloads.append(payload)
|
||||
return types.SimpleNamespace(**payload)
|
||||
|
||||
async def update(self, kb_id, data):
|
||||
raise AssertionError("create_database should insert new database metadata")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_base_repository.KnowledgeBaseRepository",
|
||||
FakeKnowledgeBaseRepository,
|
||||
)
|
||||
|
||||
kb = FakeKnowledgeBase(str(tmp_path))
|
||||
share_config = {"access_level": "user", "department_ids": [], "user_uids": ["root"]}
|
||||
|
||||
await kb.create_database(
|
||||
"New database",
|
||||
"New description",
|
||||
embedding_model_spec="provider:embedding",
|
||||
record_fields={
|
||||
"share_config": share_config,
|
||||
"created_by": "root",
|
||||
"unexpected_field": "ignored",
|
||||
},
|
||||
auto_generate_questions=False,
|
||||
)
|
||||
|
||||
assert len(created_payloads) == 1
|
||||
payload = created_payloads[0]
|
||||
assert payload["share_config"] == share_config
|
||||
assert payload["created_by"] == "root"
|
||||
assert "unexpected_field" not in payload
|
||||
assert "share_config" not in payload["additional_params"]
|
||||
assert "created_by" not in payload["additional_params"]
|
||||
|
||||
|
||||
async def test_update_database_keeps_llm_spec_when_field_is_omitted(tmp_path):
|
||||
kb = make_kb(tmp_path)
|
||||
|
||||
result = kb.update_database("db", "New name", "New description")
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert result["llm_model_spec"] == "provider:model-a"
|
||||
assert kb.databases_meta["db"]["llm_model_spec"] == "provider:model-a"
|
||||
|
||||
|
||||
async def test_update_database_clears_llm_spec_when_field_is_explicit(tmp_path):
|
||||
kb = make_kb(tmp_path)
|
||||
|
||||
result = kb.update_database("db", "New name", "New description", None, update_llm_model_spec=True)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert result["llm_model_spec"] is None
|
||||
assert kb.databases_meta["db"]["llm_model_spec"] is None
|
||||
|
||||
|
||||
def test_get_database_info_returns_persisted_content_stats(tmp_path):
|
||||
kb = make_kb(tmp_path)
|
||||
kb.databases_meta["db"]["metadata"] = {
|
||||
"stats": {"row_count": 3, "file_count": 2, "chunk_count": 5, "token_count": 25}
|
||||
}
|
||||
|
||||
result = kb.get_database_info("db")
|
||||
|
||||
assert result["row_count"] == 3
|
||||
assert result["stats"]["file_count"] == 2
|
||||
assert result["stats"]["chunk_count"] == 5
|
||||
assert result["stats"]["token_count"] == 25
|
||||
assert result["files"] == {}
|
||||
assert result["files_truncated"] is True
|
||||
|
||||
|
||||
def test_get_database_info_prefers_metadata_stats(tmp_path):
|
||||
kb = make_kb(tmp_path)
|
||||
kb.databases_meta["db"]["metadata"] = {"stats": {"file_count": 2, "chunk_count": 8, "token_count": 40}}
|
||||
|
||||
result = kb.get_database_info("db")
|
||||
|
||||
assert result["stats"]["file_count"] == 2
|
||||
assert result["stats"]["chunk_count"] == 8
|
||||
assert result["stats"]["token_count"] == 40
|
||||
|
||||
|
||||
async def test_refresh_database_stats_persists_metadata(tmp_path, monkeypatch):
|
||||
kb = make_kb(tmp_path)
|
||||
kb.databases_meta["db"]["metadata"] = {}
|
||||
records = make_file_records({
|
||||
"file-1": {"kb_id": "db", "filename": "alpha.md", "chunk_count": 2, "token_count": 10},
|
||||
"folder-1": {
|
||||
"kb_id": "db",
|
||||
"filename": "folder",
|
||||
"is_folder": True,
|
||||
"chunk_count": 99,
|
||||
"token_count": 99,
|
||||
},
|
||||
})
|
||||
file_repo = FakeFileRepository(records)
|
||||
persisted_kbs = []
|
||||
|
||||
async def persist_kb(kb_id):
|
||||
persisted_kbs.append((kb_id, dict(kb.databases_meta[kb_id]["metadata"])))
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
|
||||
lambda: file_repo,
|
||||
)
|
||||
kb._persist_kb = persist_kb
|
||||
|
||||
stats = await kb.refresh_database_stats("db")
|
||||
|
||||
assert stats["file_count"] == 1
|
||||
assert stats["chunk_count"] == 2
|
||||
assert stats["token_count"] == 10
|
||||
assert kb.databases_meta["db"]["metadata"]["stats"] == stats
|
||||
assert persisted_kbs == [("db", {"stats": stats})]
|
||||
|
||||
|
||||
async def test_repair_missing_file_stats_updates_files_and_database_metadata(tmp_path, monkeypatch):
|
||||
kb = make_kb(tmp_path)
|
||||
kb.databases_meta["db"]["metadata"] = {}
|
||||
records = make_file_records({
|
||||
"file-1": {"kb_id": "db", "filename": "alpha.md", "chunk_count": 0, "token_count": 0},
|
||||
"file-2": {"kb_id": "db", "filename": "beta.md", "chunk_count": 1, "token_count": 7},
|
||||
"folder-1": {
|
||||
"kb_id": "db",
|
||||
"filename": "folder",
|
||||
"is_folder": True,
|
||||
"chunk_count": 99,
|
||||
"token_count": 99,
|
||||
},
|
||||
})
|
||||
file_repo = FakeFileRepository(records)
|
||||
persisted_kbs = []
|
||||
|
||||
class FakeChunkRepo:
|
||||
async def count_by_file_ids(self, file_ids):
|
||||
assert file_ids == ["file-1", "file-2"]
|
||||
return {"file-1": 2, "file-2": 3}
|
||||
|
||||
async def list_by_file_ids(self, file_ids):
|
||||
assert file_ids == ["file-1"]
|
||||
return [
|
||||
types.SimpleNamespace(file_id="file-1", content="alpha beta"),
|
||||
types.SimpleNamespace(file_id="file-1", content="中文"),
|
||||
]
|
||||
|
||||
async def persist_kb(kb_id):
|
||||
persisted_kbs.append((kb_id, dict(kb.databases_meta[kb_id]["metadata"])))
|
||||
|
||||
monkeypatch.setattr("yuxi.repositories.knowledge_chunk_repository.KnowledgeChunkRepository", FakeChunkRepo)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
|
||||
lambda: file_repo,
|
||||
)
|
||||
kb._persist_kb = persist_kb
|
||||
|
||||
result = await kb.repair_missing_file_stats("db")
|
||||
|
||||
expected_token_count = count_tokens("alpha beta") + count_tokens("中文")
|
||||
expected_stats = {"file_count": 2, "chunk_count": 5, "token_count": expected_token_count + 7}
|
||||
assert records["file-1"].chunk_count == 2
|
||||
assert records["file-1"].token_count == expected_token_count
|
||||
assert records["file-2"].chunk_count == 3
|
||||
assert records["file-2"].token_count == 7
|
||||
for key, value in expected_stats.items():
|
||||
assert result["stats"][key] == value
|
||||
assert result["scanned_token_files"] == 1
|
||||
assert result["updated_chunk_files"] == 2
|
||||
assert result["updated_token_files"] == 1
|
||||
assert {file_id for file_id, _, _ in file_repo.update_calls} == {"file-1", "file-2"}
|
||||
persisted_stats = persisted_kbs[0][1]["stats"]
|
||||
for key, value in expected_stats.items():
|
||||
assert persisted_stats[key] == value
|
||||
|
||||
|
||||
async def test_repair_missing_file_stats_skips_unindexed_files(tmp_path, monkeypatch):
|
||||
kb = make_kb(tmp_path)
|
||||
kb.databases_meta["db"]["metadata"] = {}
|
||||
records = make_file_records({
|
||||
"file-indexed": {
|
||||
"kb_id": "db",
|
||||
"filename": "alpha.md",
|
||||
"status": "indexed",
|
||||
"chunk_count": 0,
|
||||
"token_count": 0,
|
||||
},
|
||||
"file-uploaded": {
|
||||
"kb_id": "db",
|
||||
"filename": "beta.md",
|
||||
"status": "uploaded",
|
||||
"chunk_count": 9,
|
||||
"token_count": 90,
|
||||
},
|
||||
"file-parsed": {
|
||||
"kb_id": "db",
|
||||
"filename": "gamma.md",
|
||||
"status": "parsed",
|
||||
"chunk_count": 3,
|
||||
"token_count": 30,
|
||||
},
|
||||
})
|
||||
file_repo = FakeFileRepository(records)
|
||||
|
||||
class FakeChunkRepo:
|
||||
async def count_by_file_ids(self, file_ids):
|
||||
assert file_ids == ["file-indexed"]
|
||||
return {"file-indexed": 2}
|
||||
|
||||
async def list_by_file_ids(self, file_ids):
|
||||
assert file_ids == ["file-indexed"]
|
||||
return [types.SimpleNamespace(file_id="file-indexed", content="alpha beta")]
|
||||
|
||||
async def persist_kb(kb_id):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr("yuxi.repositories.knowledge_chunk_repository.KnowledgeChunkRepository", FakeChunkRepo)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
|
||||
lambda: file_repo,
|
||||
)
|
||||
kb._persist_kb = persist_kb
|
||||
|
||||
result = await kb.repair_missing_file_stats("db")
|
||||
|
||||
expected_token_count = count_tokens("alpha beta")
|
||||
assert records["file-indexed"].chunk_count == 2
|
||||
assert records["file-indexed"].token_count == expected_token_count
|
||||
assert records["file-uploaded"].chunk_count == 0
|
||||
assert records["file-uploaded"].token_count == 0
|
||||
assert records["file-parsed"].chunk_count == 0
|
||||
assert records["file-parsed"].token_count == 0
|
||||
assert result["stats"]["file_count"] == 3
|
||||
assert result["stats"]["chunk_count"] == 2
|
||||
assert result["stats"]["token_count"] == expected_token_count
|
||||
assert result["scanned_files"] == 3
|
||||
assert result["scanned_indexed_files"] == 1
|
||||
assert result["skipped_unindexed_files"] == 2
|
||||
assert result["updated_files"] == 3
|
||||
assert {file_id for file_id, _, _ in file_repo.update_calls} == {
|
||||
"file-indexed",
|
||||
"file-uploaded",
|
||||
"file-parsed",
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.knowledge.base import KnowledgeBase
|
||||
|
||||
|
||||
class FakeKnowledgeBase(KnowledgeBase):
|
||||
@property
|
||||
def kb_type(self) -> str:
|
||||
return "fake"
|
||||
|
||||
async def _create_kb_instance(self, kb_id: str, config: dict):
|
||||
return None
|
||||
|
||||
async def _initialize_kb_instance(self, instance) -> None:
|
||||
pass
|
||||
|
||||
async def index_file(self, kb_id: str, file_id: str, operator_id: str | None = None) -> dict:
|
||||
return {}
|
||||
|
||||
async def update_content(self, kb_id: str, file_ids: list[str], params: dict | None = None) -> list[dict]:
|
||||
return []
|
||||
|
||||
async def aquery(self, query_text: str, kb_id: str, **kwargs) -> list[dict]:
|
||||
return []
|
||||
|
||||
def get_query_params_config(self, kb_id: str, **kwargs) -> dict:
|
||||
return {"options": []}
|
||||
|
||||
async def delete_file(self, kb_id: str, file_id: str) -> None:
|
||||
pass
|
||||
|
||||
async def get_file_basic_info(self, kb_id: str, file_id: str) -> dict:
|
||||
return {}
|
||||
|
||||
async def get_file_content(self, kb_id: str, file_id: str) -> dict:
|
||||
return {}
|
||||
|
||||
async def get_file_info(self, kb_id: str, file_id: str) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
def make_file_record(**overrides):
|
||||
data = {
|
||||
"file_id": "file-1",
|
||||
"kb_id": "db",
|
||||
"parent_id": None,
|
||||
"filename": "demo.md",
|
||||
"file_type": "md",
|
||||
"path": "minio://knowledgebases/db/upload/demo.md",
|
||||
"markdown_file": None,
|
||||
"status": "uploaded",
|
||||
"content_hash": "hash",
|
||||
"file_size": 123,
|
||||
"chunk_count": 0,
|
||||
"token_count": 0,
|
||||
"content_type": "file",
|
||||
"processing_params": {"ocr_engine": "disable"},
|
||||
"is_folder": False,
|
||||
"error_message": None,
|
||||
"created_by": "user",
|
||||
"updated_by": None,
|
||||
"created_at": None,
|
||||
"updated_at": None,
|
||||
"original_filename": None,
|
||||
"minio_url": None,
|
||||
}
|
||||
data.update(overrides)
|
||||
return SimpleNamespace(**data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_metadata_does_not_load_file_records(monkeypatch, tmp_path):
|
||||
kb = FakeKnowledgeBase(str(tmp_path))
|
||||
|
||||
class FakeKbRepo:
|
||||
async def get_all(self):
|
||||
return [
|
||||
SimpleNamespace(
|
||||
kb_id="db",
|
||||
name="Docs",
|
||||
description="",
|
||||
kb_type="fake",
|
||||
embedding_model_spec=None,
|
||||
llm_model_spec=None,
|
||||
query_params=None,
|
||||
additional_params={"chunk_preset_id": "general"},
|
||||
created_at=None,
|
||||
)
|
||||
]
|
||||
|
||||
def fail_resolve_processing_params(*args, **kwargs):
|
||||
raise AssertionError("startup metadata loading should not normalize every file")
|
||||
|
||||
monkeypatch.setattr("yuxi.repositories.knowledge_base_repository.KnowledgeBaseRepository", lambda: FakeKbRepo())
|
||||
monkeypatch.setattr("yuxi.knowledge.base.resolve_processing_params", fail_resolve_processing_params)
|
||||
|
||||
await kb._load_metadata()
|
||||
|
||||
assert kb._metadata_loaded is True
|
||||
assert set(kb.databases_meta) == {"db"}
|
||||
assert not hasattr(kb, "files_meta")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_file_params_lazy_loads_single_file(monkeypatch, tmp_path):
|
||||
kb = FakeKnowledgeBase(str(tmp_path))
|
||||
kb.databases_meta["db"] = {"metadata": {"chunk_preset_id": "general"}}
|
||||
|
||||
class FakeFileRepo:
|
||||
def __init__(self):
|
||||
self.updated = []
|
||||
|
||||
async def get_by_file_id(self, file_id: str):
|
||||
assert file_id == "file-1"
|
||||
return make_file_record()
|
||||
|
||||
async def update_fields(self, *, file_id: str, kb_id: str | None = None, data: dict):
|
||||
self.updated.append((file_id, kb_id, data))
|
||||
return make_file_record(
|
||||
processing_params=data["processing_params"],
|
||||
updated_by=data.get("updated_by"),
|
||||
)
|
||||
|
||||
file_repo = FakeFileRepo()
|
||||
monkeypatch.setattr("yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository", lambda: file_repo)
|
||||
|
||||
await kb.update_file_params("db", "file-1", {"chunk_preset_id": "qa"}, operator_id="user-2")
|
||||
|
||||
assert not hasattr(kb, "files_meta")
|
||||
assert len(file_repo.updated) == 1
|
||||
file_id, kb_id, update_data = file_repo.updated[0]
|
||||
assert file_id == "file-1"
|
||||
assert kb_id == "db"
|
||||
assert update_data["processing_params"]["chunk_preset_id"] == "qa"
|
||||
assert update_data["processing_params"]["ocr_engine"] == "disable"
|
||||
assert update_data["updated_by"] == "user-2"
|
||||
@@ -0,0 +1,51 @@
|
||||
from yuxi.knowledge.graphs.milvus_graph_service import MilvusGraphService
|
||||
from yuxi.knowledge.implementations.milvus import MilvusKB, _retrieval_config_options
|
||||
|
||||
|
||||
def test_milvus_retrieval_config_exposes_graph_and_dependencies():
|
||||
options = _retrieval_config_options()
|
||||
by_key = {option["key"]: option for option in options}
|
||||
|
||||
assert by_key["use_graph_retrieval"]["default"] is False
|
||||
assert by_key["graph_max_nodes"]["default"] == 10000
|
||||
assert by_key["graph_max_nodes"]["depend_on"] == ("use_graph_retrieval", True)
|
||||
assert by_key["graph_top_k"]["depend_on"] == ("use_graph_retrieval", True)
|
||||
assert by_key["reranker_model"]["depend_on"] == ("use_reranker", True)
|
||||
|
||||
|
||||
def test_graph_ppr_ranks_chunk_nodes_from_seed_entities():
|
||||
subgraph = {
|
||||
"nodes": [
|
||||
{"id": "e1", "type": "Entity", "properties": {"entity_id": "seed"}},
|
||||
{"id": "c1", "type": "Chunk", "properties": {"chunk_id": "chunk_a"}},
|
||||
{"id": "e2", "type": "Entity", "properties": {"entity_id": "other"}},
|
||||
{"id": "c2", "type": "Chunk", "properties": {"chunk_id": "chunk_b"}},
|
||||
],
|
||||
"edges": [
|
||||
{"source_id": "e1", "target_id": "c1"},
|
||||
{"source_id": "e1", "target_id": "e2"},
|
||||
{"source_id": "e2", "target_id": "c2"},
|
||||
],
|
||||
}
|
||||
|
||||
ranked = MilvusGraphService.rank_chunks_by_ppr(subgraph, {"seed": 1.0}, top_k=2, damping=0.85)
|
||||
|
||||
assert [chunk_id for chunk_id, _ in ranked] == ["chunk_a", "chunk_b"]
|
||||
|
||||
|
||||
def test_rrf_fusion_merges_chunk_and_graph_rankings():
|
||||
kb = object.__new__(MilvusKB)
|
||||
base_chunks = [
|
||||
{"content": "base a", "metadata": {"chunk_id": "a"}, "score": 0.9},
|
||||
{"content": "base b", "metadata": {"chunk_id": "b"}, "score": 0.8},
|
||||
]
|
||||
graph_chunks = [
|
||||
{"content": "graph b", "metadata": {"chunk_id": "b"}, "score": 0.7, "graph_score": 0.7},
|
||||
{"content": "graph c", "metadata": {"chunk_id": "c"}, "score": 0.6, "graph_score": 0.6},
|
||||
]
|
||||
|
||||
fused = kb._fuse_chunk_rankings(base_chunks, graph_chunks, graph_weight=1.0)
|
||||
|
||||
assert [chunk["metadata"]["chunk_id"] for chunk in fused] == ["b", "a", "c"]
|
||||
assert fused[0]["graph_score"] == 0.7
|
||||
assert fused[0]["fusion_sources"] == ["chunk", "graph"]
|
||||
@@ -0,0 +1,118 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from yuxi.knowledge.utils import mindmap_utils as mm
|
||||
|
||||
|
||||
def make_kb(**overrides):
|
||||
data = {
|
||||
"kb_id": "kb_1",
|
||||
"name": "知识库",
|
||||
"mindmap": {"content": "知识库", "children": [{"content": "tracked.pdf", "children": []}]},
|
||||
"mindmap_file_ids": {"tracked": "tracked.pdf"},
|
||||
"mindmap_metadata": {},
|
||||
}
|
||||
data.update(overrides)
|
||||
return SimpleNamespace(**data)
|
||||
|
||||
|
||||
def make_file(file_id: str, filename: str, *, kb_id: str = "kb_1"):
|
||||
return SimpleNamespace(
|
||||
file_id=file_id,
|
||||
kb_id=kb_id,
|
||||
filename=filename,
|
||||
file_type="pdf",
|
||||
status="indexed",
|
||||
is_folder=False,
|
||||
created_at=None,
|
||||
)
|
||||
|
||||
|
||||
class FakeKnowledgeBaseRepository:
|
||||
def __init__(self, kb):
|
||||
self.kb = kb
|
||||
self.updates = []
|
||||
|
||||
async def get_by_kb_id(self, kb_id):
|
||||
return self.kb if kb_id == self.kb.kb_id else None
|
||||
|
||||
async def update(self, kb_id, data):
|
||||
self.updates.append((kb_id, data))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_mindmap_diff_keeps_tracked_file_outside_first_page(monkeypatch):
|
||||
kb_repo = FakeKnowledgeBaseRepository(make_kb())
|
||||
|
||||
class FakeFileRepository:
|
||||
async def list_documents(self, **kwargs):
|
||||
assert kwargs["page_size"] == mm.MINDMAP_FILE_PAGE_SIZE
|
||||
return [make_file("new", "new.pdf")], 100
|
||||
|
||||
async def list_by_file_ids(self, file_ids):
|
||||
assert file_ids == ["tracked"]
|
||||
return [make_file("tracked", "tracked.pdf")]
|
||||
|
||||
monkeypatch.setattr(mm, "KnowledgeBaseRepository", lambda: kb_repo)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
|
||||
FakeFileRepository,
|
||||
)
|
||||
|
||||
result = await mm.get_mindmap_diff("kb_1")
|
||||
|
||||
assert result["removed_file_ids"] == []
|
||||
assert result["unchanged_count"] == 1
|
||||
assert result["added_files"] == [{"file_id": "new", "filename": "new.pdf", "type": "pdf"}]
|
||||
assert result["current_files_truncated"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_database_mindmap_loads_selected_file_ids_directly(monkeypatch):
|
||||
kb_repo = FakeKnowledgeBaseRepository(make_kb(mindmap=None, mindmap_file_ids=None))
|
||||
|
||||
class FakeFileRepository:
|
||||
async def list_documents(self, **kwargs):
|
||||
raise AssertionError("selected file generation should query by file id")
|
||||
|
||||
async def list_by_file_ids(self, file_ids):
|
||||
assert file_ids == ["outside-page"]
|
||||
return [make_file("outside-page", "outside.pdf")]
|
||||
|
||||
class FakeModel:
|
||||
async def call(self, messages, stream):
|
||||
assert "outside.pdf" in messages[1]["content"]
|
||||
return SimpleNamespace(content='{"content":"知识库","children":[{"content":"outside.pdf","children":[]}]}')
|
||||
|
||||
monkeypatch.setattr(mm, "KnowledgeBaseRepository", lambda: kb_repo)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
|
||||
FakeFileRepository,
|
||||
)
|
||||
monkeypatch.setattr(mm, "select_model", lambda model_spec: FakeModel())
|
||||
|
||||
result = await mm.generate_database_mindmap("kb_1", file_ids=["outside-page"])
|
||||
|
||||
assert result["file_count"] == 1
|
||||
assert result["original_file_count"] == 1
|
||||
assert kb_repo.updates[0][1]["mindmap_file_ids"] == {"outside-page": "outside.pdf"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_database_mindmap_rejects_missing_selected_files(monkeypatch):
|
||||
kb_repo = FakeKnowledgeBaseRepository(make_kb(mindmap=None, mindmap_file_ids=None))
|
||||
|
||||
class FakeFileRepository:
|
||||
async def list_by_file_ids(self, file_ids):
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(mm, "KnowledgeBaseRepository", lambda: kb_repo)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository",
|
||||
FakeFileRepository,
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException, match="选择的文件不存在"):
|
||||
await mm.generate_database_mindmap("kb_1", file_ids=["missing"])
|
||||
@@ -0,0 +1,171 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.knowledge.base import KnowledgeBase
|
||||
from yuxi.services.file_preview import MAX_BINARY_PREVIEW_SIZE_BYTES
|
||||
|
||||
|
||||
class FakeKnowledgeBase(KnowledgeBase):
|
||||
@property
|
||||
def kb_type(self) -> str:
|
||||
return "fake"
|
||||
|
||||
async def _create_kb_instance(self, kb_id: str, config: dict):
|
||||
return None
|
||||
|
||||
async def _initialize_kb_instance(self, instance) -> None:
|
||||
pass
|
||||
|
||||
async def index_file(self, kb_id: str, file_id: str, operator_id: str | None = None) -> dict:
|
||||
return {}
|
||||
|
||||
async def update_content(self, kb_id: str, file_ids: list[str], params: dict | None = None) -> list[dict]:
|
||||
return []
|
||||
|
||||
async def aquery(self, query_text: str, kb_id: str, **kwargs) -> list[dict]:
|
||||
return []
|
||||
|
||||
def get_query_params_config(self, kb_id: str, **kwargs) -> dict:
|
||||
return {"options": []}
|
||||
|
||||
async def delete_file(self, kb_id: str, file_id: str) -> None:
|
||||
pass
|
||||
|
||||
async def get_file_basic_info(self, kb_id: str, file_id: str) -> dict:
|
||||
return {}
|
||||
|
||||
async def get_file_content(self, kb_id: str, file_id: str) -> dict:
|
||||
return {}
|
||||
|
||||
async def get_file_info(self, kb_id: str, file_id: str) -> dict:
|
||||
return {}
|
||||
|
||||
async def _save_metadata(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class FakeMinioClient:
|
||||
KB_BUCKETS = {"parsed": "knowledgebases"}
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.objects: dict[tuple[str, str], bytes] = {}
|
||||
|
||||
async def astat_file(self, bucket_name: str, object_name: str) -> int | None:
|
||||
content = self.objects.get((bucket_name, object_name))
|
||||
return len(content) if content is not None else None
|
||||
|
||||
async def adownload_file(self, bucket_name: str, object_name: str) -> bytes:
|
||||
return self.objects[(bucket_name, object_name)]
|
||||
|
||||
async def aupload_file(
|
||||
self,
|
||||
bucket_name: str,
|
||||
object_name: str,
|
||||
data: bytes,
|
||||
content_type: str | None = None,
|
||||
) -> SimpleNamespace:
|
||||
assert content_type == "application/pdf"
|
||||
self.objects[(bucket_name, object_name)] = data
|
||||
return SimpleNamespace(url=f"http://localhost:9000/{bucket_name}/{object_name}")
|
||||
|
||||
|
||||
def make_kb(tmp_path) -> FakeKnowledgeBase:
|
||||
kb = FakeKnowledgeBase(str(tmp_path))
|
||||
kb.databases_meta["db1"] = {"metadata": {}}
|
||||
kb.test_file_meta = {
|
||||
"file_id": "file1",
|
||||
"kb_id": "db1",
|
||||
"filename": "demo.docx",
|
||||
"path": "minio://knowledgebases/db1/upload/demo.docx",
|
||||
"markdown_file": "minio://knowledgebases/db1/parsed/file1.md",
|
||||
"status": "parsed",
|
||||
}
|
||||
|
||||
async def load_file_meta(kb_id: str, file_id: str, *, refresh: bool = False) -> dict:
|
||||
assert kb_id == "db1"
|
||||
assert file_id == "file1"
|
||||
return kb.test_file_meta
|
||||
|
||||
kb._load_file_meta = load_file_meta
|
||||
return kb
|
||||
|
||||
|
||||
def test_office_file_entry_exposes_logical_file_availability(tmp_path) -> None:
|
||||
kb = make_kb(tmp_path)
|
||||
|
||||
entry = kb._knowledge_file_entry("db1", "file1", kb.test_file_meta)
|
||||
|
||||
assert entry["has_original_file"] is True
|
||||
assert entry["has_parsed_markdown"] is True
|
||||
assert "preview_modes" not in entry
|
||||
assert "default_preview_mode" not in entry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_office_pdf_preview_converts_and_caches_pdf(tmp_path, monkeypatch) -> None:
|
||||
kb = make_kb(tmp_path)
|
||||
minio_client = FakeMinioClient()
|
||||
minio_client.objects[("knowledgebases", "db1/upload/demo.docx")] = b"office"
|
||||
minio_client.objects[("knowledgebases", "db1/parsed/file1.md")] = b"# parsed"
|
||||
|
||||
convert_calls = 0
|
||||
|
||||
async def fake_convert(filename: str, content: bytes) -> bytes:
|
||||
nonlocal convert_calls
|
||||
convert_calls += 1
|
||||
assert filename == "demo.docx"
|
||||
assert content == b"office"
|
||||
return b"%PDF-1.4\nconverted"
|
||||
|
||||
monkeypatch.setattr("yuxi.storage.minio.get_minio_client", lambda: minio_client)
|
||||
monkeypatch.setattr("yuxi.knowledge.base.convert_office_to_pdf", fake_convert)
|
||||
|
||||
response = await kb.read_file_preview("db1", "file1")
|
||||
cached_response = await kb.read_file_preview("db1", "file1")
|
||||
|
||||
assert response["preview_type"] == "pdf"
|
||||
assert response["supported"] is True
|
||||
assert response["binary"] is True
|
||||
assert response["content"] == b"%PDF-1.4\nconverted"
|
||||
assert response["media_type"] == "application/pdf"
|
||||
assert cached_response["content"] == b"%PDF-1.4\nconverted"
|
||||
assert minio_client.objects[("knowledgebases", "db1/preview/file1.pdf")] == b"%PDF-1.4\nconverted"
|
||||
assert convert_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_docx_pptx_office_files_do_not_get_pdf_preview(tmp_path, monkeypatch) -> None:
|
||||
kb = make_kb(tmp_path)
|
||||
kb.test_file_meta["filename"] = "demo.xlsx"
|
||||
minio_client = FakeMinioClient()
|
||||
minio_client.objects[("knowledgebases", "db1/upload/demo.docx")] = b"PK\x03\x04excel"
|
||||
monkeypatch.setattr("yuxi.storage.minio.get_minio_client", lambda: minio_client)
|
||||
|
||||
entry = kb._knowledge_file_entry("db1", "file1", kb.test_file_meta)
|
||||
response = await kb.read_file_preview("db1", "file1")
|
||||
|
||||
assert entry["has_original_file"] is True
|
||||
assert entry["has_parsed_markdown"] is True
|
||||
assert response["preview_type"] == "unsupported"
|
||||
assert response["supported"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_file_preview_rejects_large_original_before_download(tmp_path, monkeypatch) -> None:
|
||||
kb = make_kb(tmp_path)
|
||||
kb.test_file_meta["filename"] = "large.pdf"
|
||||
kb.test_file_meta["size"] = MAX_BINARY_PREVIEW_SIZE_BYTES + 1
|
||||
|
||||
async def fail_read(_path: str) -> bytes:
|
||||
raise AssertionError("large preview should not download file content")
|
||||
|
||||
monkeypatch.setattr(kb, "_read_minio_bytes", fail_read)
|
||||
|
||||
response = await kb.read_file_preview("db1", "file1")
|
||||
|
||||
assert response["preview_type"] == "unsupported"
|
||||
assert response["supported"] is False
|
||||
assert response["limit"] == MAX_BINARY_PREVIEW_SIZE_BYTES
|
||||
@@ -0,0 +1,289 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
import yuxi.knowledge.parser.paddleocr_api as paddleocr_api
|
||||
from yuxi.knowledge.parser.base import DocumentParserException
|
||||
from yuxi.knowledge.parser.paddleocr_api import PaddleOCRPPOCRv6Parser, PaddleOCRVLParser
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeResponse:
|
||||
status_code: int
|
||||
json_body: dict[str, Any] | None = None
|
||||
text: str = ""
|
||||
content: bytes = b""
|
||||
headers: dict[str, str] | None = None
|
||||
|
||||
def json(self) -> dict[str, Any]:
|
||||
assert self.json_body is not None
|
||||
return self.json_body
|
||||
|
||||
|
||||
def _build_file(tmp_path: Path, suffix: str = ".png") -> Path:
|
||||
file_path = tmp_path / f"sample{suffix}"
|
||||
file_path.write_bytes(b"fake image")
|
||||
return file_path
|
||||
|
||||
|
||||
def test_paddleocr_vl_submits_model_specific_payload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
file_path = _build_file(tmp_path)
|
||||
submitted: dict[str, Any] = {}
|
||||
|
||||
def fake_post(url, headers=None, data=None, files=None, json=None, timeout=None): # noqa: A002
|
||||
submitted["url"] = url
|
||||
submitted["data"] = data
|
||||
submitted["json"] = json
|
||||
submitted["files"] = files
|
||||
return FakeResponse(200, {"code": 0, "data": {"jobId": "job-vl"}})
|
||||
|
||||
def fake_get(url, headers=None, timeout=None):
|
||||
if url.endswith("/job-vl"):
|
||||
return FakeResponse(200, {"data": {"state": "done", "resultUrl": {"jsonUrl": "https://result.test/vl"}}})
|
||||
row = {
|
||||
"result": {
|
||||
"layoutParsingResults": [
|
||||
{"markdown": {"text": "VL markdown", "images": {}}, "outputImages": {"debug": "ignored"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
return FakeResponse(200, text=json.dumps(row))
|
||||
|
||||
monkeypatch.setattr(paddleocr_api.requests, "post", fake_post)
|
||||
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
|
||||
|
||||
parser = PaddleOCRVLParser(api_token="token")
|
||||
result = parser.process_file(
|
||||
str(file_path),
|
||||
params={
|
||||
"optional_payload": {
|
||||
"useChartRecognition": True,
|
||||
"useTextlineOrientation": True,
|
||||
"ignored": True,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert result == "VL markdown"
|
||||
assert submitted["data"]["model"] == "PaddleOCR-VL-1.6"
|
||||
optional_payload = json.loads(submitted["data"]["optionalPayload"])
|
||||
assert optional_payload == {
|
||||
"useDocOrientationClassify": False,
|
||||
"useDocUnwarping": False,
|
||||
"useChartRecognition": True,
|
||||
}
|
||||
|
||||
|
||||
def test_paddleocr_pp_ocrv6_submits_model_specific_payload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
file_path = _build_file(tmp_path)
|
||||
submitted: dict[str, Any] = {}
|
||||
|
||||
def fake_post(url, headers=None, data=None, files=None, json=None, timeout=None): # noqa: A002
|
||||
submitted["data"] = data
|
||||
return FakeResponse(200, {"data": {"jobId": "job-ocr"}})
|
||||
|
||||
def fake_get(url, headers=None, timeout=None):
|
||||
if url.endswith("/job-ocr"):
|
||||
return FakeResponse(200, {"data": {"state": "done", "resultUrl": {"jsonUrl": "https://result.test/ocr"}}})
|
||||
row = {
|
||||
"result": {
|
||||
"ocrResults": [
|
||||
{"prunedResult": {"rec_texts": ["PaddleOCR API Test", "", "Invoice total: 123.45"]}}
|
||||
]
|
||||
}
|
||||
}
|
||||
return FakeResponse(200, text=json.dumps(row))
|
||||
|
||||
monkeypatch.setattr(paddleocr_api.requests, "post", fake_post)
|
||||
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
|
||||
|
||||
parser = PaddleOCRPPOCRv6Parser(api_token="token")
|
||||
result = parser.process_file(
|
||||
str(file_path),
|
||||
params={
|
||||
"optional_payload": {
|
||||
"useTextlineOrientation": True,
|
||||
"useChartRecognition": True,
|
||||
"ignored": True,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert result == "PaddleOCR API Test\nInvoice total: 123.45"
|
||||
assert submitted["data"]["model"] == "PP-OCRv6"
|
||||
optional_payload = json.loads(submitted["data"]["optionalPayload"])
|
||||
assert optional_payload == {
|
||||
"useDocOrientationClassify": False,
|
||||
"useDocUnwarping": False,
|
||||
"useTextlineOrientation": True,
|
||||
}
|
||||
|
||||
|
||||
def test_paddleocr_url_input_uses_json_payload(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
submitted: dict[str, Any] = {}
|
||||
|
||||
def fake_post(url, headers=None, data=None, files=None, json=None, timeout=None): # noqa: A002
|
||||
submitted["headers"] = headers
|
||||
submitted["data"] = data
|
||||
submitted["files"] = files
|
||||
submitted["json"] = json
|
||||
return FakeResponse(200, {"data": {"jobId": "job-url"}})
|
||||
|
||||
def fake_get(url, headers=None, timeout=None):
|
||||
if url.endswith("/job-url"):
|
||||
return FakeResponse(200, {"data": {"state": "done", "resultUrl": {"jsonUrl": "https://result.test/url"}}})
|
||||
row = {"result": {"ocrResults": [{"prunedResult": {"rec_texts": ["url text"]}}]}}
|
||||
return FakeResponse(200, text=json.dumps(row))
|
||||
|
||||
monkeypatch.setattr(paddleocr_api.requests, "post", fake_post)
|
||||
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
|
||||
|
||||
parser = PaddleOCRPPOCRv6Parser(api_token="token")
|
||||
result = parser.process_file("https://example.test/file.png?signature=abc")
|
||||
|
||||
assert result == "url text"
|
||||
assert submitted["headers"]["Content-Type"] == "application/json"
|
||||
assert submitted["data"] is None
|
||||
assert submitted["files"] is None
|
||||
assert submitted["json"]["fileUrl"] == "https://example.test/file.png?signature=abc"
|
||||
assert submitted["json"]["model"] == "PP-OCRv6"
|
||||
|
||||
|
||||
def test_paddleocr_poll_handles_pending_running_and_done(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
file_path = _build_file(tmp_path)
|
||||
states = iter(["pending", "running", "done"])
|
||||
sleep_calls: list[float] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
paddleocr_api.requests,
|
||||
"post",
|
||||
lambda *args, **kwargs: FakeResponse(200, {"data": {"jobId": "job-poll"}}),
|
||||
)
|
||||
|
||||
def fake_get(url, headers=None, timeout=None):
|
||||
if url.endswith("/job-poll"):
|
||||
state = next(states)
|
||||
data: dict[str, Any] = {"state": state}
|
||||
if state == "done":
|
||||
data["resultUrl"] = {"jsonUrl": "https://result.test/poll"}
|
||||
return FakeResponse(200, {"data": data})
|
||||
row = {"result": {"ocrResults": [{"prunedResult": {"rec_texts": ["done text"]}}]}}
|
||||
return FakeResponse(200, text=json.dumps(row))
|
||||
|
||||
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
|
||||
monkeypatch.setattr(paddleocr_api.time, "sleep", lambda seconds: sleep_calls.append(seconds))
|
||||
|
||||
parser = PaddleOCRPPOCRv6Parser(api_token="token")
|
||||
result = parser.process_file(str(file_path), params={"poll_interval_seconds": 0.25})
|
||||
|
||||
assert result == "done text"
|
||||
assert sleep_calls == [0.25, 0.25]
|
||||
|
||||
|
||||
def test_paddleocr_failed_job_raises_parser_exception(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
file_path = _build_file(tmp_path)
|
||||
|
||||
monkeypatch.setattr(
|
||||
paddleocr_api.requests,
|
||||
"post",
|
||||
lambda *args, **kwargs: FakeResponse(200, {"data": {"jobId": "job-failed"}}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
paddleocr_api.requests,
|
||||
"get",
|
||||
lambda *args, **kwargs: FakeResponse(
|
||||
200,
|
||||
{"data": {"state": "failed", "errorMsg": "quota exceeded"}},
|
||||
),
|
||||
)
|
||||
|
||||
parser = PaddleOCRVLParser(api_token="token")
|
||||
with pytest.raises(DocumentParserException, match="quota exceeded"):
|
||||
parser.process_file(str(file_path))
|
||||
|
||||
|
||||
def test_paddleocr_missing_token_health_and_parse_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
file_path = _build_file(tmp_path)
|
||||
monkeypatch.delenv("PADDLEOCR_API_TOKEN", raising=False)
|
||||
parser = PaddleOCRVLParser()
|
||||
|
||||
health = parser.check_health()
|
||||
|
||||
assert health["status"] == "unavailable"
|
||||
assert "PADDLEOCR_API_TOKEN" in health["message"]
|
||||
with pytest.raises(DocumentParserException, match="PADDLEOCR_API_TOKEN"):
|
||||
parser.process_file(str(file_path))
|
||||
|
||||
|
||||
def test_paddleocr_configured_token_health_does_not_submit_job() -> None:
|
||||
parser = PaddleOCRVLParser(api_token="token")
|
||||
|
||||
health = parser.check_health()
|
||||
|
||||
assert health["status"] == "configured"
|
||||
assert "解析时验证" in health["message"]
|
||||
assert health["details"]["model"] == "PaddleOCR-VL-1.6"
|
||||
|
||||
|
||||
def test_paddleocr_vl_uploads_markdown_images(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
file_path = _build_file(tmp_path)
|
||||
uploaded: dict[str, Any] = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
paddleocr_api.requests,
|
||||
"post",
|
||||
lambda *args, **kwargs: FakeResponse(200, {"data": {"jobId": "job-images"}}),
|
||||
)
|
||||
|
||||
def fake_get(url, headers=None, timeout=None):
|
||||
if url.endswith("/job-images"):
|
||||
return FakeResponse(
|
||||
200,
|
||||
{"data": {"state": "done", "resultUrl": {"jsonUrl": "https://result.test/images"}}},
|
||||
)
|
||||
if url == "https://image.test/table.png":
|
||||
return FakeResponse(200, content=b"image-bytes", headers={"Content-Type": "image/png"})
|
||||
row = {
|
||||
"result": {
|
||||
"layoutParsingResults": [
|
||||
{
|
||||
"markdown": {
|
||||
"text": "before  after",
|
||||
"images": {"images/table.png": "https://image.test/table.png"},
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
return FakeResponse(200, text=json.dumps(row))
|
||||
|
||||
class FakeMinioClient:
|
||||
def ensure_bucket_exists(self, bucket_name):
|
||||
uploaded["bucket_name"] = bucket_name
|
||||
|
||||
def upload_file(self, bucket_name, object_name, data):
|
||||
uploaded["object_name"] = object_name
|
||||
uploaded["data"] = data
|
||||
return type("UploadResult", (), {"url": "minio://public/kb/table.png"})()
|
||||
|
||||
monkeypatch.setattr(paddleocr_api.requests, "get", fake_get)
|
||||
monkeypatch.setattr(paddleocr_api, "get_minio_client", lambda: FakeMinioClient())
|
||||
monkeypatch.setattr(paddleocr_api.time, "time", lambda: 1.0)
|
||||
|
||||
parser = PaddleOCRVLParser(api_token="token")
|
||||
result = parser.process_file(
|
||||
str(file_path),
|
||||
params={"image_bucket": "public", "image_prefix": "kb/images"},
|
||||
)
|
||||
|
||||
assert result == "before  after"
|
||||
assert uploaded == {
|
||||
"bucket_name": "public",
|
||||
"object_name": "kb/images/1000000_table.png",
|
||||
"data": b"image-bytes",
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import fitz
|
||||
import pandas as pd
|
||||
import pytest
|
||||
import yuxi.knowledge.parser.unified as parser_unified
|
||||
from docx import Document
|
||||
from PIL import Image
|
||||
|
||||
from yuxi.knowledge.parser import Parser
|
||||
from yuxi.knowledge.parser.factory import DocumentProcessorFactory
|
||||
|
||||
DATA_DIR = Path(__file__).resolve().parents[2] / "data"
|
||||
|
||||
|
||||
def _build_pdf(file_path: Path, text: str) -> None:
|
||||
doc = fitz.open()
|
||||
page = doc.new_page()
|
||||
page.insert_text((72, 72), text)
|
||||
doc.save(str(file_path))
|
||||
doc.close()
|
||||
|
||||
|
||||
def _build_docx(file_path: Path, text: str) -> None:
|
||||
document = Document()
|
||||
document.add_paragraph(text)
|
||||
document.save(str(file_path))
|
||||
|
||||
|
||||
def _build_png(file_path: Path) -> None:
|
||||
image = Image.new("RGB", (120, 80), "white")
|
||||
image.save(str(file_path))
|
||||
|
||||
|
||||
def test_parser_parse_pdf_file_returns_markdown_text(tmp_path: Path):
|
||||
file_path = tmp_path / "parser_test.pdf"
|
||||
_build_pdf(file_path, "Parser PDF content")
|
||||
|
||||
markdown = Parser.parse(str(file_path), params={"ocr_engine": "disable"})
|
||||
|
||||
assert isinstance(markdown, str)
|
||||
assert "Parser" in markdown
|
||||
assert "content" in markdown
|
||||
assert len(markdown.strip()) > 0
|
||||
|
||||
|
||||
def test_parser_parse_docx_file_returns_markdown_text(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
file_path = tmp_path / "parser_test.docx"
|
||||
_build_docx(file_path, "Parser DOCX content")
|
||||
|
||||
# 避免测试依赖 docling 行为,直接验证统一 parser 可回退到 python-docx。
|
||||
def _raise_docling_error(*args, **kwargs):
|
||||
raise RuntimeError("force fallback to python-docx")
|
||||
|
||||
monkeypatch.setattr(parser_unified, "_convert_with_docling", _raise_docling_error)
|
||||
|
||||
markdown = Parser.parse(str(file_path))
|
||||
|
||||
assert isinstance(markdown, str)
|
||||
assert "Parser DOCX content" in markdown
|
||||
assert len(markdown.strip()) > 0
|
||||
|
||||
|
||||
def test_convert_csv_to_markdown_preserves_column_dtypes(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
file_path = tmp_path / "parser_test.csv"
|
||||
file_path.write_text("id,score\n9007199254740993,2.5\n", encoding="utf-8")
|
||||
captured_dtypes: list[dict[str, object]] = []
|
||||
original_to_markdown = pd.DataFrame.to_markdown
|
||||
|
||||
def _capture_dtypes(dataframe: pd.DataFrame, *args, **kwargs) -> str:
|
||||
captured_dtypes.append(dataframe.dtypes.to_dict())
|
||||
return original_to_markdown(dataframe, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(pd.DataFrame, "to_markdown", _capture_dtypes)
|
||||
|
||||
markdown = parser_unified._convert_csv_to_markdown(file_path)
|
||||
|
||||
assert markdown
|
||||
assert str(captured_dtypes[0]["id"]) == "int64"
|
||||
|
||||
|
||||
def test_convert_with_docling_reinserts_image_links_in_document_order(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
file_path = tmp_path / "parser_test.docx"
|
||||
file_path.write_bytes(b"fake docx")
|
||||
first_image = base64.b64encode(b"first image").decode()
|
||||
second_image = base64.b64encode(b"second image").decode()
|
||||
fake_doc = SimpleNamespace(
|
||||
pictures=[
|
||||
SimpleNamespace(image=SimpleNamespace(uri=f"data:image/png;base64,{first_image}")),
|
||||
SimpleNamespace(image=SimpleNamespace(uri="https://example.test/remote.png")),
|
||||
SimpleNamespace(image=SimpleNamespace(uri=f"data:image/png;base64,{second_image}")),
|
||||
],
|
||||
export_to_markdown=lambda: "before\n<!-- image -->\nremote\n<!-- image -->\nbetween\n<!-- image -->\nafter",
|
||||
)
|
||||
fake_result = SimpleNamespace(status=SimpleNamespace(name="SUCCESS"), document=fake_doc)
|
||||
uploaded_images: list[bytes] = []
|
||||
|
||||
class FakeConverter:
|
||||
def convert(self, path: Path):
|
||||
assert path == file_path
|
||||
return fake_result
|
||||
|
||||
def _fake_upload_image_to_minio(image_data, filename, bucket_name, object_prefix):
|
||||
uploaded_images.append(image_data)
|
||||
return f"https://example.test/{len(uploaded_images)}.png"
|
||||
|
||||
monkeypatch.setattr(parser_unified, "_get_docling_converter", lambda: FakeConverter())
|
||||
monkeypatch.setattr(parser_unified, "_upload_image_to_minio", _fake_upload_image_to_minio)
|
||||
image_timestamps = iter([1.0, 2.0])
|
||||
monkeypatch.setattr(parser_unified.time, "time", lambda: next(image_timestamps))
|
||||
|
||||
markdown = parser_unified._convert_with_docling(file_path)
|
||||
|
||||
assert uploaded_images == [b"first image", b"second image"]
|
||||
assert markdown == (
|
||||
"before\n"
|
||||
"\n"
|
||||
"remote\n"
|
||||
"\n"
|
||||
"between\n"
|
||||
"\n"
|
||||
"after"
|
||||
)
|
||||
|
||||
|
||||
def test_convert_with_docling_keeps_image_placeholder_when_upload_fails(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
file_path = tmp_path / "parser_test.docx"
|
||||
file_path.write_bytes(b"fake docx")
|
||||
image = base64.b64encode(b"image data").decode()
|
||||
fake_doc = SimpleNamespace(
|
||||
pictures=[SimpleNamespace(image=SimpleNamespace(uri=f"data:image/png;base64,{image}"))],
|
||||
export_to_markdown=lambda: "before\n<!-- image -->\nafter",
|
||||
)
|
||||
fake_result = SimpleNamespace(status=SimpleNamespace(name="SUCCESS"), document=fake_doc)
|
||||
|
||||
class FakeConverter:
|
||||
def convert(self, path: Path):
|
||||
assert path == file_path
|
||||
return fake_result
|
||||
|
||||
def _raise_upload_error(*args, **kwargs):
|
||||
raise RuntimeError("upload failed")
|
||||
|
||||
monkeypatch.setattr(parser_unified, "_get_docling_converter", lambda: FakeConverter())
|
||||
monkeypatch.setattr(parser_unified, "_upload_image_to_minio", _raise_upload_error)
|
||||
monkeypatch.setattr(parser_unified.time, "time", lambda: 1.0)
|
||||
|
||||
markdown = parser_unified._convert_with_docling(file_path)
|
||||
|
||||
assert markdown == "before\n[图片: image_1000000.png]\nafter"
|
||||
|
||||
|
||||
def test_parser_parse_png_file_returns_markdown_text_with_mocked_ocr(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
file_path = tmp_path / "parser_test.png"
|
||||
_build_png(file_path)
|
||||
|
||||
async def _fake_parse_image_async(file, params=None):
|
||||
return "Parser PNG content"
|
||||
|
||||
monkeypatch.setattr(parser_unified, "parse_image_async", _fake_parse_image_async)
|
||||
|
||||
markdown = Parser.parse(str(file_path), params={"ocr_engine": "rapid_ocr"})
|
||||
|
||||
assert isinstance(markdown, str)
|
||||
assert "Parser PNG content" in markdown
|
||||
assert len(markdown.strip()) > 0
|
||||
|
||||
|
||||
def test_parse_image_uses_ocr_engine_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
file_path = tmp_path / "parser_test.png"
|
||||
_build_png(file_path)
|
||||
captured = {}
|
||||
|
||||
def _fake_process_file(processor_type, file, params=None):
|
||||
captured["processor_type"] = processor_type
|
||||
captured["file"] = file
|
||||
captured["params"] = params
|
||||
return "OCR content"
|
||||
|
||||
monkeypatch.setattr(DocumentProcessorFactory, "process_file", _fake_process_file)
|
||||
|
||||
result = parser_unified.parse_image(
|
||||
str(file_path),
|
||||
params={
|
||||
"ocr_engine": "mineru_ocr",
|
||||
"backend": "old-backend",
|
||||
"ocr_engine_config": {"backend": "pipeline", "formula_enable": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert result == "OCR content"
|
||||
assert captured["processor_type"] == "mineru_ocr"
|
||||
assert captured["file"] == str(file_path)
|
||||
assert captured["params"]["backend"] == "pipeline"
|
||||
assert captured["params"]["formula_enable"] is False
|
||||
|
||||
|
||||
def test_parse_image_ignores_enable_ocr(tmp_path: Path) -> None:
|
||||
file_path = tmp_path / "parser_test.png"
|
||||
_build_png(file_path)
|
||||
|
||||
with pytest.raises(ValueError, match="必须启用OCR"):
|
||||
parser_unified.parse_image(str(file_path), params={"ocr_engine": "disable", "enable_ocr": "rapid_ocr"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parser_aparse_pdf_file_returns_markdown_text(tmp_path: Path):
|
||||
file_path = tmp_path / "parser_test_async.pdf"
|
||||
_build_pdf(file_path, "Async Parser PDF content")
|
||||
|
||||
markdown = await Parser.aparse(str(file_path), params={"ocr_engine": "disable"})
|
||||
|
||||
assert isinstance(markdown, str)
|
||||
assert "Async" in markdown
|
||||
assert "content" in markdown
|
||||
assert len(markdown.strip()) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parser_aparse_docx_does_not_block_event_loop(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
file_path = tmp_path / "parser_test_async.docx"
|
||||
file_path.write_bytes(b"fake docx")
|
||||
completion_order: list[str] = []
|
||||
|
||||
def _slow_docling_conversion(*args, **kwargs) -> str:
|
||||
time.sleep(0.1)
|
||||
return "Async DOCX content"
|
||||
|
||||
async def _parse_document() -> None:
|
||||
await Parser.aparse(str(file_path))
|
||||
completion_order.append("parse")
|
||||
|
||||
async def _record_event_loop_progress() -> None:
|
||||
await asyncio.sleep(0.01)
|
||||
completion_order.append("event_loop")
|
||||
|
||||
monkeypatch.setattr(parser_unified, "_convert_with_docling", _slow_docling_conversion)
|
||||
|
||||
await asyncio.gather(_parse_document(), _record_event_loop_progress())
|
||||
|
||||
assert completion_order == ["event_loop", "parse"]
|
||||
|
||||
|
||||
def test_parse_pdf_uses_config_default_ocr_when_engine_missing(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import yuxi
|
||||
|
||||
file_path = tmp_path / "parser_test.pdf"
|
||||
_build_pdf(file_path, "Parser PDF content")
|
||||
captured = {}
|
||||
|
||||
def _fake_process_file(processor_type, file, params=None):
|
||||
captured["processor_type"] = processor_type
|
||||
captured["file"] = file
|
||||
captured["params"] = params
|
||||
return "default OCR content"
|
||||
|
||||
monkeypatch.setattr(yuxi.config, "default_ocr_engine", "mineru_ocr")
|
||||
monkeypatch.setattr(DocumentProcessorFactory, "process_file", _fake_process_file)
|
||||
|
||||
result = parser_unified.parse_pdf(str(file_path), params={})
|
||||
|
||||
assert result == "default OCR content"
|
||||
assert captured["processor_type"] == "mineru_ocr"
|
||||
assert captured["file"] == str(file_path)
|
||||
|
||||
|
||||
def test_parse_pdf_keeps_explicit_disable_when_default_ocr_enabled(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import yuxi
|
||||
|
||||
file_path = tmp_path / "parser_test.pdf"
|
||||
_build_pdf(file_path, "Parser PDF content")
|
||||
monkeypatch.setattr(yuxi.config, "default_ocr_engine", "mineru_ocr")
|
||||
|
||||
result = parser_unified.parse_pdf(str(file_path), params={"ocr_engine": "disable"})
|
||||
|
||||
assert "Parser PDF content" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parser_aparse_image_file_with_mineru_when_available():
|
||||
file_path = DATA_DIR / "测试图片.png"
|
||||
assert file_path.exists(), f"测试文件不存在: {file_path}"
|
||||
|
||||
health = await asyncio.to_thread(DocumentProcessorFactory.check_health, "mineru_ocr")
|
||||
if health.get("status") != "healthy":
|
||||
pytest.skip(f"mineru_ocr 不可用: {health.get('message', 'unknown')}")
|
||||
|
||||
markdown = await Parser.aparse(
|
||||
str(file_path),
|
||||
params={"ocr_engine": "mineru_ocr", "backend": "pipeline"},
|
||||
)
|
||||
|
||||
assert isinstance(markdown, str)
|
||||
assert len(markdown) > 100
|
||||
assert len(markdown.strip()) > 0
|
||||
@@ -0,0 +1,108 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from yuxi.knowledge.utils import sample_question_utils as sq
|
||||
|
||||
|
||||
def test_parse_sample_questions_content_strips_json_fence():
|
||||
questions = sq.parse_sample_questions_content('```json\n{"questions": ["什么是测试?"]}\n```')
|
||||
|
||||
assert questions == ["什么是测试?"]
|
||||
|
||||
|
||||
def test_parse_sample_questions_content_rejects_invalid_payload():
|
||||
with pytest.raises(ValueError, match="问题格式"):
|
||||
sq.parse_sample_questions_content('{"items": []}')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_database_sample_questions_rejects_empty_files(monkeypatch):
|
||||
class FakeKnowledgeBase:
|
||||
async def get_database_info(self, kb_id: str, include_files: bool = False) -> dict:
|
||||
return {"name": "空知识库", "kb_type": "milvus", "files": {}}
|
||||
|
||||
monkeypatch.setattr(sq, "knowledge_base", FakeKnowledgeBase())
|
||||
monkeypatch.setattr(
|
||||
sq.KnowledgeBaseFactory,
|
||||
"get_kb_class",
|
||||
lambda _kb_type: SimpleNamespace(supports_documents=True),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await sq.generate_database_sample_questions("kb_1")
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "没有文件" in exc_info.value.detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_database_sample_questions_saves_and_returns_questions(monkeypatch):
|
||||
saved: dict = {}
|
||||
|
||||
class FakeKnowledgeBase:
|
||||
async def get_database_info(self, kb_id: str, include_files: bool = False) -> dict:
|
||||
return {
|
||||
"name": "测试知识库",
|
||||
"kb_type": "milvus",
|
||||
"files": {"file_1": {"filename": "demo.md", "file_type": "md"}},
|
||||
}
|
||||
|
||||
class FakeModel:
|
||||
async def call(self, messages, stream: bool = False):
|
||||
assert messages[0]["role"] == "system"
|
||||
assert "demo.md" in messages[1]["content"]
|
||||
return SimpleNamespace(content='{"questions": ["如何使用 demo?"]}')
|
||||
|
||||
class FakeRepository:
|
||||
async def update(self, kb_id: str, data: dict) -> None:
|
||||
saved[kb_id] = data["sample_questions"]
|
||||
|
||||
async def get_by_kb_id(self, kb_id: str):
|
||||
return SimpleNamespace(name="测试知识库", sample_questions=saved.get(kb_id))
|
||||
|
||||
monkeypatch.setattr(sq, "knowledge_base", FakeKnowledgeBase())
|
||||
monkeypatch.setattr(
|
||||
sq.KnowledgeBaseFactory,
|
||||
"get_kb_class",
|
||||
lambda _kb_type: SimpleNamespace(supports_documents=True),
|
||||
)
|
||||
monkeypatch.setattr(sq, "select_model", lambda model_spec: FakeModel())
|
||||
monkeypatch.setattr(sq, "KnowledgeBaseRepository", lambda: FakeRepository())
|
||||
|
||||
generated = await sq.generate_database_sample_questions("kb_1", count=1)
|
||||
stored = await sq.get_database_sample_questions("kb_1")
|
||||
|
||||
assert generated["questions"] == ["如何使用 demo?"]
|
||||
assert generated["count"] == 1
|
||||
assert stored["questions"] == ["如何使用 demo?"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_database_sample_questions_maps_invalid_json(monkeypatch):
|
||||
class FakeKnowledgeBase:
|
||||
async def get_database_info(self, kb_id: str, include_files: bool = False) -> dict:
|
||||
return {
|
||||
"name": "测试知识库",
|
||||
"kb_type": "milvus",
|
||||
"files": {"file_1": {"filename": "demo.md", "file_type": "md"}},
|
||||
}
|
||||
|
||||
class FakeModel:
|
||||
async def call(self, messages, stream: bool = False):
|
||||
return SimpleNamespace(content="not json")
|
||||
|
||||
monkeypatch.setattr(sq, "knowledge_base", FakeKnowledgeBase())
|
||||
monkeypatch.setattr(
|
||||
sq.KnowledgeBaseFactory,
|
||||
"get_kb_class",
|
||||
lambda _kb_type: SimpleNamespace(supports_documents=True),
|
||||
)
|
||||
monkeypatch.setattr(sq, "select_model", lambda model_spec: FakeModel())
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await sq.generate_database_sample_questions("kb_1")
|
||||
|
||||
assert exc_info.value.status_code == 500
|
||||
assert "AI返回格式错误" in exc_info.value.detail
|
||||
@@ -0,0 +1,316 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import SystemMessage, ToolMessage
|
||||
from langgraph.types import Command
|
||||
|
||||
import yuxi.agents.middlewares.skills as skills_middleware
|
||||
from yuxi.agents.middlewares.skills import (
|
||||
SkillsMiddleware,
|
||||
resolve_runtime_skills_for_context,
|
||||
resolve_skill_gated_tools,
|
||||
)
|
||||
from yuxi.agents.toolkits.service import resolve_configured_runtime_tools
|
||||
|
||||
_KB_TOOL_NAMES = {
|
||||
"list_kbs",
|
||||
"query_kb",
|
||||
"find_kb_document",
|
||||
"open_kb_document",
|
||||
"get_mindmap",
|
||||
}
|
||||
|
||||
|
||||
def _system_message_text(message: SystemMessage) -> str:
|
||||
return "\n".join(block.get("text", "") for block in message.content_blocks if isinstance(block, dict))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_runtime_skills_derives_prompt_and_readable_closure(monkeypatch):
|
||||
async def fake_list_skills_from_db(db=None, user=None):
|
||||
del db, user
|
||||
return [
|
||||
SimpleNamespace(
|
||||
slug="alpha",
|
||||
name="Alpha",
|
||||
description="alpha desc",
|
||||
tool_dependencies=[],
|
||||
mcp_dependencies=[],
|
||||
skill_dependencies=["beta"],
|
||||
),
|
||||
SimpleNamespace(
|
||||
slug="beta",
|
||||
name="Beta",
|
||||
description="beta desc",
|
||||
tool_dependencies=[],
|
||||
mcp_dependencies=[],
|
||||
skill_dependencies=[],
|
||||
),
|
||||
]
|
||||
|
||||
monkeypatch.setattr(skills_middleware, "_list_skills_from_db", fake_list_skills_from_db)
|
||||
|
||||
context = SimpleNamespace(skills=["alpha", "missing"])
|
||||
|
||||
scope = await resolve_runtime_skills_for_context(context)
|
||||
|
||||
assert scope["context_skills"] == ["alpha"]
|
||||
assert scope["prompt_skills"] == ["alpha", "beta"]
|
||||
assert scope["readable_skills"] == ["alpha", "beta"]
|
||||
assert set(scope["runtime_skill_metadata"]) == {"alpha", "beta"}
|
||||
assert scope["runtime_skill_dependency_map"]["alpha"]["skills"] == ["beta"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skills_prompt_uses_prepared_prompt_skills_at_request_level():
|
||||
context = SimpleNamespace(
|
||||
system_prompt="context base",
|
||||
skills=["configured-only"],
|
||||
_prompt_skills=["alpha"],
|
||||
_runtime_skill_metadata={
|
||||
"alpha": {
|
||||
"name": "Alpha",
|
||||
"description": "alpha desc",
|
||||
"path": "/home/gem/skills/alpha/SKILL.md",
|
||||
},
|
||||
"configured-only": {
|
||||
"name": "Configured Only",
|
||||
"description": "should not appear",
|
||||
"path": "/home/gem/skills/configured-only/SKILL.md",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
class FakeRequest:
|
||||
def __init__(self, *, system_message=None, tools=None):
|
||||
self.runtime = SimpleNamespace(context=context)
|
||||
self.state = {}
|
||||
self.tools = tools or []
|
||||
self.system_message = system_message or SystemMessage(content="base")
|
||||
|
||||
def override(self, **kwargs):
|
||||
return FakeRequest(
|
||||
system_message=kwargs.get("system_message", self.system_message),
|
||||
tools=kwargs.get("tools", self.tools),
|
||||
)
|
||||
|
||||
captured = {}
|
||||
|
||||
async def handler(request):
|
||||
captured["system_message"] = request.system_message
|
||||
return "ok"
|
||||
|
||||
result = await SkillsMiddleware().awrap_model_call(FakeRequest(), handler)
|
||||
prompt_text = _system_message_text(captured["system_message"])
|
||||
|
||||
assert result == "ok"
|
||||
assert "base" in prompt_text
|
||||
assert "Alpha" in prompt_text
|
||||
assert "Configured Only" not in prompt_text
|
||||
assert context.system_prompt == "context base"
|
||||
assert not hasattr(context, "_skills_prompt_injected")
|
||||
assert not hasattr(context, "_visible_skills")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_awrap_model_call_mounts_dependencies_only_for_readable_activated_skills(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
skills_middleware,
|
||||
"get_all_tool_instances",
|
||||
lambda: [SimpleNamespace(name="tool-a"), SimpleNamespace(name="tool-b")],
|
||||
)
|
||||
|
||||
class FakeRequest:
|
||||
def __init__(self, tools=None):
|
||||
self.runtime = SimpleNamespace(
|
||||
context=SimpleNamespace(
|
||||
_readable_skills=["alpha"],
|
||||
_runtime_skill_dependency_map={
|
||||
"alpha": {"tools": ["tool-a"], "mcps": [], "skills": []},
|
||||
"beta": {"tools": ["tool-b"], "mcps": [], "skills": []},
|
||||
},
|
||||
mcps=[],
|
||||
)
|
||||
)
|
||||
self.state = {"activated_skills": ["alpha", "beta"]}
|
||||
self.tools = tools or []
|
||||
|
||||
def override(self, *, tools):
|
||||
new_request = FakeRequest(tools=tools)
|
||||
new_request.runtime = self.runtime
|
||||
new_request.state = self.state
|
||||
return new_request
|
||||
|
||||
captured = {}
|
||||
|
||||
async def handler(request):
|
||||
captured["tools"] = [tool.name for tool in request.tools]
|
||||
return "ok"
|
||||
|
||||
result = await SkillsMiddleware().awrap_model_call(FakeRequest(), handler)
|
||||
|
||||
assert result == "ok"
|
||||
assert captured["tools"] == ["tool-a"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_awrap_model_call_mounts_knowledge_base_skill_tools():
|
||||
class FakeRequest:
|
||||
def __init__(self, tools=None):
|
||||
self.runtime = SimpleNamespace(
|
||||
context=SimpleNamespace(
|
||||
_readable_skills=["knowledge-base"],
|
||||
_runtime_skill_dependency_map={
|
||||
"knowledge-base": {
|
||||
"tools": [
|
||||
"list_kbs",
|
||||
"query_kb",
|
||||
"find_kb_document",
|
||||
"open_kb_document",
|
||||
"get_mindmap",
|
||||
],
|
||||
"mcps": [],
|
||||
"skills": [],
|
||||
}
|
||||
},
|
||||
mcps=[],
|
||||
)
|
||||
)
|
||||
self.state = {"activated_skills": ["knowledge-base"]}
|
||||
self.tools = tools or []
|
||||
|
||||
def override(self, *, tools):
|
||||
new_request = FakeRequest(tools=tools)
|
||||
new_request.runtime = self.runtime
|
||||
new_request.state = self.state
|
||||
return new_request
|
||||
|
||||
captured = {}
|
||||
|
||||
async def handler(request):
|
||||
captured["tools"] = {tool.name for tool in request.tools}
|
||||
return "ok"
|
||||
|
||||
result = await SkillsMiddleware().awrap_model_call(FakeRequest(), handler)
|
||||
|
||||
assert result == "ok"
|
||||
assert captured["tools"] == {
|
||||
"list_kbs",
|
||||
"query_kb",
|
||||
"find_kb_document",
|
||||
"open_kb_document",
|
||||
"get_mindmap",
|
||||
}
|
||||
|
||||
|
||||
def test_resolve_skill_gated_tools_collects_readable_dependency_tools():
|
||||
"""门控工具必须能从可见 Skill 的依赖解析出真实工具实例,供构建期注册进 ToolNode。"""
|
||||
context = SimpleNamespace(
|
||||
_readable_skills=["knowledge-base"],
|
||||
_runtime_skill_dependency_map={"knowledge-base": {"tools": sorted(_KB_TOOL_NAMES), "mcps": [], "skills": []}},
|
||||
)
|
||||
|
||||
tools = resolve_skill_gated_tools(context)
|
||||
|
||||
assert {tool.name for tool in tools} == _KB_TOOL_NAMES
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_configured_runtime_tools_registers_skill_gated_tools():
|
||||
"""门控工具必须随基础工具一起进入 create_agent 工具列表(即注册进 ToolNode),否则激活后仍报 not a valid tool。"""
|
||||
context = SimpleNamespace(
|
||||
tools=None,
|
||||
mcps=None,
|
||||
_readable_skills=["knowledge-base"],
|
||||
_runtime_skill_dependency_map={"knowledge-base": {"tools": sorted(_KB_TOOL_NAMES), "mcps": [], "skills": []}},
|
||||
)
|
||||
|
||||
tools = await resolve_configured_runtime_tools(context)
|
||||
|
||||
assert _KB_TOOL_NAMES <= {tool.name for tool in tools}
|
||||
|
||||
|
||||
def _make_gated_request(activated):
|
||||
base = SimpleNamespace(name="read_file")
|
||||
gated = [SimpleNamespace(name="list_kbs"), SimpleNamespace(name="query_kb")]
|
||||
|
||||
class FakeRequest:
|
||||
def __init__(self, tools):
|
||||
self.runtime = SimpleNamespace(
|
||||
context=SimpleNamespace(
|
||||
_readable_skills=["knowledge-base"],
|
||||
_runtime_skill_dependency_map={
|
||||
"knowledge-base": {"tools": ["list_kbs", "query_kb"], "mcps": [], "skills": []}
|
||||
},
|
||||
mcps=[],
|
||||
)
|
||||
)
|
||||
self.state = {"activated_skills": activated}
|
||||
self.tools = tools
|
||||
|
||||
def override(self, *, tools):
|
||||
new_request = FakeRequest(tools)
|
||||
new_request.runtime = self.runtime
|
||||
new_request.state = self.state
|
||||
return new_request
|
||||
|
||||
# ToolNode 默认绑定 = 基础工具 + 门控工具
|
||||
return FakeRequest([base, *gated])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_awrap_model_call_hides_gated_tools_until_activated():
|
||||
"""未激活 Skill 时门控工具对模型不可见(懒加载),激活后才放出。"""
|
||||
request = _make_gated_request(activated=[])
|
||||
captured = {}
|
||||
|
||||
async def handler(req):
|
||||
captured["tools"] = {tool.name for tool in req.tools}
|
||||
return "ok"
|
||||
|
||||
await SkillsMiddleware().awrap_model_call(request, handler)
|
||||
|
||||
assert captured["tools"] == {"read_file"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_awrap_model_call_keeps_gated_tools_when_activated():
|
||||
request = _make_gated_request(activated=["knowledge-base"])
|
||||
captured = {}
|
||||
|
||||
async def handler(req):
|
||||
captured["tools"] = {tool.name for tool in req.tools}
|
||||
return "ok"
|
||||
|
||||
await SkillsMiddleware().awrap_model_call(request, handler)
|
||||
|
||||
assert captured["tools"] == {"read_file", "list_kbs", "query_kb"}
|
||||
|
||||
|
||||
def test_read_file_activates_only_readable_skill() -> None:
|
||||
middleware = SkillsMiddleware()
|
||||
result = ToolMessage(content="ok", tool_call_id="tool-1", name="read_file")
|
||||
request = SimpleNamespace(
|
||||
runtime=SimpleNamespace(context=SimpleNamespace(_readable_skills=["alpha"])),
|
||||
tool_call={"name": "read_file", "args": {"file_path": "/home/gem/skills/alpha/SKILL.md"}},
|
||||
)
|
||||
|
||||
updated = middleware._process_tool_call_result(result, request)
|
||||
|
||||
assert isinstance(updated, Command)
|
||||
assert updated.update["activated_skills"] == ["alpha"]
|
||||
|
||||
|
||||
def test_read_file_denies_skill_outside_readable_scope() -> None:
|
||||
middleware = SkillsMiddleware()
|
||||
result = ToolMessage(content="ok", tool_call_id="tool-1", name="read_file")
|
||||
request = SimpleNamespace(
|
||||
runtime=SimpleNamespace(context=SimpleNamespace(_readable_skills=["alpha"])),
|
||||
tool_call={"name": "read_file", "args": {"file_path": "/home/gem/skills/beta/SKILL.md"}},
|
||||
)
|
||||
|
||||
updated = middleware._process_tool_call_result(result, request)
|
||||
|
||||
assert updated is result
|
||||
@@ -0,0 +1,792 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import yuxi.agents.middlewares.subagent_task as subagent_task_middleware
|
||||
import yuxi.services.agent_run_service as agent_run_service
|
||||
import yuxi.services.run_queue_service as run_queue_service
|
||||
import yuxi.services.subagent_run_service as subagent_run_service
|
||||
from langchain.agents._subagent_transformer import SubagentTransformer as LangChainSubagentTransformer
|
||||
from langgraph.prebuilt.tool_node import ToolRuntime
|
||||
from langgraph.stream._mux import StreamMux
|
||||
from langgraph.types import Command
|
||||
from yuxi.agents.buildin.chatbot.state import merge_subagent_runs
|
||||
from yuxi.agents.middlewares.subagent_task import YUXI_SUBAGENTS_STREAM_KEY, YuxiSubAgentMiddleware
|
||||
from yuxi.repositories.agent_repository import SUB_AGENT_BACKEND_ID
|
||||
from yuxi.services.input_message_service import AgentRunInputMessage
|
||||
from yuxi.utils.hash_utils import subagent_child_thread_id
|
||||
|
||||
|
||||
def make_child_thread_id(parent_thread_id: str, agent_slug: str, tool_call_id: str) -> str:
|
||||
return subagent_child_thread_id(parent_thread_id, agent_slug, tool_call_id)
|
||||
|
||||
|
||||
class _ChildContext:
|
||||
def __init__(self):
|
||||
self.model = None
|
||||
|
||||
def update_from_dict(self, values: dict):
|
||||
for key, value in values.items():
|
||||
if hasattr(self, key):
|
||||
setattr(self, key, value)
|
||||
|
||||
|
||||
class _SessionContext:
|
||||
async def __aenter__(self):
|
||||
return object()
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
|
||||
def _patch_session(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
subagent_task_middleware,
|
||||
"pg_manager",
|
||||
SimpleNamespace(get_async_session_context=lambda: _SessionContext()),
|
||||
)
|
||||
|
||||
|
||||
def _patch_subagent_run_service(monkeypatch, service_class) -> None:
|
||||
monkeypatch.setattr(
|
||||
subagent_task_middleware,
|
||||
"_subagent_run_service_module",
|
||||
lambda: SimpleNamespace(
|
||||
SubagentRunService=service_class,
|
||||
SubagentRunBusy=subagent_run_service.SubagentRunBusy,
|
||||
serialize_subagent_run_state=subagent_run_service.serialize_subagent_run_state,
|
||||
subagent_run_urls=subagent_run_service.subagent_run_urls,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _async_tool_middleware(*, model: str | None = None) -> YuxiSubAgentMiddleware:
|
||||
parent_context = SimpleNamespace(thread_id="parent-thread", uid="user-1", run_id="parent-run")
|
||||
if model:
|
||||
parent_context.model = model
|
||||
return YuxiSubAgentMiddleware(
|
||||
parent_context=parent_context,
|
||||
subagents=[
|
||||
SimpleNamespace(
|
||||
slug="worker",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _subagent_run(
|
||||
*,
|
||||
status: str = "running",
|
||||
thread_id: str = "child-thread",
|
||||
tool_call_id: str = "tool-async",
|
||||
subagent_slug: str = "worker",
|
||||
subagent_name: str = "Worker",
|
||||
description: str = "run in background",
|
||||
error_message: str | None = None,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
id="child-run",
|
||||
conversation_thread_id=thread_id,
|
||||
agent_slug=subagent_slug,
|
||||
status=status,
|
||||
created_by_run_id="parent-run",
|
||||
subagent_thread_relation_id=77,
|
||||
input_payload={
|
||||
"runtime": {
|
||||
"tool_call_id": tool_call_id,
|
||||
"subagent_name": subagent_name,
|
||||
"description": description,
|
||||
},
|
||||
},
|
||||
created_at=None,
|
||||
finished_at=None,
|
||||
error_message=error_message,
|
||||
)
|
||||
|
||||
|
||||
def _patch_task_start_and_await(
|
||||
monkeypatch,
|
||||
captured: dict,
|
||||
*,
|
||||
status: str = "completed",
|
||||
output: str = "child done",
|
||||
thread_id: str = "child-thread",
|
||||
error_message: str | None = None,
|
||||
wait_timeout: bool = False,
|
||||
):
|
||||
class _SubagentRunService:
|
||||
def __init__(self, db):
|
||||
captured["db"] = db
|
||||
|
||||
async def start(self, **kwargs):
|
||||
captured["start"] = kwargs
|
||||
return SimpleNamespace(
|
||||
run=_subagent_run(
|
||||
status="pending",
|
||||
thread_id=thread_id,
|
||||
tool_call_id=kwargs["tool_call_id"],
|
||||
subagent_slug=kwargs["agent_item"].slug,
|
||||
subagent_name=kwargs["agent_item"].name,
|
||||
description=kwargs["input_message"].content,
|
||||
),
|
||||
created=True,
|
||||
continuing=bool(kwargs.get("requested_thread_id")),
|
||||
relation=SimpleNamespace(id=77, child_thread_id=thread_id),
|
||||
)
|
||||
|
||||
async def get_run_for_creator(self, **kwargs):
|
||||
captured["get_run_for_creator"] = kwargs
|
||||
started = captured["start"]
|
||||
return _subagent_run(
|
||||
status=status,
|
||||
thread_id=thread_id,
|
||||
tool_call_id=started["tool_call_id"],
|
||||
subagent_slug=started["agent_item"].slug,
|
||||
subagent_name=started["agent_item"].name,
|
||||
description=started["input_message"].content,
|
||||
error_message=error_message,
|
||||
)
|
||||
|
||||
async def fake_await_agent_run_result(*, run_id: str, current_uid: str):
|
||||
captured["await"] = {"run_id": run_id, "current_uid": current_uid}
|
||||
result = {
|
||||
"status": status,
|
||||
"output": output,
|
||||
"agent_run_id": run_id,
|
||||
"thread_id": thread_id,
|
||||
}
|
||||
if error_message:
|
||||
result["error"] = {"type": "RuntimeError", "message": error_message}
|
||||
if wait_timeout:
|
||||
raise agent_run_service.AgentRunWaitTimeout(result)
|
||||
return result
|
||||
|
||||
_patch_session(monkeypatch)
|
||||
_patch_subagent_run_service(monkeypatch, _SubagentRunService)
|
||||
monkeypatch.setattr(agent_run_service, "await_agent_run_result", fake_await_agent_run_result)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_task_middleware_loads_all_visible_subagents_when_empty(monkeypatch) -> None:
|
||||
class _UserRepository:
|
||||
async def get_by_uid_with_db(self, _db, uid):
|
||||
assert uid == "user-1"
|
||||
return SimpleNamespace(uid="user-1", role="user")
|
||||
|
||||
class _AgentRepository:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def list_visible_subagents(self, *, user):
|
||||
assert user.uid == "user-1"
|
||||
return [
|
||||
SimpleNamespace(
|
||||
slug="worker",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={},
|
||||
),
|
||||
]
|
||||
|
||||
async def get_visible_by_slug(self, *, slug, user, kind="main"):
|
||||
del slug
|
||||
del user
|
||||
del kind
|
||||
raise AssertionError("empty subagents should load all visible subagents")
|
||||
|
||||
_patch_session(monkeypatch)
|
||||
monkeypatch.setattr(subagent_task_middleware, "UserRepository", _UserRepository)
|
||||
monkeypatch.setattr(subagent_task_middleware, "AgentRepository", _AgentRepository)
|
||||
|
||||
middleware = await subagent_task_middleware.create_subagent_task_middleware(
|
||||
SimpleNamespace(thread_id="parent-thread", uid="user-1", subagents=[]),
|
||||
)
|
||||
|
||||
assert isinstance(middleware, YuxiSubAgentMiddleware)
|
||||
assert middleware.subagent_names == frozenset({"worker"})
|
||||
assert {tool.name for tool in middleware.tools} >= {"task", "subagent_start"}
|
||||
|
||||
|
||||
def test_yuxi_subagent_transformer_does_not_conflict_with_langchain_default() -> None:
|
||||
middleware = YuxiSubAgentMiddleware(
|
||||
parent_context=SimpleNamespace(thread_id="parent-thread", uid="user-1"),
|
||||
subagents=[
|
||||
SimpleNamespace(
|
||||
slug="worker",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
mux = StreamMux(
|
||||
factories=[LangChainSubagentTransformer, *middleware.transformers],
|
||||
is_async=True,
|
||||
)
|
||||
|
||||
assert "subagents" in mux.extensions
|
||||
assert YUXI_SUBAGENTS_STREAM_KEY in mux.extensions
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_tool_rejects_unconfigured_subagent() -> None:
|
||||
middleware = YuxiSubAgentMiddleware(
|
||||
parent_context=SimpleNamespace(thread_id="parent-thread", uid="user-1", model=""),
|
||||
subagents=[
|
||||
SimpleNamespace(
|
||||
slug="worker",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={},
|
||||
)
|
||||
],
|
||||
)
|
||||
runtime = ToolRuntime(
|
||||
state={},
|
||||
context=None,
|
||||
tool_call_id="tool-1",
|
||||
store=None,
|
||||
stream_writer=lambda _: None,
|
||||
config={},
|
||||
)
|
||||
|
||||
result = await middleware.tools[0].ainvoke(
|
||||
{"description": "do work", "subagent_slug": "missing", "runtime": runtime}
|
||||
)
|
||||
|
||||
assert result == "无法调用子智能体 missing,可用子智能体只有:`worker`"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_tool_invokes_subagent_with_child_scope(monkeypatch) -> None:
|
||||
captured = {}
|
||||
_patch_task_start_and_await(monkeypatch, captured, thread_id="child-thread")
|
||||
|
||||
middleware = YuxiSubAgentMiddleware(
|
||||
parent_context=SimpleNamespace(
|
||||
thread_id="child-runtime-thread",
|
||||
parent_thread_id="parent-thread",
|
||||
file_thread_id="parent-file-thread",
|
||||
uid="user-1",
|
||||
run_id="parent-run",
|
||||
),
|
||||
subagents=[
|
||||
SimpleNamespace(
|
||||
slug="worker.agent",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={"context": {"model": "provider:model", "subagents": ["nested"]}},
|
||||
)
|
||||
],
|
||||
)
|
||||
runtime = SimpleNamespace(tool_call_id="tool-1", state={}, config={})
|
||||
|
||||
result = await middleware.tools[0].coroutine(
|
||||
description="write a report",
|
||||
subagent_slug="worker.agent",
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert result.update["messages"][0].content == "> 子智能体线程 ID: child-thread\n\n---\n\nchild done"
|
||||
assert result.update["messages"][0].tool_call_id == "tool-1"
|
||||
assert captured["start"]["uid"] == "user-1"
|
||||
assert captured["start"]["created_by_run_id"] == "parent-run"
|
||||
assert captured["start"]["requested_thread_id"] is None
|
||||
assert captured["start"]["file_thread_id"] == "parent-file-thread"
|
||||
assert captured["start"]["model_spec"] is None
|
||||
assert captured["await"] == {"run_id": "child-run", "current_uid": "user-1"}
|
||||
assert result.update["subagent_runs"][0]["run_id"] == "child-run"
|
||||
assert result.update["subagent_runs"][0]["child_thread_id"] == "child-thread"
|
||||
assert result.update["subagent_runs"][0]["status"] == "completed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_tool_inherits_parent_model_when_subagent_model_empty(monkeypatch) -> None:
|
||||
captured = {}
|
||||
_patch_task_start_and_await(monkeypatch, captured)
|
||||
|
||||
middleware = YuxiSubAgentMiddleware(
|
||||
parent_context=SimpleNamespace(
|
||||
thread_id="parent-thread",
|
||||
uid="user-1",
|
||||
run_id="parent-run",
|
||||
model="parent:model",
|
||||
),
|
||||
subagents=[
|
||||
SimpleNamespace(
|
||||
slug="worker",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={"context": {"model": ""}},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
await middleware.tools[0].coroutine(
|
||||
description="write a report",
|
||||
subagent_slug="worker",
|
||||
runtime=SimpleNamespace(tool_call_id="tool-1", state={}, config={}),
|
||||
)
|
||||
|
||||
assert captured["start"]["model_spec"] == "parent:model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_tool_records_failed_subagent_run(monkeypatch) -> None:
|
||||
captured = {}
|
||||
_patch_task_start_and_await(monkeypatch, captured, status="failed", output="", error_message="child boom")
|
||||
|
||||
middleware = YuxiSubAgentMiddleware(
|
||||
parent_context=SimpleNamespace(thread_id="parent-thread", uid="user-1", run_id="parent-run", model=""),
|
||||
subagents=[
|
||||
SimpleNamespace(
|
||||
slug="worker",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
result = await middleware.tools[0].coroutine(
|
||||
description="write a report",
|
||||
subagent_slug="worker",
|
||||
runtime=SimpleNamespace(tool_call_id="tool-1", state={}, config={}),
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert result.update["messages"][0].content == "> 子智能体线程 ID: child-thread\n\n---\n\nchild boom"
|
||||
assert result.update["subagent_runs"][0]["status"] == "failed"
|
||||
assert result.update["subagent_runs"][0]["error"] == "child boom"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_tool_reports_running_subagent_after_wait_timeout(monkeypatch) -> None:
|
||||
captured = {}
|
||||
_patch_task_start_and_await(monkeypatch, captured, status="running", output="", wait_timeout=True)
|
||||
|
||||
middleware = YuxiSubAgentMiddleware(
|
||||
parent_context=SimpleNamespace(thread_id="parent-thread", uid="user-1", run_id="parent-run", model=""),
|
||||
subagents=[
|
||||
SimpleNamespace(
|
||||
slug="worker",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
result = await middleware.tools[0].coroutine(
|
||||
description="write a long report",
|
||||
subagent_slug="worker",
|
||||
runtime=SimpleNamespace(tool_call_id="tool-1", state={}, config={}),
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
content = result.update["messages"][0].content
|
||||
assert "子智能体仍在运行" in content
|
||||
assert "run_id: child-run" in content
|
||||
assert "不要把当前结果视为任务已完成" in content
|
||||
assert "子智能体已完成任务" not in content
|
||||
assert result.update["subagent_runs"][0]["status"] == "running"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_tool_continues_existing_subagent_thread(monkeypatch) -> None:
|
||||
captured = {}
|
||||
_patch_task_start_and_await(monkeypatch, captured, output="continued done", thread_id="child-thread")
|
||||
|
||||
middleware = YuxiSubAgentMiddleware(
|
||||
parent_context=SimpleNamespace(thread_id="parent-thread", uid="user-1", run_id="parent-run", model=""),
|
||||
subagents=[
|
||||
SimpleNamespace(
|
||||
slug="worker.agent",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
result = await middleware.tools[0].coroutine(
|
||||
description="continue the report",
|
||||
subagent_slug="worker.agent",
|
||||
runtime=SimpleNamespace(tool_call_id="tool-2", state={}, config={}),
|
||||
thread_id="child-thread",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert result.update["messages"][0].content == "> 子智能体线程 ID: child-thread\n\n---\n\ncontinued done"
|
||||
assert captured["start"]["requested_thread_id"] == "child-thread"
|
||||
assert result.update["subagent_runs"][0]["child_thread_id"] == "child-thread"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_tool_rejects_invalid_continuation_thread(monkeypatch) -> None:
|
||||
class _SubagentRunService:
|
||||
def __init__(self, db):
|
||||
del db
|
||||
|
||||
async def start(self, **kwargs):
|
||||
raise ValueError(
|
||||
f"无法继续子智能体线程 {kwargs['requested_thread_id']}:当前对话中没有找到对应的运行记录"
|
||||
)
|
||||
|
||||
_patch_session(monkeypatch)
|
||||
_patch_subagent_run_service(monkeypatch, _SubagentRunService)
|
||||
middleware = YuxiSubAgentMiddleware(
|
||||
parent_context=SimpleNamespace(thread_id="parent-thread", uid="user-1", run_id="parent-run"),
|
||||
subagents=[
|
||||
SimpleNamespace(
|
||||
slug="worker",
|
||||
name="Worker",
|
||||
description="work on scoped tasks",
|
||||
backend_id=SUB_AGENT_BACKEND_ID,
|
||||
config_json={},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
unknown_thread_id = "opaque-child-thread"
|
||||
runtime = SimpleNamespace(tool_call_id="tool-2", state={}, config={})
|
||||
result = await middleware.tools[0].coroutine(
|
||||
description="continue",
|
||||
subagent_slug="worker",
|
||||
runtime=runtime,
|
||||
thread_id=unknown_thread_id,
|
||||
)
|
||||
|
||||
assert result == f"无法继续子智能体线程 {unknown_thread_id}:当前对话中没有找到对应的运行记录"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_start_creates_child_run_and_enqueues(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class _SubagentRunService:
|
||||
def __init__(self, db):
|
||||
captured["db"] = db
|
||||
|
||||
async def start(self, **kwargs):
|
||||
captured["start"] = kwargs
|
||||
child_thread_id = make_child_thread_id("parent-thread", "worker", "tool-async")
|
||||
return SimpleNamespace(
|
||||
run=_subagent_run(status="pending", thread_id=child_thread_id),
|
||||
created=True,
|
||||
continuing=False,
|
||||
relation=SimpleNamespace(id=77, child_thread_id=child_thread_id),
|
||||
)
|
||||
|
||||
_patch_session(monkeypatch)
|
||||
_patch_subagent_run_service(monkeypatch, _SubagentRunService)
|
||||
|
||||
middleware = _async_tool_middleware(model="provider:parent-model")
|
||||
runtime = SimpleNamespace(tool_call_id="tool-async", state={}, config={})
|
||||
tool = next(item for item in middleware.tools if item.name == "subagent_start")
|
||||
|
||||
result = await tool.coroutine(
|
||||
description="run in background",
|
||||
subagent_slug="worker",
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
child_thread_id = make_child_thread_id("parent-thread", "worker", "tool-async")
|
||||
assert isinstance(result, Command)
|
||||
payload = json.loads(result.update["messages"][0].content)
|
||||
assert payload["status"] == "started"
|
||||
assert payload["run_id"] == "child-run"
|
||||
assert payload["thread_id"] == child_thread_id
|
||||
assert payload["events_url"] == "/api/agent/runs/child-run/events"
|
||||
assert payload["subagent_thread_relation_id"] == 77
|
||||
assert captured["start"]["uid"] == "user-1"
|
||||
assert captured["start"]["created_by_run_id"] == "parent-run"
|
||||
assert isinstance(captured["start"]["input_message"], AgentRunInputMessage)
|
||||
assert captured["start"]["input_message"].content == "run in background"
|
||||
assert captured["start"]["input_message"].raw_message()["type"] == "human"
|
||||
assert captured["start"]["input_message"].raw_message()["content"] == "run in background"
|
||||
assert "description" not in captured["start"]
|
||||
assert captured["start"]["requested_thread_id"] is None
|
||||
assert captured["start"]["model_spec"] == "provider:parent-model"
|
||||
assert result.update["subagent_runs"][0]["child_thread_id"] == child_thread_id
|
||||
assert result.update["subagent_runs"][0]["run_id"] == "child-run"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_status_returns_terminal_result(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class _SubagentRunService:
|
||||
def __init__(self, db):
|
||||
captured["db"] = db
|
||||
|
||||
async def get_run_for_creator(self, **kwargs):
|
||||
captured["get_run_for_creator"] = kwargs
|
||||
return _subagent_run(status="completed")
|
||||
|
||||
async def fake_get_agent_run_result(*, run_id: str, current_uid: str, db):
|
||||
captured["get_agent_run_result"] = {"run_id": run_id, "current_uid": current_uid, "db": db}
|
||||
return {"status": "completed", "output": "final result"}
|
||||
|
||||
async def fake_get_agent_run_progress(run_id: str):
|
||||
captured["get_agent_run_progress"] = run_id
|
||||
return {
|
||||
"last_seq": "3-0",
|
||||
"messages": [{"seq": "3-0", "kind": "assistant_message", "message_id": "msg-1", "content": "working"}],
|
||||
}
|
||||
|
||||
_patch_session(monkeypatch)
|
||||
_patch_subagent_run_service(monkeypatch, _SubagentRunService)
|
||||
monkeypatch.setattr(agent_run_service, "get_agent_run_result", fake_get_agent_run_result)
|
||||
monkeypatch.setattr(agent_run_service, "get_agent_run_progress", fake_get_agent_run_progress)
|
||||
|
||||
tool = next(item for item in _async_tool_middleware().tools if item.name == "subagent_status")
|
||||
result = await tool.coroutine(run_id="child-run", runtime=SimpleNamespace(tool_call_id="status-call"))
|
||||
|
||||
assert isinstance(result, Command)
|
||||
payload = json.loads(result.update["messages"][0].content)
|
||||
assert payload["status"] == "completed"
|
||||
assert payload["progress"]["messages"][0]["content"] == "working"
|
||||
assert payload["result"]["output"] == "final result"
|
||||
assert captured["get_agent_run_progress"] == "child-run"
|
||||
assert captured["get_run_for_creator"] == {
|
||||
"uid": "user-1",
|
||||
"created_by_run_id": "parent-run",
|
||||
"run_id": "child-run",
|
||||
}
|
||||
assert captured["get_agent_run_result"]["current_uid"] == "user-1"
|
||||
assert result.update["subagent_runs"] == [
|
||||
{
|
||||
"id": "tool-async",
|
||||
"run_id": "child-run",
|
||||
"subagent_slug": "worker",
|
||||
"subagent_name": "Worker",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "completed",
|
||||
"events_url": "/api/agent/runs/child-run/events",
|
||||
"result_url": "/api/agent/runs/child-run/result",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_status_returns_progress_for_running_run(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class _SubagentRunService:
|
||||
def __init__(self, db):
|
||||
captured["db"] = db
|
||||
|
||||
async def get_run_for_creator(self, **kwargs):
|
||||
captured["get_run_for_creator"] = kwargs
|
||||
return _subagent_run(status="running")
|
||||
|
||||
async def fake_get_agent_run_result(**kwargs):
|
||||
raise AssertionError(f"running status should not load terminal result: {kwargs}")
|
||||
|
||||
async def fake_get_agent_run_progress(run_id: str):
|
||||
captured["get_agent_run_progress"] = run_id
|
||||
return {
|
||||
"last_seq": "9-0",
|
||||
"messages": [
|
||||
{"seq": "8-0", "kind": "tool_call", "tool_call_id": "call-1", "content": "调用工具 read_file"},
|
||||
{"seq": "9-0", "kind": "assistant_message", "message_id": "msg-2", "content": "正在整理结果"},
|
||||
],
|
||||
}
|
||||
|
||||
_patch_session(monkeypatch)
|
||||
_patch_subagent_run_service(monkeypatch, _SubagentRunService)
|
||||
monkeypatch.setattr(agent_run_service, "get_agent_run_result", fake_get_agent_run_result)
|
||||
monkeypatch.setattr(agent_run_service, "get_agent_run_progress", fake_get_agent_run_progress)
|
||||
|
||||
tool = next(item for item in _async_tool_middleware().tools if item.name == "subagent_status")
|
||||
result = await tool.coroutine(run_id="child-run", runtime=SimpleNamespace(tool_call_id="status-call"))
|
||||
|
||||
payload = json.loads(result.update["messages"][0].content)
|
||||
assert payload["status"] == "running"
|
||||
assert "result" not in payload
|
||||
assert payload["progress"]["last_seq"] == "9-0"
|
||||
assert [item["content"] for item in payload["progress"]["messages"]] == ["调用工具 read_file", "正在整理结果"]
|
||||
assert captured["get_run_for_creator"] == {
|
||||
"uid": "user-1",
|
||||
"created_by_run_id": "parent-run",
|
||||
"run_id": "child-run",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_events_cancel_and_await_use_parent_run_scope(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {"loads": []}
|
||||
|
||||
async def fake_get_verified_subagent_run(self, *, run_id: str, uid: str, created_by_run_id: str):
|
||||
del self
|
||||
captured["loads"].append(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"uid": uid,
|
||||
"created_by_run_id": created_by_run_id,
|
||||
}
|
||||
)
|
||||
return _subagent_run(status="completed")
|
||||
|
||||
async def fake_list_run_stream_events(run_id: str, *, after_seq: str, limit: int):
|
||||
captured["events"] = {"run_id": run_id, "after_seq": after_seq, "limit": limit}
|
||||
return [{"seq": "2-0", "event_type": "messages", "payload": {"chunk": {"status": "loading"}}}]
|
||||
|
||||
async def fake_request_cancel_agent_run(*, run_id: str, current_uid: str, db):
|
||||
captured["cancel"] = {"run_id": run_id, "current_uid": current_uid, "db": db}
|
||||
return _subagent_run(status="cancelling")
|
||||
|
||||
async def fake_await_agent_run_result(*, run_id: str, current_uid: str):
|
||||
captured["await"] = {"run_id": run_id, "current_uid": current_uid}
|
||||
return {"status": "completed", "output": "awaited result"}
|
||||
|
||||
_patch_session(monkeypatch)
|
||||
monkeypatch.setattr(YuxiSubAgentMiddleware, "_get_verified_subagent_run", fake_get_verified_subagent_run)
|
||||
monkeypatch.setattr(run_queue_service, "list_run_stream_events", fake_list_run_stream_events)
|
||||
monkeypatch.setattr(agent_run_service, "request_cancel_agent_run", fake_request_cancel_agent_run)
|
||||
monkeypatch.setattr(agent_run_service, "await_agent_run_result", fake_await_agent_run_result)
|
||||
|
||||
tools = {item.name: item for item in _async_tool_middleware().tools}
|
||||
|
||||
events_result = await tools["subagent_events"].coroutine(
|
||||
run_id="child-run",
|
||||
after_seq="1-0",
|
||||
limit=99,
|
||||
runtime=SimpleNamespace(tool_call_id="events-call"),
|
||||
)
|
||||
events_payload = json.loads(events_result.update["messages"][0].content)
|
||||
assert events_payload["last_seq"] == "2-0"
|
||||
assert captured["events"] == {"run_id": "child-run", "after_seq": "1-0", "limit": 50}
|
||||
|
||||
cancel_result = await tools["subagent_cancel"].coroutine(
|
||||
run_id="child-run",
|
||||
runtime=SimpleNamespace(tool_call_id="cancel-call"),
|
||||
)
|
||||
cancel_payload = json.loads(cancel_result.update["messages"][0].content)
|
||||
assert cancel_payload["status"] == "cancelling"
|
||||
assert captured["cancel"]["current_uid"] == "user-1"
|
||||
|
||||
await_result = await tools["subagent_await"].coroutine(
|
||||
run_id="child-run",
|
||||
runtime=SimpleNamespace(tool_call_id="await-call"),
|
||||
)
|
||||
await_payload = json.loads(await_result.update["messages"][0].content)
|
||||
assert await_payload["result"]["output"] == "awaited result"
|
||||
assert captured["await"] == {"run_id": "child-run", "current_uid": "user-1"}
|
||||
assert captured["loads"]
|
||||
assert all(
|
||||
load == {"run_id": "child-run", "uid": "user-1", "created_by_run_id": "parent-run"}
|
||||
for load in captured["loads"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_await_reports_timeout_when_run_is_still_active(monkeypatch) -> None:
|
||||
captured: dict[str, object] = {"loads": []}
|
||||
|
||||
async def fake_get_verified_subagent_run(self, *, run_id: str, uid: str, created_by_run_id: str):
|
||||
del self
|
||||
captured["loads"].append(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"uid": uid,
|
||||
"created_by_run_id": created_by_run_id,
|
||||
}
|
||||
)
|
||||
return _subagent_run(status="running")
|
||||
|
||||
async def fake_await_agent_run_result(*, run_id: str, current_uid: str):
|
||||
captured["await"] = {"run_id": run_id, "current_uid": current_uid}
|
||||
raise agent_run_service.AgentRunWaitTimeout(
|
||||
{"status": "running", "agent_run_id": run_id, "thread_id": "child-thread", "output": ""}
|
||||
)
|
||||
|
||||
_patch_session(monkeypatch)
|
||||
monkeypatch.setattr(YuxiSubAgentMiddleware, "_get_verified_subagent_run", fake_get_verified_subagent_run)
|
||||
monkeypatch.setattr(agent_run_service, "await_agent_run_result", fake_await_agent_run_result)
|
||||
|
||||
result = await {item.name: item for item in _async_tool_middleware().tools}["subagent_await"].coroutine(
|
||||
run_id="child-run",
|
||||
runtime=SimpleNamespace(tool_call_id="await-call"),
|
||||
)
|
||||
|
||||
payload = json.loads(result.update["messages"][0].content)
|
||||
assert payload["status"] == "running"
|
||||
assert payload["wait_timed_out"] is True
|
||||
assert payload["message"] == "子智能体仍在运行,等待最终结果超时;请稍后继续查询。"
|
||||
assert payload["result"]["status"] == "running"
|
||||
assert captured["await"] == {"run_id": "child-run", "current_uid": "user-1"}
|
||||
assert len(captured["loads"]) == 2
|
||||
|
||||
|
||||
def test_merge_subagent_runs_keeps_new_run_on_same_child_thread() -> None:
|
||||
child_thread_id = make_child_thread_id("parent-thread", "worker", "tool-old")
|
||||
|
||||
merged = merge_subagent_runs(
|
||||
[
|
||||
{
|
||||
"id": "tool-old",
|
||||
"run_id": "run-old",
|
||||
"subagent_slug": "worker",
|
||||
"subagent_name": "Worker",
|
||||
"child_thread_id": child_thread_id,
|
||||
"description": "first task",
|
||||
"status": "completed",
|
||||
"created_at": "2026-05-31T01:00:00Z",
|
||||
"completed_at": "2026-05-31T01:01:00Z",
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"id": "tool-new",
|
||||
"run_id": "run-new",
|
||||
"subagent_slug": "worker",
|
||||
"subagent_name": "Worker",
|
||||
"child_thread_id": child_thread_id,
|
||||
"description": "continue task",
|
||||
"status": "pending",
|
||||
"created_at": "2026-05-31T02:00:00Z",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert merged == [
|
||||
{
|
||||
"id": "tool-old",
|
||||
"run_id": "run-old",
|
||||
"subagent_slug": "worker",
|
||||
"subagent_name": "Worker",
|
||||
"child_thread_id": child_thread_id,
|
||||
"description": "first task",
|
||||
"status": "completed",
|
||||
"created_at": "2026-05-31T01:00:00Z",
|
||||
"completed_at": "2026-05-31T01:01:00Z",
|
||||
},
|
||||
{
|
||||
"id": "tool-new",
|
||||
"run_id": "run-new",
|
||||
"subagent_slug": "worker",
|
||||
"subagent_name": "Worker",
|
||||
"child_thread_id": child_thread_id,
|
||||
"description": "continue task",
|
||||
"status": "pending",
|
||||
"created_at": "2026-05-31T02:00:00Z",
|
||||
},
|
||||
]
|
||||
@@ -0,0 +1,818 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from langchain.agents.middleware.types import ExtendedModelResponse, ModelRequest, ModelResponse
|
||||
from deepagents.middleware.summarization import SummarizationMiddleware
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage, get_buffer_string
|
||||
from langchain_core.exceptions import ContextOverflowError
|
||||
|
||||
from yuxi.agents.backends.composite import create_agent_composite_backend
|
||||
from yuxi.agents.middlewares.summary import (
|
||||
YuxiSummarizationMiddleware,
|
||||
create_summary_middleware,
|
||||
sanitize_messages_for_summary,
|
||||
)
|
||||
from yuxi.utils.paths import VIRTUAL_PATH_CONVERSATION_HISTORY, VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
|
||||
|
||||
class _DummyModel:
|
||||
_llm_type = "test-chat"
|
||||
profile = {"max_input_tokens": 128000}
|
||||
|
||||
def _get_ls_params(self) -> dict[str, str]:
|
||||
return {"ls_provider": "openai"}
|
||||
|
||||
def invoke(self, _prompt: str, config: dict | None = None) -> SimpleNamespace:
|
||||
return SimpleNamespace(text="summary")
|
||||
|
||||
|
||||
class _RecordingModel(_DummyModel):
|
||||
def __init__(self) -> None:
|
||||
self.prompts: list[str] = []
|
||||
|
||||
def invoke(self, prompt: str, config: dict | None = None) -> SimpleNamespace:
|
||||
self.prompts.append(prompt)
|
||||
return SimpleNamespace(text="summary")
|
||||
|
||||
|
||||
class _MemoryBackend:
|
||||
def __init__(self) -> None:
|
||||
self.writes: list[tuple[str, str]] = []
|
||||
self.files: dict[str, str] = {}
|
||||
|
||||
def download_files(self, paths: list[str]) -> list[SimpleNamespace]:
|
||||
responses = []
|
||||
for path in paths:
|
||||
if path in self.files:
|
||||
responses.append(SimpleNamespace(content=self.files[path].encode("utf-8"), error=None))
|
||||
else:
|
||||
responses.append(SimpleNamespace(content=None, error="file_not_found"))
|
||||
return responses
|
||||
|
||||
def write(self, path: str, content: str) -> SimpleNamespace:
|
||||
self.writes.append((path, content))
|
||||
self.files[path] = content
|
||||
return SimpleNamespace(error=None)
|
||||
|
||||
def edit(self, path: str, old_string: str, new_string: str) -> SimpleNamespace:
|
||||
self.writes.append((path, new_string))
|
||||
self.files[path] = new_string
|
||||
return SimpleNamespace(error=None)
|
||||
|
||||
async def adownload_files(self, paths: list[str]) -> list[SimpleNamespace]:
|
||||
return self.download_files(paths)
|
||||
|
||||
async def awrite(self, path: str, content: str) -> SimpleNamespace:
|
||||
return self.write(path, content)
|
||||
|
||||
async def aedit(self, path: str, old_string: str, new_string: str) -> SimpleNamespace:
|
||||
return self.edit(path, old_string, new_string)
|
||||
|
||||
|
||||
def _expected_tool_result_path(content: str, tool_name: str = "query_kb") -> str:
|
||||
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()[:16]
|
||||
return f"{VIRTUAL_PATH_LARGE_TOOL_RESULTS}/{tool_name}-{digest}.txt"
|
||||
|
||||
|
||||
def _tool_messages() -> list:
|
||||
return [
|
||||
HumanMessage(content="请查询一下项目资料"),
|
||||
AIMessage(
|
||||
content="我先查资料",
|
||||
tool_calls=[
|
||||
{
|
||||
"id": "call-1",
|
||||
"name": "query_kb",
|
||||
"args": {"query": "very sensitive query payload"},
|
||||
}
|
||||
],
|
||||
additional_kwargs={
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"function": {"name": "query_kb", "arguments": '{"query":"raw"}'},
|
||||
}
|
||||
],
|
||||
"function_call": {"name": "query_kb"},
|
||||
},
|
||||
response_metadata={"finish_reason": "tool_calls"},
|
||||
),
|
||||
ToolMessage(content="TOOL_RESULT_SHOULD_NOT_BE_SUMMARIZED", tool_call_id="call-1", name="query_kb"),
|
||||
AIMessage(content="最终答案保留"),
|
||||
]
|
||||
|
||||
|
||||
def _model_request(messages: list) -> ModelRequest:
|
||||
return ModelRequest(
|
||||
model=_DummyModel(),
|
||||
messages=messages,
|
||||
system_message=None,
|
||||
tools=[],
|
||||
runtime=SimpleNamespace(context={}, config={}),
|
||||
state={"messages": messages},
|
||||
)
|
||||
|
||||
|
||||
def _content_char_counter(messages, **_kwargs) -> int:
|
||||
total = 0
|
||||
for message in messages:
|
||||
if message is None:
|
||||
continue
|
||||
content = getattr(message, "content", "")
|
||||
if isinstance(content, list):
|
||||
total += sum(len(str(item)) for item in content)
|
||||
else:
|
||||
total += len(str(content))
|
||||
return total
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def compression_events(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
|
||||
"""捕获 YuxiSummarizationMiddleware 通过 stream writer 推送的压缩事件。"""
|
||||
emitted: list[dict] = []
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.middlewares.summary.get_stream_writer",
|
||||
lambda: lambda payload: emitted.append(payload),
|
||||
)
|
||||
return emitted
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_create_summary_middleware_uses_deepagents_with_yuxi_outputs_root() -> None:
|
||||
middleware = create_summary_middleware(
|
||||
model=_DummyModel(),
|
||||
trigger=("tokens", 90_000),
|
||||
keep=("tokens", 45_000),
|
||||
trim_tokens_to_summarize=4000,
|
||||
)
|
||||
|
||||
assert isinstance(middleware, SummarizationMiddleware)
|
||||
assert isinstance(middleware, YuxiSummarizationMiddleware)
|
||||
assert middleware._backend is create_agent_composite_backend
|
||||
assert middleware._history_path_prefix == VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
assert middleware._large_tool_results_prefix == VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
assert middleware._lc_helper.trigger == ("tokens", 90_000)
|
||||
assert middleware._lc_helper.keep == ("tokens", 45_000)
|
||||
assert middleware._lc_helper.trim_tokens_to_summarize == 4000
|
||||
assert middleware.tool_result_offload_token_limit == 300
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_create_summary_middleware_passes_custom_summary_prompt() -> None:
|
||||
model = _RecordingModel()
|
||||
middleware = create_summary_middleware(
|
||||
model=model,
|
||||
trigger=("messages", 3),
|
||||
keep=("messages", 1),
|
||||
summary_prompt="CUSTOM SUMMARY PROMPT\n用户要求和偏好必须记录\n{messages}",
|
||||
trim_tokens_to_summarize=None,
|
||||
)
|
||||
|
||||
assert middleware._create_summary(_tool_messages()) == "summary"
|
||||
|
||||
prompt = model.prompts[0]
|
||||
assert prompt.startswith("CUSTOM SUMMARY PROMPT")
|
||||
assert "用户要求和偏好必须记录" in prompt
|
||||
assert "最终答案保留" in prompt
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_wrap_model_call_ignores_provider_reported_usage_for_token_trigger() -> None:
|
||||
backend = _MemoryBackend()
|
||||
model = _RecordingModel()
|
||||
messages = [
|
||||
HumanMessage(content="short user turn"),
|
||||
AIMessage(
|
||||
content="short answer",
|
||||
usage_metadata={"input_tokens": 200_000, "output_tokens": 100, "total_tokens": 200_100},
|
||||
response_metadata={"model_provider": "openai"},
|
||||
),
|
||||
HumanMessage(content="next short turn"),
|
||||
]
|
||||
middleware = create_summary_middleware(
|
||||
model=model,
|
||||
trigger=("tokens", 1_000),
|
||||
keep=("messages", 1),
|
||||
trim_tokens_to_summarize=None,
|
||||
)
|
||||
captured_messages: list | None = None
|
||||
|
||||
def handler(request: ModelRequest) -> ModelResponse:
|
||||
nonlocal captured_messages
|
||||
captured_messages = request.messages
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
middleware._backend_for_request = lambda _request: backend
|
||||
result = middleware.wrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert not isinstance(result, ExtendedModelResponse)
|
||||
assert captured_messages == messages
|
||||
assert model.prompts == []
|
||||
assert backend.writes == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_sanitize_messages_for_summary_only_replaces_tool_message_content() -> None:
|
||||
backend = _MemoryBackend()
|
||||
messages = _tool_messages()
|
||||
|
||||
sanitized = sanitize_messages_for_summary(messages, backend=backend)
|
||||
|
||||
assert [message.type for message in sanitized] == ["human", "ai", "tool", "ai"]
|
||||
assert sanitized[0] is messages[0]
|
||||
assert sanitized[1] is messages[1]
|
||||
assert sanitized[3] is messages[3]
|
||||
assert sanitized[1].tool_calls == messages[1].tool_calls
|
||||
assert sanitized[1].additional_kwargs == messages[1].additional_kwargs
|
||||
assert sanitized[1].response_metadata == messages[1].response_metadata
|
||||
assert isinstance(sanitized[2], ToolMessage)
|
||||
assert sanitized[2] is not messages[2]
|
||||
assert sanitized[2].tool_call_id == messages[2].tool_call_id
|
||||
assert sanitized[2].content != messages[2].content
|
||||
|
||||
assert backend.writes == [(_expected_tool_result_path(messages[2].content), messages[2].content)]
|
||||
formatted = get_buffer_string(sanitized)
|
||||
assert "Tool calls omitted from summary input" not in formatted
|
||||
assert "[Tool result saved]" in formatted
|
||||
assert "Tool: query_kb" in formatted
|
||||
assert "Tool call id" not in formatted
|
||||
assert f"Full output path: {_expected_tool_result_path(messages[2].content)}" in formatted
|
||||
assert "TOOL_RESULT_SHOULD_NOT_BE_SUMMARIZED" in formatted
|
||||
assert "最终答案保留" in formatted
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_sanitize_messages_for_summary_writes_large_tool_result_and_limits_preview() -> None:
|
||||
backend = _MemoryBackend()
|
||||
large_result = "BEGIN\n" + ("middle\n" * 2000) + "END"
|
||||
messages = [
|
||||
HumanMessage(content="查资料"),
|
||||
AIMessage(content="", tool_calls=[{"id": "call-1", "name": "query_kb", "args": {}}]),
|
||||
ToolMessage(content=large_result, tool_call_id="call-1", name="query_kb"),
|
||||
]
|
||||
|
||||
sanitized = sanitize_messages_for_summary(messages, backend=backend, tool_result_offload_token_limit=10)
|
||||
formatted = get_buffer_string(sanitized)
|
||||
|
||||
assert backend.writes == [(_expected_tool_result_path(large_result), large_result)]
|
||||
assert sanitized[1] is messages[1]
|
||||
assert isinstance(sanitized[2], ToolMessage)
|
||||
assert "[Tool result saved]" in formatted
|
||||
assert f"Full output path: {_expected_tool_result_path(large_result)}" in formatted
|
||||
assert "BEGIN" in formatted
|
||||
assert "END" not in formatted
|
||||
assert "Truncated" in formatted
|
||||
assert len(sanitized[2].content) < len(large_result)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_sanitize_messages_for_summary_omits_preview_when_limit_is_zero() -> None:
|
||||
backend = _MemoryBackend()
|
||||
result_content = "SECRET_RESULT_SHOULD_NOT_BE_IN_PROMPT"
|
||||
messages = [
|
||||
ToolMessage(content=result_content, tool_call_id="call-1", name="query_kb"),
|
||||
]
|
||||
|
||||
sanitized = sanitize_messages_for_summary(messages, backend=backend, tool_result_offload_token_limit=0)
|
||||
formatted = get_buffer_string(sanitized)
|
||||
|
||||
assert backend.writes == [(_expected_tool_result_path(result_content), result_content)]
|
||||
assert f"Full output path: {_expected_tool_result_path(result_content)}" in formatted
|
||||
assert result_content not in formatted
|
||||
assert "Output preview:" not in formatted
|
||||
assert "Truncated" in formatted
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_wrap_model_call_offloads_large_tool_messages_in_l1_without_state_mutation() -> None:
|
||||
backend = _MemoryBackend()
|
||||
model = _RecordingModel()
|
||||
large_result = "BEGIN\n" + ("raw result payload\n" * 200)
|
||||
messages = [
|
||||
HumanMessage(content="查资料"),
|
||||
AIMessage(content="", tool_calls=[{"id": "call-1", "name": "query_kb", "args": {}}]),
|
||||
ToolMessage(content=large_result, tool_call_id="call-1", name="query_kb"),
|
||||
AIMessage(content="资料已整理"),
|
||||
HumanMessage(content="继续"),
|
||||
]
|
||||
middleware = YuxiSummarizationMiddleware(
|
||||
model=model,
|
||||
backend=backend,
|
||||
trigger=("tokens", 500),
|
||||
keep=("messages", 3),
|
||||
token_counter=_content_char_counter,
|
||||
trim_tokens_to_summarize=None,
|
||||
tool_result_offload_token_limit=1,
|
||||
l1_l2_trigger_ratio=100.0,
|
||||
)
|
||||
middleware._history_path_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
captured_messages: list | None = None
|
||||
|
||||
def handler(request: ModelRequest) -> ModelResponse:
|
||||
nonlocal captured_messages
|
||||
captured_messages = request.messages
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = middleware.wrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert not isinstance(result, ExtendedModelResponse)
|
||||
assert model.prompts == []
|
||||
assert captured_messages is not None
|
||||
formatted = get_buffer_string(captured_messages)
|
||||
assert "[Tool result saved]" in formatted
|
||||
assert "Truncated" in formatted
|
||||
assert "END" not in formatted
|
||||
assert messages[2].content == large_result
|
||||
assert (_expected_tool_result_path(large_result), large_result) in backend.writes
|
||||
assert not any(write_path.startswith(VIRTUAL_PATH_CONVERSATION_HISTORY) for write_path, _content in backend.writes)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_wrap_model_call_does_not_sanitize_without_summary_trigger() -> None:
|
||||
backend = _MemoryBackend()
|
||||
messages = [
|
||||
*_tool_messages(),
|
||||
HumanMessage(content="新的问题"),
|
||||
]
|
||||
middleware = create_summary_middleware(
|
||||
model=_DummyModel(),
|
||||
trigger=("messages", 100),
|
||||
keep=("messages", 10),
|
||||
trim_tokens_to_summarize=None,
|
||||
)
|
||||
captured_messages: list | None = None
|
||||
|
||||
def handler(request: ModelRequest) -> ModelResponse:
|
||||
nonlocal captured_messages
|
||||
captured_messages = request.messages
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
middleware._backend_for_request = lambda _request: backend
|
||||
result = middleware.wrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert isinstance(result, ModelResponse)
|
||||
assert captured_messages is not None
|
||||
formatted = get_buffer_string(captured_messages)
|
||||
assert backend.writes == []
|
||||
assert "TOOL_RESULT_SHOULD_NOT_BE_SUMMARIZED" in formatted
|
||||
assert "[Tool result saved]" not in formatted
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_awrap_model_call_emits_completed_for_l1_without_summary(
|
||||
compression_events: list[dict],
|
||||
) -> None:
|
||||
backend = _MemoryBackend()
|
||||
large_result = "BEGIN\n" + ("raw result payload\n" * 200)
|
||||
messages = [
|
||||
HumanMessage(content="查资料"),
|
||||
AIMessage(content="", tool_calls=[{"id": "call-1", "name": "query_kb", "args": {}}]),
|
||||
ToolMessage(content=large_result, tool_call_id="call-1", name="query_kb"),
|
||||
HumanMessage(content="继续"),
|
||||
]
|
||||
middleware = YuxiSummarizationMiddleware(
|
||||
model=_RecordingModel(),
|
||||
backend=backend,
|
||||
trigger=("tokens", 500),
|
||||
keep=("messages", 2),
|
||||
token_counter=_content_char_counter,
|
||||
trim_tokens_to_summarize=None,
|
||||
l1_l2_trigger_ratio=100.0,
|
||||
)
|
||||
middleware._history_path_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
captured_messages: list | None = None
|
||||
|
||||
async def handler(request: ModelRequest) -> ModelResponse:
|
||||
nonlocal captured_messages
|
||||
captured_messages = request.messages
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = await middleware.awrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert not isinstance(result, ExtendedModelResponse)
|
||||
assert [event["status"] for event in compression_events] == ["started", "completed"]
|
||||
assert captured_messages is not None
|
||||
formatted = get_buffer_string(captured_messages)
|
||||
assert "[Tool result saved]" in formatted
|
||||
assert "Truncated" in formatted
|
||||
assert messages[2].content == large_result
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_wrap_model_call_truncates_large_write_file_args_only_in_l1_view() -> None:
|
||||
backend = _MemoryBackend()
|
||||
large_content = "x" * 5000
|
||||
raw_arguments = '{"file_path": "/tmp/a.txt", "content": "' + large_content + '"}'
|
||||
messages = [
|
||||
HumanMessage(content="写文件" + ("y" * 1000)),
|
||||
AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"id": "call-1",
|
||||
"name": "write_file",
|
||||
"args": {"file_path": "/tmp/a.txt", "content": large_content},
|
||||
}
|
||||
],
|
||||
additional_kwargs={
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"function": {"name": "write_file", "arguments": raw_arguments},
|
||||
}
|
||||
]
|
||||
},
|
||||
),
|
||||
ToolMessage(content="ok", tool_call_id="call-1", name="write_file"),
|
||||
HumanMessage(content="继续"),
|
||||
]
|
||||
middleware = YuxiSummarizationMiddleware(
|
||||
model=_RecordingModel(),
|
||||
backend=backend,
|
||||
trigger=("tokens", 500),
|
||||
keep=("messages", 2),
|
||||
token_counter=_content_char_counter,
|
||||
trim_tokens_to_summarize=None,
|
||||
l1_l2_trigger_ratio=100.0,
|
||||
tool_arg_max_length=100,
|
||||
)
|
||||
captured_messages: list | None = None
|
||||
|
||||
def handler(request: ModelRequest) -> ModelResponse:
|
||||
nonlocal captured_messages
|
||||
captured_messages = request.messages
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = middleware.wrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert not isinstance(result, ExtendedModelResponse)
|
||||
assert captured_messages is not None
|
||||
compact_ai = captured_messages[1]
|
||||
assert isinstance(compact_ai, AIMessage)
|
||||
assert compact_ai is not messages[1]
|
||||
assert compact_ai.tool_calls[0]["args"]["content"].endswith("...(argument truncated for context view)")
|
||||
provider_arguments = compact_ai.additional_kwargs["tool_calls"][0]["function"]["arguments"]
|
||||
assert provider_arguments.endswith("...(argument truncated for context view)")
|
||||
assert messages[1].tool_calls[0]["args"]["content"] == large_content
|
||||
assert messages[1].additional_kwargs["tool_calls"][0]["function"]["arguments"] == raw_arguments
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_wrap_model_call_offloads_tool_messages_outside_keep_window_when_summary_triggers() -> None:
|
||||
backend = _MemoryBackend()
|
||||
model = _RecordingModel()
|
||||
old_result = "BEGIN\n" + ("raw result payload\n" * 200)
|
||||
messages = [
|
||||
HumanMessage(content="查资料"),
|
||||
AIMessage(content="", tool_calls=[{"id": "call-1", "name": "query_kb", "args": {}}]),
|
||||
ToolMessage(content=old_result, tool_call_id="call-1", name="query_kb"),
|
||||
AIMessage(content="资料已整理"),
|
||||
HumanMessage(content="继续"),
|
||||
AIMessage(content="可以继续"),
|
||||
HumanMessage(content="新问题"),
|
||||
]
|
||||
middleware = YuxiSummarizationMiddleware(
|
||||
model=model,
|
||||
backend=backend,
|
||||
trigger=("tokens", 500),
|
||||
keep=("messages", 2),
|
||||
token_counter=_content_char_counter,
|
||||
trim_tokens_to_summarize=None,
|
||||
tool_result_offload_token_limit=1,
|
||||
l1_l2_trigger_ratio=0.01,
|
||||
)
|
||||
middleware._history_path_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
captured_messages: list | None = None
|
||||
|
||||
def handler(request: ModelRequest) -> ModelResponse:
|
||||
nonlocal captured_messages
|
||||
captured_messages = request.messages
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = middleware.wrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert isinstance(result, ExtendedModelResponse)
|
||||
assert len(model.prompts) == 1
|
||||
assert captured_messages is not None
|
||||
formatted = get_buffer_string(captured_messages)
|
||||
assert "[Tool result saved]" in model.prompts[0]
|
||||
assert "[Tool result saved]" not in formatted
|
||||
assert "raw result payload" not in formatted
|
||||
tool_result_write = (_expected_tool_result_path(old_result), old_result)
|
||||
assert backend.writes.count(tool_result_write) == 1
|
||||
assert any(write_path.startswith(VIRTUAL_PATH_CONVERSATION_HISTORY) for write_path, _content in backend.writes)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_l1_offload_uses_summary_tool_result_preview_limit_for_l2_summary() -> None:
|
||||
backend = _MemoryBackend()
|
||||
model = _RecordingModel()
|
||||
old_result = "BEGIN\n" + ("raw result payload\n" * 200) + "END"
|
||||
messages = [
|
||||
HumanMessage(content="查资料"),
|
||||
AIMessage(content="", tool_calls=[{"id": "call-1", "name": "query_kb", "args": {}}]),
|
||||
ToolMessage(content=old_result, tool_call_id="call-1", name="query_kb"),
|
||||
AIMessage(content="资料已整理"),
|
||||
HumanMessage(content="继续"),
|
||||
]
|
||||
middleware = YuxiSummarizationMiddleware(
|
||||
model=model,
|
||||
backend=backend,
|
||||
trigger=("tokens", 500),
|
||||
keep=("messages", 2),
|
||||
token_counter=_content_char_counter,
|
||||
trim_tokens_to_summarize=None,
|
||||
tool_result_offload_token_limit=None,
|
||||
l1_l2_trigger_ratio=0.01,
|
||||
)
|
||||
middleware._history_path_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
|
||||
result = middleware.wrap_model_call(
|
||||
_model_request(messages),
|
||||
lambda _request: ModelResponse(result=[AIMessage(content="ok")]),
|
||||
)
|
||||
|
||||
assert isinstance(result, ExtendedModelResponse)
|
||||
assert len(model.prompts) == 1
|
||||
assert "END" in model.prompts[0]
|
||||
assert backend.writes.count((_expected_tool_result_path(old_result), old_result)) == 1
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_summary_event_reuses_original_preserved_window_on_later_calls() -> None:
|
||||
backend = _MemoryBackend()
|
||||
old_result = "SAFE\n" + ("PRESERVED_TOOL_RESULT_SHOULD_STAY_INLINE\n" * 200)
|
||||
new_result = "NEW_TOOL_RESULT_MUST_STAY_INLINE"
|
||||
messages = [
|
||||
HumanMessage(content="查资料"),
|
||||
AIMessage(content="", tool_calls=[{"id": "call-old", "name": "query_kb", "args": {}}]),
|
||||
ToolMessage(content=old_result, tool_call_id="call-old", name="query_kb"),
|
||||
AIMessage(content="资料已整理"),
|
||||
HumanMessage(content="继续"),
|
||||
]
|
||||
middleware = YuxiSummarizationMiddleware(
|
||||
model=_RecordingModel(),
|
||||
backend=backend,
|
||||
trigger=("messages", 5),
|
||||
keep=("messages", 3),
|
||||
token_counter=_content_char_counter,
|
||||
trim_tokens_to_summarize=None,
|
||||
tool_result_offload_token_limit=1,
|
||||
l1_l2_trigger_ratio=0.01,
|
||||
)
|
||||
middleware._history_path_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
captured: list[str] = []
|
||||
|
||||
def handler(request: ModelRequest) -> ModelResponse:
|
||||
captured.append(get_buffer_string(request.messages))
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = middleware.wrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert isinstance(result, ExtendedModelResponse)
|
||||
assert "[Tool result saved]" in captured[-1]
|
||||
assert "Truncated" in captured[-1]
|
||||
|
||||
event = result.command.update["_summarization_event"]
|
||||
state_messages = [
|
||||
*messages,
|
||||
AIMessage(content="ok"),
|
||||
HumanMessage(content="继续使用新工具"),
|
||||
AIMessage(content="", tool_calls=[{"id": "call-new", "name": "query_kb", "args": {}}]),
|
||||
ToolMessage(content=new_result, tool_call_id="call-new", name="query_kb"),
|
||||
]
|
||||
middleware._lc_helper._trigger_clauses = [{"messages": 999}]
|
||||
later_request = ModelRequest(
|
||||
model=_DummyModel(),
|
||||
messages=state_messages,
|
||||
system_message=None,
|
||||
tools=[],
|
||||
runtime=SimpleNamespace(context={}, config={}),
|
||||
state={"messages": state_messages, "_summarization_event": event},
|
||||
)
|
||||
|
||||
later_result = middleware.wrap_model_call(later_request, handler)
|
||||
|
||||
assert isinstance(later_result, ModelResponse)
|
||||
assert "[Tool result saved]" not in captured[-1]
|
||||
assert "PRESERVED_TOOL_RESULT_SHOULD_STAY_INLINE" in captured[-1]
|
||||
assert new_result in captured[-1]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_create_summary_uses_sanitized_messages() -> None:
|
||||
backend = _MemoryBackend()
|
||||
model = _RecordingModel()
|
||||
middleware = YuxiSummarizationMiddleware(
|
||||
model=model,
|
||||
backend=backend,
|
||||
trigger=("messages", 3),
|
||||
keep=("messages", 1),
|
||||
trim_tokens_to_summarize=None,
|
||||
tool_result_offload_token_limit=0,
|
||||
)
|
||||
middleware._history_path_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
|
||||
l1_messages = middleware._sanitize_messages_for_l1(_tool_messages(), backend=backend)
|
||||
|
||||
assert middleware._create_summary(l1_messages) == "summary"
|
||||
|
||||
prompt = model.prompts[0]
|
||||
assert "Tool calls omitted from summary input" not in prompt
|
||||
assert "[Tool result saved]" in prompt
|
||||
assert "最终答案保留" in prompt
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_offload_history_uses_tool_messages_with_replaced_content() -> None:
|
||||
backend = _MemoryBackend()
|
||||
middleware = YuxiSummarizationMiddleware(
|
||||
model=_DummyModel(),
|
||||
backend=backend,
|
||||
trigger=("messages", 3),
|
||||
keep=("messages", 1),
|
||||
trim_tokens_to_summarize=None,
|
||||
tool_result_offload_token_limit=0,
|
||||
)
|
||||
middleware._history_path_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
|
||||
l1_messages = middleware._sanitize_messages_for_l1(_tool_messages(), backend=backend)
|
||||
path = middleware._offload_to_backend(backend, l1_messages)
|
||||
|
||||
assert path is not None
|
||||
assert backend.writes
|
||||
tool_result_path = _expected_tool_result_path("TOOL_RESULT_SHOULD_NOT_BE_SUMMARIZED")
|
||||
assert (tool_result_path, "TOOL_RESULT_SHOULD_NOT_BE_SUMMARIZED") in backend.writes
|
||||
history_content = next(content for write_path, content in backend.writes if write_path != tool_result_path)
|
||||
assert "Tool calls omitted from summary input" not in history_content
|
||||
assert "[Tool result saved]" in history_content
|
||||
assert "最终答案保留" in history_content
|
||||
assert f"Full output path: {tool_result_path}" in history_content
|
||||
assert "TOOL_RESULT_SHOULD_NOT_BE_SUMMARIZED" not in history_content
|
||||
|
||||
|
||||
def _make_compressing_middleware(backend: _MemoryBackend) -> tuple[YuxiSummarizationMiddleware, str]:
|
||||
large_result = "BEGIN\n" + ("raw result payload\n" * 200)
|
||||
middleware = YuxiSummarizationMiddleware(
|
||||
model=_RecordingModel(),
|
||||
backend=backend,
|
||||
trigger=("tokens", 500),
|
||||
keep=("messages", 3),
|
||||
token_counter=_content_char_counter,
|
||||
trim_tokens_to_summarize=None,
|
||||
tool_result_offload_token_limit=1,
|
||||
l1_l2_trigger_ratio=0.01,
|
||||
)
|
||||
middleware._history_path_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
middleware._backend_for_request = lambda _request: backend
|
||||
return middleware, large_result
|
||||
|
||||
|
||||
def _compressing_messages(large_result: str) -> list:
|
||||
return [
|
||||
HumanMessage(content="查资料"),
|
||||
AIMessage(content="", tool_calls=[{"id": "call-1", "name": "query_kb", "args": {}}]),
|
||||
ToolMessage(content=large_result, tool_call_id="call-1", name="query_kb"),
|
||||
AIMessage(content="资料已整理"),
|
||||
HumanMessage(content="继续"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_awrap_model_call_emits_started_and_completed_when_summary_triggers(
|
||||
compression_events: list[dict],
|
||||
) -> None:
|
||||
backend = _MemoryBackend()
|
||||
middleware, large_result = _make_compressing_middleware(backend)
|
||||
messages = _compressing_messages(large_result)
|
||||
|
||||
async def handler(request: ModelRequest) -> ModelResponse:
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = await middleware.awrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert isinstance(result, ExtendedModelResponse)
|
||||
statuses = [event["status"] for event in compression_events]
|
||||
assert statuses == ["started", "completed"]
|
||||
assert all(event["type"] == "yuxi.context_compression" for event in compression_events)
|
||||
completed = compression_events[-1]
|
||||
assert isinstance(completed.get("cutoff_index"), int)
|
||||
assert completed.get("file_path") is not None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_awrap_model_call_emits_nothing_when_summary_not_triggered(compression_events: list[dict]) -> None:
|
||||
backend = _MemoryBackend()
|
||||
middleware = create_summary_middleware(
|
||||
model=_DummyModel(),
|
||||
trigger=("messages", 100),
|
||||
keep=("messages", 10),
|
||||
trim_tokens_to_summarize=None,
|
||||
)
|
||||
middleware._backend_for_request = lambda _request: backend
|
||||
messages = [*_tool_messages(), HumanMessage(content="新的问题")]
|
||||
|
||||
async def handler(request: ModelRequest) -> ModelResponse:
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = await middleware.awrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert not isinstance(result, ExtendedModelResponse)
|
||||
assert compression_events == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_awrap_model_call_emits_started_when_overflow_falls_back_to_summary(
|
||||
compression_events: list[dict],
|
||||
) -> None:
|
||||
backend = _MemoryBackend()
|
||||
middleware, large_result = _make_compressing_middleware(backend)
|
||||
middleware._lc_helper.trigger = [("tokens", 100_000)]
|
||||
middleware._lc_helper._trigger_clauses = [{"tokens": 100_000}]
|
||||
messages = _compressing_messages(large_result)
|
||||
calls = 0
|
||||
|
||||
async def handler(request: ModelRequest) -> ModelResponse:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise ContextOverflowError("context overflow")
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = await middleware.awrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert isinstance(result, ExtendedModelResponse)
|
||||
assert calls == 2
|
||||
assert [event["status"] for event in compression_events] == ["started", "completed"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_awrap_model_call_falls_back_to_summary_when_l1_only_overflows(
|
||||
compression_events: list[dict],
|
||||
) -> None:
|
||||
backend = _MemoryBackend()
|
||||
middleware, large_result = _make_compressing_middleware(backend)
|
||||
middleware.l1_l2_trigger_ratio = 100.0
|
||||
messages = _compressing_messages(large_result)
|
||||
calls = 0
|
||||
|
||||
async def handler(request: ModelRequest) -> ModelResponse:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise ContextOverflowError("context overflow after l1")
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = await middleware.awrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert isinstance(result, ExtendedModelResponse)
|
||||
assert calls == 2
|
||||
assert [event["status"] for event in compression_events] == ["started", "completed"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
async def test_awrap_model_call_emits_failed_when_handler_raises_after_started(
|
||||
compression_events: list[dict],
|
||||
) -> None:
|
||||
backend = _MemoryBackend()
|
||||
middleware, large_result = _make_compressing_middleware(backend)
|
||||
messages = _compressing_messages(large_result)
|
||||
|
||||
async def handler(request: ModelRequest) -> ModelResponse:
|
||||
raise RuntimeError("model boom")
|
||||
|
||||
with pytest.raises(RuntimeError, match="model boom"):
|
||||
await middleware.awrap_model_call(_model_request(messages), handler)
|
||||
|
||||
statuses = [event["status"] for event in compression_events]
|
||||
assert statuses == ["started", "failed"]
|
||||
assert "model boom" in compression_events[-1]["error"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_wrap_model_call_emits_started_and_completed_sync(compression_events: list[dict]) -> None:
|
||||
backend = _MemoryBackend()
|
||||
middleware, large_result = _make_compressing_middleware(backend)
|
||||
messages = _compressing_messages(large_result)
|
||||
|
||||
def handler(request: ModelRequest) -> ModelResponse:
|
||||
return ModelResponse(result=[AIMessage(content="ok")])
|
||||
|
||||
result = middleware.wrap_model_call(_model_request(messages), handler)
|
||||
|
||||
assert isinstance(result, ExtendedModelResponse)
|
||||
statuses = [event["status"] for event in compression_events]
|
||||
assert statuses == ["started", "completed"]
|
||||
@@ -0,0 +1,98 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from langchain.agents.middleware.types import ExtendedModelResponse, ModelResponse
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
|
||||
from yuxi.agents.middlewares.token_usage import TokenUsageMiddleware
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_token_usage_middleware_records_request_and_state_tokens() -> None:
|
||||
middleware = TokenUsageMiddleware()
|
||||
tool_schema = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_docs",
|
||||
"description": "Search project documents.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
}
|
||||
request = SimpleNamespace(
|
||||
model=SimpleNamespace(profile={"max_input_tokens": 2000}),
|
||||
state={"messages": [HumanMessage(content="old message")]},
|
||||
messages=[
|
||||
HumanMessage(content="current message"),
|
||||
ToolMessage(content="tool result", tool_call_id="call_1"),
|
||||
],
|
||||
system_message=SystemMessage(content="system prompt"),
|
||||
tools=[tool_schema],
|
||||
runtime=SimpleNamespace(context=SimpleNamespace(summary_threshold=2)),
|
||||
)
|
||||
|
||||
async def handler(_request):
|
||||
return ModelResponse(
|
||||
result=[
|
||||
AIMessage(
|
||||
content="answer",
|
||||
usage_metadata={"input_tokens": 12, "output_tokens": 5, "total_tokens": 17},
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result = await middleware.awrap_model_call(request, handler)
|
||||
|
||||
assert isinstance(result, ExtendedModelResponse)
|
||||
token_usage = result.command.update["token_usage"]
|
||||
assert token_usage["state_message_count"] == 2
|
||||
assert token_usage["state_message_count_before_call"] == 1
|
||||
assert token_usage["llm_message_count"] == 2
|
||||
assert token_usage["llm_content_message_count"] == 1
|
||||
assert token_usage["llm_content_message_tokens"] > 0
|
||||
assert token_usage["llm_tool_message_count"] == 1
|
||||
assert token_usage["llm_tool_message_tokens"] > 0
|
||||
assert token_usage["state_messages_tokens"] >= token_usage["state_messages_tokens_before_call"]
|
||||
assert token_usage["llm_input_tokens"] >= token_usage["llm_messages_tokens"]
|
||||
assert token_usage["system_tokens"] > 0
|
||||
assert token_usage["tools_tokens"] > 0
|
||||
assert token_usage["tool_count"] == 1
|
||||
assert token_usage["context_window"] == 2000
|
||||
assert token_usage["remaining_context_tokens"] == 2000 - token_usage["llm_input_tokens"]
|
||||
assert token_usage["summary_trigger_tokens"] == 2048
|
||||
assert "summary_keep_tokens" not in token_usage
|
||||
assert token_usage["model_usage"] == {"input_tokens": 12, "output_tokens": 5, "total_tokens": 17}
|
||||
assert token_usage["estimate"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_token_usage_middleware_detects_effective_summary_message() -> None:
|
||||
middleware = TokenUsageMiddleware()
|
||||
summary_message = HumanMessage(
|
||||
content="conversation summary",
|
||||
additional_kwargs={"lc_source": "summarization"},
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
model=SimpleNamespace(profile={}),
|
||||
state={"messages": [HumanMessage(content="raw history")]},
|
||||
messages=[summary_message, HumanMessage(content="recent user turn")],
|
||||
system_message=None,
|
||||
tools=[],
|
||||
runtime=SimpleNamespace(context=SimpleNamespace(summary_threshold=100)),
|
||||
)
|
||||
|
||||
async def handler(_request):
|
||||
return ModelResponse(result=[AIMessage(content="answer")])
|
||||
|
||||
result = await middleware.awrap_model_call(request, handler)
|
||||
token_usage = result.command.update["token_usage"]
|
||||
|
||||
assert token_usage["summary_active"] is True
|
||||
assert token_usage["summary_message_tokens"] > 0
|
||||
assert token_usage["context_window"] is None
|
||||
assert token_usage["context_usage_ratio"] is None
|
||||
@@ -0,0 +1,153 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.knowledge.implementations.dify import DifyKB
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: dict):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict:
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
def __init__(self, response_payload: dict | None = None, raises: Exception | None = None, **kwargs):
|
||||
del kwargs
|
||||
self._response_payload = response_payload or {}
|
||||
self._raises = raises
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
return False
|
||||
|
||||
async def post(self, url: str, json: dict, headers: dict):
|
||||
assert "/datasets/" in url
|
||||
assert headers.get("Authorization", "").startswith("Bearer ")
|
||||
if self._raises:
|
||||
raise self._raises
|
||||
assert json["retrieval_model"]["search_method"] == "semantic_search"
|
||||
assert json["retrieval_model"]["top_k"] == 5
|
||||
assert json["retrieval_model"]["reranking_enable"] is False
|
||||
assert json["retrieval_model"]["score_threshold_enabled"] is True
|
||||
assert json["retrieval_model"]["score_threshold"] == 0.3
|
||||
return _FakeResponse(self._response_payload)
|
||||
|
||||
|
||||
def test_dify_create_params_config_and_validation():
|
||||
config = DifyKB.get_create_params_config()
|
||||
keys = [option["key"] for option in config["options"]]
|
||||
assert keys == ["dify_api_url", "dify_token", "dify_dataset_id"]
|
||||
assert all(option["required"] for option in config["options"])
|
||||
|
||||
params = DifyKB.normalize_additional_params(
|
||||
{
|
||||
"dify_api_url": " https://api.dify.ai/v1 ",
|
||||
"dify_token": " token ",
|
||||
"dify_dataset_id": " dataset-123 ",
|
||||
}
|
||||
)
|
||||
assert params == {
|
||||
"dify_api_url": "https://api.dify.ai/v1",
|
||||
"dify_token": "token",
|
||||
"dify_dataset_id": "dataset-123",
|
||||
}
|
||||
assert "chunk_preset_id" not in params
|
||||
|
||||
|
||||
def test_dify_validation_rejects_missing_or_invalid_params():
|
||||
with pytest.raises(ValueError, match="Dify 参数缺失"):
|
||||
DifyKB.normalize_additional_params({"dify_api_url": "https://api.dify.ai/v1"})
|
||||
|
||||
with pytest.raises(ValueError, match="必须以 /v1 结尾"):
|
||||
DifyKB.normalize_additional_params(
|
||||
{
|
||||
"dify_api_url": "https://api.dify.ai",
|
||||
"dify_token": "token",
|
||||
"dify_dataset_id": "dataset-123",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dify_kb_aquery_maps_records(monkeypatch, tmp_path):
|
||||
kb = DifyKB(str(tmp_path))
|
||||
slug = "kb_test_dify"
|
||||
kb.databases_meta[slug] = {
|
||||
"name": "dify-kb",
|
||||
"description": "test",
|
||||
"kb_type": "dify",
|
||||
"query_params": {
|
||||
"options": {
|
||||
"search_mode": "vector",
|
||||
"final_top_k": 5,
|
||||
"score_threshold_enabled": True,
|
||||
"similarity_threshold": 0.3,
|
||||
}
|
||||
},
|
||||
"metadata": {
|
||||
"dify_api_url": "https://api.dify.ai/v1",
|
||||
"dify_token": "token",
|
||||
"dify_dataset_id": "dataset-123",
|
||||
},
|
||||
}
|
||||
|
||||
payload = {
|
||||
"records": [
|
||||
{
|
||||
"score": 0.98,
|
||||
"segment": {
|
||||
"id": "seg-1",
|
||||
"position": 2,
|
||||
"content": "hello world",
|
||||
"document": {"id": "doc-1", "name": "Doc One"},
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.knowledge.implementations.dify.httpx.AsyncClient",
|
||||
lambda **kwargs: _FakeAsyncClient(response_payload=payload, **kwargs),
|
||||
)
|
||||
|
||||
result = await kb.aquery("hello", slug)
|
||||
assert len(result) == 1
|
||||
assert result[0]["content"] == "hello world"
|
||||
assert result[0]["score"] == 0.98
|
||||
assert result[0]["metadata"]["source"] == "Doc One"
|
||||
assert result[0]["metadata"]["file_id"] == "doc-1"
|
||||
assert result[0]["metadata"]["chunk_id"] == "seg-1"
|
||||
assert result[0]["metadata"]["chunk_index"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dify_kb_aquery_error_returns_empty(monkeypatch, tmp_path):
|
||||
kb = DifyKB(str(tmp_path))
|
||||
slug = "kb_test_dify_error"
|
||||
kb.databases_meta[slug] = {
|
||||
"name": "dify-kb",
|
||||
"description": "test",
|
||||
"kb_type": "dify",
|
||||
"query_params": {"options": {}},
|
||||
"metadata": {
|
||||
"dify_api_url": "https://api.dify.ai/v1",
|
||||
"dify_token": "token",
|
||||
"dify_dataset_id": "dataset-123",
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.knowledge.implementations.dify.httpx.AsyncClient",
|
||||
lambda **kwargs: _FakeAsyncClient(raises=RuntimeError("boom"), **kwargs),
|
||||
)
|
||||
|
||||
result = await kb.aquery("hello", slug)
|
||||
assert result == []
|
||||
@@ -0,0 +1,574 @@
|
||||
import types
|
||||
|
||||
import pytest
|
||||
from pymilvus import CollectionSchema, DataType, FieldSchema, Function, FunctionType
|
||||
|
||||
from yuxi.knowledge.base import FileStatus
|
||||
from yuxi.knowledge.chunking.ragflow_like.nlp import count_tokens
|
||||
from yuxi.knowledge.implementations.milvus import (
|
||||
CONTENT_ANALYZER_PARAMS,
|
||||
CONTENT_SPARSE_FIELD,
|
||||
VECTOR_METRIC_TYPE,
|
||||
MilvusKB,
|
||||
)
|
||||
|
||||
|
||||
class FakeHit:
|
||||
def __init__(self, content: str, distance: float):
|
||||
self.distance = distance
|
||||
self.entity = {
|
||||
"content": content,
|
||||
"chunk_id": "chunk-1",
|
||||
"file_id": "file-1",
|
||||
"chunk_index": 0,
|
||||
}
|
||||
|
||||
|
||||
class FakeCollection:
|
||||
def __init__(self, distance: float = 0.8):
|
||||
self.search_calls = []
|
||||
self.hybrid_calls = []
|
||||
self.insert_calls = []
|
||||
self.distance = distance
|
||||
|
||||
def search(self, **kwargs):
|
||||
self.search_calls.append(kwargs)
|
||||
return [[FakeHit("BM25 result", self.distance)]]
|
||||
|
||||
def hybrid_search(self, **kwargs):
|
||||
self.hybrid_calls.append(kwargs)
|
||||
return [[FakeHit("Hybrid result", self.distance)]]
|
||||
|
||||
def insert(self, entities):
|
||||
self.insert_calls.append(entities)
|
||||
|
||||
|
||||
def make_kb(collection: FakeCollection) -> MilvusKB:
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
kb.databases_meta = {"db": {"embedding_model_spec": "test-provider:test-embedding"}}
|
||||
kb._get_query_params = lambda kb_id: {}
|
||||
kb._get_embedding_function = lambda embedding_model_spec, **kwargs: lambda texts: [[0.1, 0.2] for _ in texts]
|
||||
|
||||
async def get_collection(kb_id: str):
|
||||
return collection
|
||||
|
||||
async def hydrate_chunk_sources(kb_id: str, chunks: list[dict]) -> None:
|
||||
for chunk in chunks:
|
||||
chunk["metadata"]["source"] = "demo.md"
|
||||
|
||||
kb._get_milvus_collection = get_collection
|
||||
kb._hydrate_chunk_sources = hydrate_chunk_sources
|
||||
return kb
|
||||
|
||||
|
||||
def make_file_record(**overrides):
|
||||
data = {
|
||||
"file_id": "file-1",
|
||||
"kb_id": "db",
|
||||
"parent_id": None,
|
||||
"filename": "demo.md",
|
||||
"file_type": "md",
|
||||
"path": "/tmp/demo.md",
|
||||
"minio_url": None,
|
||||
"markdown_file": "minio://parsed/db/file-1.md",
|
||||
"status": FileStatus.PARSED,
|
||||
"content_hash": None,
|
||||
"file_size": 0,
|
||||
"chunk_count": 0,
|
||||
"token_count": 0,
|
||||
"content_type": "file",
|
||||
"processing_params": {},
|
||||
"is_folder": False,
|
||||
"error_message": None,
|
||||
"created_by": None,
|
||||
"updated_by": None,
|
||||
"created_at": None,
|
||||
"updated_at": None,
|
||||
"original_filename": None,
|
||||
}
|
||||
data.update(overrides)
|
||||
return types.SimpleNamespace(**data)
|
||||
|
||||
|
||||
class FakeKnowledgeFileRepository:
|
||||
def __init__(self, records: dict[str, types.SimpleNamespace]):
|
||||
self.records = records
|
||||
self.update_calls = []
|
||||
self.conditional_update_calls = []
|
||||
self.deleted = []
|
||||
|
||||
async def get_by_file_id(self, file_id: str):
|
||||
return self.records.get(file_id)
|
||||
|
||||
async def update_fields_if_status(self, *, kb_id: str, file_id: str, allowed_statuses: set[str], data: dict):
|
||||
record = self.records.get(file_id)
|
||||
self.conditional_update_calls.append((kb_id, file_id, set(allowed_statuses), dict(data)))
|
||||
if record is None or record.kb_id != kb_id or record.status not in allowed_statuses:
|
||||
return None
|
||||
for key, value in data.items():
|
||||
setattr(record, key, value)
|
||||
return record
|
||||
|
||||
async def update_fields(self, *, file_id: str, data: dict, kb_id: str | None = None):
|
||||
record = self.records.get(file_id)
|
||||
if record is None or (kb_id and record.kb_id != kb_id):
|
||||
return None
|
||||
for key, value in data.items():
|
||||
setattr(record, key, value)
|
||||
self.update_calls.append((file_id, kb_id, dict(data)))
|
||||
return record
|
||||
|
||||
async def get_filenames_by_file_ids(self, *, kb_id: str, file_ids: list[str]):
|
||||
return {
|
||||
file_id: record.filename
|
||||
for file_id in file_ids
|
||||
if (record := self.records.get(file_id)) is not None and record.kb_id == kb_id
|
||||
}
|
||||
|
||||
async def list_file_ids_by_filename_contains(self, *, kb_id: str, filename_pattern: str, limit: int = 10_000):
|
||||
return [
|
||||
file_id
|
||||
for file_id, record in self.records.items()
|
||||
if record.kb_id == kb_id and filename_pattern.lower() in record.filename.lower()
|
||||
][:limit]
|
||||
|
||||
async def delete(self, file_id: str) -> None:
|
||||
self.deleted.append(file_id)
|
||||
self.records.pop(file_id, None)
|
||||
|
||||
|
||||
def patch_file_repository(monkeypatch, file_repo: FakeKnowledgeFileRepository) -> None:
|
||||
monkeypatch.setattr("yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository", lambda: file_repo)
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.milvus.KnowledgeFileRepository", lambda: file_repo)
|
||||
|
||||
|
||||
def make_chunk(index: int, content: str = "content") -> dict:
|
||||
return {
|
||||
"id": f"id-{index}",
|
||||
"chunk_id": f"chunk-{index}",
|
||||
"file_id": "file-1",
|
||||
"chunk_index": index,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
|
||||
def test_build_chunk_pg_records_preserves_extraction_result():
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
|
||||
records = kb._build_chunk_pg_records(
|
||||
"db",
|
||||
[
|
||||
{
|
||||
"chunk_id": "chunk-1",
|
||||
"file_id": "file-1",
|
||||
"chunk_index": 0,
|
||||
"content": "content",
|
||||
"extraction_result": {"entities": ["alpha"]},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert records[0]["extraction_result"] == {"entities": ["alpha"]}
|
||||
|
||||
|
||||
async def test_embed_and_store_chunks_batches_embedding_and_insert():
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
chunks = [make_chunk(index, content=f"text-{index}") for index in range(450)]
|
||||
embedding_calls = []
|
||||
store_calls = []
|
||||
|
||||
async def embedding_function(texts):
|
||||
embedding_calls.append(list(texts))
|
||||
return [[float(len(text))] for text in texts]
|
||||
|
||||
async def insert_chunks_to_stores(kb_id, file_id, collection, batch_chunks, embeddings, **kwargs):
|
||||
store_calls.append(
|
||||
{
|
||||
"kb_id": kb_id,
|
||||
"file_id": file_id,
|
||||
"chunks": list(batch_chunks),
|
||||
"embeddings": list(embeddings),
|
||||
"kwargs": kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
kb._insert_chunks_to_stores = insert_chunks_to_stores
|
||||
|
||||
await kb._embed_and_store_chunks(
|
||||
"db",
|
||||
"file-1",
|
||||
FakeCollection(),
|
||||
chunks,
|
||||
embedding_function,
|
||||
chunk_batch_size=200,
|
||||
)
|
||||
|
||||
assert [len(call) for call in embedding_calls] == [200, 200, 50]
|
||||
assert [len(call["chunks"]) for call in store_calls] == [200, 200, 50]
|
||||
assert store_calls[0]["chunks"][0]["chunk_id"] == "chunk-0"
|
||||
assert store_calls[1]["chunks"][0]["chunk_id"] == "chunk-200"
|
||||
assert store_calls[2]["chunks"][0]["chunk_id"] == "chunk-400"
|
||||
assert all(call["kwargs"] == {} for call in store_calls)
|
||||
|
||||
|
||||
def test_calculate_chunk_stats_counts_chunks_and_tokens():
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
chunks = [make_chunk(0, content="alpha beta"), make_chunk(1, content="中文")]
|
||||
|
||||
stats = kb._calculate_chunk_stats(chunks)
|
||||
|
||||
assert stats == {
|
||||
"chunk_count": 2,
|
||||
"token_count": count_tokens("alpha beta") + count_tokens("中文"),
|
||||
}
|
||||
|
||||
|
||||
async def test_index_file_persists_chunk_stats(monkeypatch):
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
kb.databases_meta = {"db": {"embedding_model_spec": "test-provider:test-embedding", "metadata": {}}}
|
||||
file_repo = FakeKnowledgeFileRepository({"file-1": make_file_record()})
|
||||
patch_file_repository(monkeypatch, file_repo)
|
||||
collection = FakeCollection()
|
||||
deleted_files = []
|
||||
store_calls = []
|
||||
refreshed_kbs = []
|
||||
chunks = [make_chunk(0, content="alpha beta"), make_chunk(1, content="中文")]
|
||||
|
||||
async def get_collection(kb_id):
|
||||
return collection
|
||||
|
||||
async def read_markdown(path):
|
||||
return "# demo"
|
||||
|
||||
async def embedding_function(texts):
|
||||
return [[0.1, 0.2] for _ in texts]
|
||||
|
||||
async def delete_file_chunks_only(kb_id, file_id):
|
||||
deleted_files.append((kb_id, file_id))
|
||||
|
||||
async def embed_and_store_chunks(kb_id, file_id, collection_arg, chunk_records, embedding_fn):
|
||||
store_calls.append((kb_id, file_id, collection_arg, list(chunk_records), embedding_fn))
|
||||
|
||||
async def refresh_database_stats(kb_id):
|
||||
refreshed_kbs.append(kb_id)
|
||||
return {}
|
||||
|
||||
kb._get_milvus_collection = get_collection
|
||||
kb._read_markdown_from_minio = read_markdown
|
||||
kb._split_text_into_chunks = lambda text, file_id, filename, params: chunks
|
||||
kb._get_embedding_function = lambda embedding_model_spec: embedding_function
|
||||
kb.delete_file_chunks_only = delete_file_chunks_only
|
||||
kb._embed_and_store_chunks = embed_and_store_chunks
|
||||
kb.refresh_database_stats = refresh_database_stats
|
||||
|
||||
result = await kb.index_file("db", "file-1", operator_id="user-1", params={})
|
||||
|
||||
assert deleted_files == [("db", "file-1")]
|
||||
assert len(store_calls) == 1
|
||||
assert [chunk["chunk_id"] for chunk in store_calls[0][3]] == ["chunk-0", "chunk-1"]
|
||||
assert result["status"] == FileStatus.INDEXED
|
||||
assert result["chunk_count"] == 2
|
||||
assert result["token_count"] == count_tokens("alpha beta") + count_tokens("中文")
|
||||
assert file_repo.records["file-1"].chunk_count == result["chunk_count"]
|
||||
assert file_repo.conditional_update_calls[0][3]["status"] == FileStatus.INDEXING
|
||||
assert file_repo.update_calls[-1][2]["status"] == FileStatus.INDEXED
|
||||
assert refreshed_kbs == ["db"]
|
||||
|
||||
|
||||
async def test_delete_file_chunks_only_resets_file_stats(monkeypatch):
|
||||
repos = []
|
||||
|
||||
class FakeChunkRepo:
|
||||
def __init__(self):
|
||||
self.delete_calls = []
|
||||
repos.append(self)
|
||||
|
||||
async def count_graph_indexed_by_file_id(self, file_id):
|
||||
return 0
|
||||
|
||||
async def delete_by_file_id(self, file_id):
|
||||
self.delete_calls.append(file_id)
|
||||
return 2
|
||||
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.milvus.KnowledgeChunkRepository", FakeChunkRepo)
|
||||
file_repo = FakeKnowledgeFileRepository(
|
||||
{"file-1": make_file_record(chunk_count=2, token_count=10, status=FileStatus.INDEXED)}
|
||||
)
|
||||
patch_file_repository(monkeypatch, file_repo)
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
refreshed_kbs = []
|
||||
|
||||
async def get_collection(kb_id):
|
||||
return None
|
||||
|
||||
async def refresh_database_stats(kb_id):
|
||||
refreshed_kbs.append(kb_id)
|
||||
return {}
|
||||
|
||||
kb._get_milvus_collection = get_collection
|
||||
kb.refresh_database_stats = refresh_database_stats
|
||||
|
||||
await kb.delete_file_chunks_only("db", "file-1")
|
||||
|
||||
assert repos[0].delete_calls == ["file-1"]
|
||||
assert file_repo.records["file-1"].chunk_count == 0
|
||||
assert file_repo.records["file-1"].token_count == 0
|
||||
assert file_repo.update_calls == [("file-1", "db", {"chunk_count": 0, "token_count": 0})]
|
||||
assert refreshed_kbs == ["db"]
|
||||
|
||||
|
||||
async def test_insert_chunks_to_stores_inserts_current_batch(monkeypatch):
|
||||
repos = []
|
||||
|
||||
class FakeChunkRepo:
|
||||
def __init__(self):
|
||||
self.upsert_calls = []
|
||||
self.delete_calls = []
|
||||
repos.append(self)
|
||||
|
||||
async def batch_upsert(self, chunks):
|
||||
self.upsert_calls.append(chunks)
|
||||
return []
|
||||
|
||||
async def delete_by_file_id(self, file_id):
|
||||
self.delete_calls.append(file_id)
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.milvus.KnowledgeChunkRepository", FakeChunkRepo)
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
collection = FakeCollection()
|
||||
chunks = [make_chunk(index) for index in range(3)]
|
||||
embeddings = [[0.1, 0.2] for _ in chunks]
|
||||
|
||||
await kb._insert_chunks_to_stores("db", "file-1", collection, chunks, embeddings)
|
||||
|
||||
assert len(collection.insert_calls) == 1
|
||||
assert collection.insert_calls[0][0] == ["id-0", "id-1", "id-2"]
|
||||
assert collection.insert_calls[0][5] == embeddings
|
||||
assert len(repos[0].upsert_calls) == 1
|
||||
assert [record["chunk_id"] for record in repos[0].upsert_calls[0]] == ["chunk-0", "chunk-1", "chunk-2"]
|
||||
|
||||
|
||||
async def test_insert_chunks_to_stores_rolls_back_file_when_milvus_insert_fails(monkeypatch):
|
||||
repos = []
|
||||
|
||||
class FakeChunkRepo:
|
||||
def __init__(self):
|
||||
self.upsert_calls = []
|
||||
self.delete_calls = []
|
||||
repos.append(self)
|
||||
|
||||
async def batch_upsert(self, chunks):
|
||||
self.upsert_calls.append(chunks)
|
||||
return []
|
||||
|
||||
async def delete_by_file_id(self, file_id):
|
||||
self.delete_calls.append(file_id)
|
||||
return 0
|
||||
|
||||
class FailingCollection(FakeCollection):
|
||||
def insert(self, entities):
|
||||
super().insert(entities)
|
||||
raise RuntimeError("milvus boom")
|
||||
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.milvus.KnowledgeChunkRepository", FakeChunkRepo)
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
collection = FailingCollection()
|
||||
milvus_delete_calls = []
|
||||
|
||||
async def delete_file_chunks_from_milvus(collection_arg, file_id):
|
||||
milvus_delete_calls.append((collection_arg, file_id))
|
||||
|
||||
kb._delete_file_chunks_from_milvus = delete_file_chunks_from_milvus
|
||||
chunks = [make_chunk(index) for index in range(2)]
|
||||
embeddings = [[0.1, 0.2] for _ in chunks]
|
||||
|
||||
with pytest.raises(RuntimeError, match="milvus boom"):
|
||||
await kb._insert_chunks_to_stores("db", "file-1", collection, chunks, embeddings)
|
||||
|
||||
assert repos[0].delete_calls == ["file-1"]
|
||||
assert milvus_delete_calls == [(collection, "file-1")]
|
||||
|
||||
|
||||
async def test_update_content_uses_streaming_chunk_store(monkeypatch):
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
kb.databases_meta = {"db": {"embedding_model_spec": "test-provider:test-embedding", "metadata": {}}}
|
||||
file_repo = FakeKnowledgeFileRepository(
|
||||
{"file-1": make_file_record(markdown_file=None, status=FileStatus.INDEXED)}
|
||||
)
|
||||
patch_file_repository(monkeypatch, file_repo)
|
||||
collection = FakeCollection()
|
||||
refreshed_kbs = []
|
||||
deleted_files = []
|
||||
store_calls = []
|
||||
|
||||
async def get_collection(kb_id):
|
||||
return collection
|
||||
|
||||
async def forbidden_embedding(texts):
|
||||
raise AssertionError("update_content should not embed the whole file directly")
|
||||
|
||||
async def refresh_database_stats(kb_id):
|
||||
refreshed_kbs.append(kb_id)
|
||||
return {}
|
||||
|
||||
async def delete_file_chunks_only(kb_id, file_id):
|
||||
deleted_files.append((kb_id, file_id))
|
||||
|
||||
async def embed_and_store_chunks(kb_id, file_id, collection_arg, chunks, embedding_function):
|
||||
store_calls.append((kb_id, file_id, collection_arg, list(chunks), embedding_function))
|
||||
|
||||
async def parse_file(source, params):
|
||||
return "# markdown"
|
||||
|
||||
kb._get_milvus_collection = get_collection
|
||||
kb._get_embedding_function = lambda embedding_model_spec: forbidden_embedding
|
||||
kb.refresh_database_stats = refresh_database_stats
|
||||
kb._split_text_into_chunks = lambda text, file_id, filename, params: [make_chunk(0), make_chunk(1)]
|
||||
kb.delete_file_chunks_only = delete_file_chunks_only
|
||||
kb._embed_and_store_chunks = embed_and_store_chunks
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.milvus.Parser.aparse", parse_file)
|
||||
|
||||
result = await kb.update_content("db", ["file-1"])
|
||||
|
||||
assert deleted_files == [("db", "file-1")]
|
||||
assert len(store_calls) == 1
|
||||
assert store_calls[0][2] is collection
|
||||
assert [chunk["chunk_id"] for chunk in store_calls[0][3]] == ["chunk-0", "chunk-1"]
|
||||
assert store_calls[0][4] is forbidden_embedding
|
||||
assert result[0]["status"] == FileStatus.INDEXED
|
||||
assert file_repo.records["file-1"].status == FileStatus.INDEXED
|
||||
assert file_repo.update_calls[0][2]["status"] == FileStatus.INDEXING
|
||||
assert file_repo.update_calls[-1][2]["status"] == FileStatus.INDEXED
|
||||
assert refreshed_kbs == ["db"]
|
||||
|
||||
|
||||
async def test_keyword_mode_uses_milvus_bm25_search():
|
||||
collection = FakeCollection()
|
||||
kb = make_kb(collection)
|
||||
|
||||
chunks = await kb.aquery(
|
||||
"alpha beta",
|
||||
"db",
|
||||
search_mode="keyword",
|
||||
bm25_top_k=7,
|
||||
bm25_drop_ratio_search=0.2,
|
||||
)
|
||||
|
||||
assert chunks[0]["content"] == "BM25 result"
|
||||
assert chunks[0]["bm25_score"] == 0.8
|
||||
search_call = collection.search_calls[0]
|
||||
assert search_call["data"] == ["alpha beta"]
|
||||
assert search_call["anns_field"] == CONTENT_SPARSE_FIELD
|
||||
assert search_call["param"] == {
|
||||
"metric_type": "BM25",
|
||||
"params": {"drop_ratio_search": 0.2},
|
||||
}
|
||||
assert search_call["limit"] == 7
|
||||
|
||||
|
||||
async def test_vector_mode_ignores_metric_type_override():
|
||||
collection = FakeCollection()
|
||||
kb = make_kb(collection)
|
||||
|
||||
chunks = await kb.aquery("vector query", "db", search_mode="vector", metric_type="L2")
|
||||
|
||||
assert chunks[0]["content"] == "BM25 result"
|
||||
search_call = collection.search_calls[0]
|
||||
assert search_call["anns_field"] == "embedding"
|
||||
assert search_call["param"]["metric_type"] == VECTOR_METRIC_TYPE
|
||||
|
||||
|
||||
async def test_hybrid_mode_uses_milvus_native_hybrid_search():
|
||||
collection = FakeCollection()
|
||||
kb = make_kb(collection)
|
||||
|
||||
chunks = await kb.aquery(
|
||||
"hybrid query",
|
||||
"db",
|
||||
search_mode="hybrid",
|
||||
final_top_k=3,
|
||||
bm25_top_k=8,
|
||||
vector_weight=0.6,
|
||||
bm25_weight=0.4,
|
||||
)
|
||||
|
||||
assert chunks[0]["content"] == "Hybrid result"
|
||||
assert chunks[0]["hybrid_score"] == 0.8
|
||||
hybrid_call = collection.hybrid_calls[0]
|
||||
assert hybrid_call["limit"] == 3
|
||||
assert hybrid_call["rerank"]._weights == [0.6, 0.4]
|
||||
|
||||
vector_request, bm25_request = hybrid_call["reqs"]
|
||||
assert vector_request.anns_field == "embedding"
|
||||
assert vector_request.data == [[0.1, 0.2]]
|
||||
assert vector_request.param["metric_type"] == VECTOR_METRIC_TYPE
|
||||
assert bm25_request.anns_field == CONTENT_SPARSE_FIELD
|
||||
assert bm25_request.data == ["hybrid query"]
|
||||
assert bm25_request.limit == 8
|
||||
assert bm25_request.param["metric_type"] == "BM25"
|
||||
|
||||
|
||||
async def test_hybrid_mode_filters_scores_below_similarity_threshold():
|
||||
collection = FakeCollection(distance=0.1)
|
||||
kb = make_kb(collection)
|
||||
|
||||
chunks = await kb.aquery(
|
||||
"hybrid query",
|
||||
"db",
|
||||
search_mode="hybrid",
|
||||
final_top_k=3,
|
||||
similarity_threshold=0.2,
|
||||
)
|
||||
|
||||
assert chunks == []
|
||||
|
||||
|
||||
def test_query_params_config_uses_bm25_parameters():
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
|
||||
config = kb.get_query_params_config("db")
|
||||
|
||||
option_keys = {option["key"] for option in config["options"]}
|
||||
assert "keyword_top_k" not in option_keys
|
||||
assert "metric_type" not in option_keys
|
||||
assert {
|
||||
"bm25_top_k",
|
||||
"vector_weight",
|
||||
"bm25_weight",
|
||||
"bm25_drop_ratio_search",
|
||||
} <= option_keys
|
||||
|
||||
search_mode = next(option for option in config["options"] if option["key"] == "search_mode")
|
||||
descriptions = {option["value"]: option["description"] for option in search_mode["options"]}
|
||||
assert "BM25" in descriptions["keyword"]
|
||||
assert "BM25" in descriptions["hybrid"]
|
||||
|
||||
|
||||
def test_collection_supports_bm25_requires_analyzed_content_sparse_field_and_function():
|
||||
kb = MilvusKB.__new__(MilvusKB)
|
||||
schema = CollectionSchema(
|
||||
fields=[
|
||||
FieldSchema(name="id", dtype=DataType.VARCHAR, max_length=100, is_primary=True),
|
||||
FieldSchema(
|
||||
name="content",
|
||||
dtype=DataType.VARCHAR,
|
||||
max_length=65535,
|
||||
enable_analyzer=True,
|
||||
analyzer_params=CONTENT_ANALYZER_PARAMS,
|
||||
),
|
||||
FieldSchema(name=CONTENT_SPARSE_FIELD, dtype=DataType.SPARSE_FLOAT_VECTOR),
|
||||
],
|
||||
functions=[
|
||||
Function(
|
||||
name="content_bm25",
|
||||
input_field_names=["content"],
|
||||
output_field_names=[CONTENT_SPARSE_FIELD],
|
||||
function_type=FunctionType.BM25,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
collection = type("Collection", (), {"schema": schema})()
|
||||
|
||||
assert kb._collection_supports_bm25(collection)
|
||||
@@ -0,0 +1,198 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.knowledge.implementations.notion import NOTION_DEFAULT_VERSION, NotionAPIError, NotionKB
|
||||
|
||||
|
||||
PAGE_ID = "page-1"
|
||||
DATA_SOURCE_ID = "ds-1"
|
||||
|
||||
|
||||
PAGE = {
|
||||
"object": "page",
|
||||
"id": PAGE_ID,
|
||||
"url": "https://www.notion.so/page-1",
|
||||
"created_time": "2026-01-01T00:00:00.000Z",
|
||||
"last_edited_time": "2026-01-02T00:00:00.000Z",
|
||||
"parent": {"type": "data_source_id", "data_source_id": DATA_SOURCE_ID},
|
||||
"properties": {
|
||||
"Name": {"type": "title", "title": [{"plain_text": "Reasoning Paper"}]},
|
||||
"Abstract": {"type": "rich_text", "rich_text": [{"plain_text": "Chain of thought reasoning"}]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
BLOCKS = {
|
||||
PAGE_ID: [
|
||||
{
|
||||
"object": "block",
|
||||
"id": "block-1",
|
||||
"type": "paragraph",
|
||||
"has_children": False,
|
||||
"paragraph": {"rich_text": [{"plain_text": "This page discusses reasoning models."}]},
|
||||
},
|
||||
{
|
||||
"object": "block",
|
||||
"id": "block-2",
|
||||
"type": "heading_2",
|
||||
"has_children": False,
|
||||
"heading_2": {"rich_text": [{"plain_text": "Evaluation"}]},
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
class _FakeNotionClient:
|
||||
def __init__(self, token: str, notion_version: str) -> None:
|
||||
self.token = token
|
||||
self.notion_version = notion_version
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
return False
|
||||
|
||||
async def search_pages(self, query_text: str, limit: int) -> list[dict]:
|
||||
del query_text, limit
|
||||
return [PAGE]
|
||||
|
||||
async def query_data_source(self, data_source_id: str, limit: int) -> list[dict]:
|
||||
del limit
|
||||
assert data_source_id == DATA_SOURCE_ID
|
||||
return [PAGE]
|
||||
|
||||
async def retrieve_page(self, page_id: str) -> dict:
|
||||
assert page_id == PAGE_ID
|
||||
return PAGE
|
||||
|
||||
async def retrieve_block_children(self, block_id: str, limit: int) -> list[dict]:
|
||||
del limit
|
||||
return BLOCKS.get(block_id, [])
|
||||
|
||||
|
||||
class _FailingNotionClient(_FakeNotionClient):
|
||||
async def search_pages(self, query_text: str, limit: int) -> list[dict]:
|
||||
del query_text, limit
|
||||
raise NotionAPIError("boom")
|
||||
|
||||
|
||||
class _UnknownParentNotionClient(_FakeNotionClient):
|
||||
async def retrieve_page(self, page_id: str) -> dict:
|
||||
page = await super().retrieve_page(page_id)
|
||||
return {**page, "parent": {"type": "page_id", "page_id": "other-page"}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def notion_kb(tmp_path):
|
||||
kb = NotionKB(str(tmp_path))
|
||||
kb.databases_meta["kb_notion"] = {
|
||||
"name": "notion-kb",
|
||||
"description": "test",
|
||||
"kb_type": "notion",
|
||||
"query_params": {"options": {}},
|
||||
"metadata": {
|
||||
"notion_token": "token",
|
||||
"notion_data_source_id": DATA_SOURCE_ID,
|
||||
"notion_version": NOTION_DEFAULT_VERSION,
|
||||
},
|
||||
}
|
||||
return kb
|
||||
|
||||
|
||||
def test_notion_create_params_config_and_validation(monkeypatch):
|
||||
monkeypatch.delenv("NOTION_TOKEN", raising=False)
|
||||
monkeypatch.delenv("NOTION_API_KEY", raising=False)
|
||||
|
||||
config = NotionKB.get_create_params_config()
|
||||
keys = [option["key"] for option in config["options"]]
|
||||
assert keys == ["notion_token", "notion_data_source_id", "notion_version"]
|
||||
|
||||
params = NotionKB.normalize_additional_params(
|
||||
{
|
||||
"notion_token": " token ",
|
||||
"notion_data_source_id": " ds-1 ",
|
||||
"notion_version": " 2026-03-11 ",
|
||||
}
|
||||
)
|
||||
assert params == {
|
||||
"notion_token": "token",
|
||||
"notion_data_source_id": DATA_SOURCE_ID,
|
||||
"notion_version": NOTION_DEFAULT_VERSION,
|
||||
}
|
||||
assert "chunk_preset_id" not in params
|
||||
|
||||
|
||||
def test_notion_validation_accepts_env_token(monkeypatch):
|
||||
monkeypatch.setenv("NOTION_TOKEN", "env-token")
|
||||
params = NotionKB.normalize_additional_params({"notion_data_source_id": DATA_SOURCE_ID})
|
||||
assert params["notion_token"] == ""
|
||||
assert params["notion_data_source_id"] == DATA_SOURCE_ID
|
||||
|
||||
|
||||
def test_notion_validation_rejects_missing_params(monkeypatch):
|
||||
monkeypatch.delenv("NOTION_TOKEN", raising=False)
|
||||
monkeypatch.delenv("NOTION_API_KEY", raising=False)
|
||||
|
||||
with pytest.raises(ValueError, match="notion_data_source_id"):
|
||||
NotionKB.normalize_additional_params({"notion_token": "token"})
|
||||
|
||||
with pytest.raises(ValueError, match="notion_token"):
|
||||
NotionKB.normalize_additional_params({"notion_data_source_id": DATA_SOURCE_ID})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notion_kb_aquery_maps_pages(monkeypatch, notion_kb):
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _FakeNotionClient)
|
||||
|
||||
result = await notion_kb.aquery("reasoning", "kb_notion")
|
||||
|
||||
assert len(result) == 1
|
||||
assert "reasoning" in result[0]["content"].lower()
|
||||
assert result[0]["score"] > 0
|
||||
assert result[0]["metadata"]["source"] == "Reasoning Paper"
|
||||
assert result[0]["metadata"]["file_id"] == PAGE_ID
|
||||
assert result[0]["metadata"]["chunk_id"].startswith(f"{PAGE_ID}:")
|
||||
assert result[0]["metadata"]["notion_url"] == "https://www.notion.so/page-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notion_open_file_content_uses_page_markdown(monkeypatch, notion_kb):
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _FakeNotionClient)
|
||||
|
||||
result = await notion_kb.open_file_content("kb_notion", PAGE_ID, offset=0, limit=3)
|
||||
|
||||
assert result["start_line"] == 1
|
||||
assert result["end_line"] == 3
|
||||
assert result["has_more_after"] is True
|
||||
assert "Reasoning Paper" in result["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notion_find_file_content_uses_page_markdown(monkeypatch, notion_kb):
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _FakeNotionClient)
|
||||
|
||||
result = await notion_kb.find_file_content("kb_notion", PAGE_ID, ["models"], window_size=4)
|
||||
|
||||
assert result["match_mode"] == "keyword"
|
||||
assert result["total_matches"] == 1
|
||||
assert result["windows"]
|
||||
assert "reasoning models" in result["windows"][0]["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notion_open_file_content_rejects_unknown_parent(monkeypatch, notion_kb):
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _UnknownParentNotionClient)
|
||||
|
||||
with pytest.raises(ValueError, match="不属于当前 Data Source"):
|
||||
await notion_kb.open_file_content("kb_notion", PAGE_ID)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notion_kb_aquery_error_returns_empty(monkeypatch, notion_kb):
|
||||
monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _FailingNotionClient)
|
||||
|
||||
result = await notion_kb.aquery("reasoning", "kb_notion")
|
||||
|
||||
assert result == []
|
||||
@@ -0,0 +1,364 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.getcwd())
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown
|
||||
from yuxi.knowledge.chunking.ragflow_like.nlp import bullets_category, count_tokens
|
||||
from yuxi.knowledge.chunking.ragflow_like.utils.semantic_utils import split_sentences_chinese
|
||||
from yuxi.knowledge.chunking.ragflow_like.presets import (
|
||||
CHUNK_ENGINE_VERSION,
|
||||
CHUNK_PRESET_IDS,
|
||||
CHUNK_PRESETS,
|
||||
get_chunk_preset_options,
|
||||
get_default_chunk_parser_config,
|
||||
map_to_internal_parser_id,
|
||||
resolve_chunk_processing_params,
|
||||
)
|
||||
from yuxi.knowledge.utils.kb_utils import resolve_processing_params, sanitize_processing_params
|
||||
|
||||
|
||||
def test_general_maps_to_naive() -> None:
|
||||
assert map_to_internal_parser_id("general") == "naive"
|
||||
|
||||
|
||||
def test_resolve_chunk_processing_params_priority() -> None:
|
||||
resolved = resolve_chunk_processing_params(
|
||||
kb_additional_params={
|
||||
"chunk_preset_id": "book",
|
||||
"chunk_parser_config": {"chunk_token_num": 300, "delimiter": "\\n"},
|
||||
},
|
||||
file_processing_params={
|
||||
"chunk_preset_id": "qa",
|
||||
"chunk_parser_config": {"delimiter": "###", "overlapped_percent": 5},
|
||||
},
|
||||
request_params={
|
||||
"chunk_preset_id": "laws",
|
||||
"chunk_parser_config": {"chunk_token_num": 666},
|
||||
},
|
||||
)
|
||||
|
||||
assert resolved["chunk_preset_id"] == "laws"
|
||||
assert resolved["chunk_engine_version"] == CHUNK_ENGINE_VERSION
|
||||
assert resolved["chunk_parser_config"] == {
|
||||
"chunk_token_num": 666,
|
||||
"delimiter": "###",
|
||||
"overlapped_percent": 5,
|
||||
}
|
||||
|
||||
|
||||
def test_resolve_chunk_processing_params_returns_only_nested_keys() -> None:
|
||||
resolved = resolve_chunk_processing_params(
|
||||
kb_additional_params={"chunk_parser_config": {"chunk_token_num": 300}},
|
||||
file_processing_params={},
|
||||
request_params={},
|
||||
)
|
||||
|
||||
assert resolved["chunk_parser_config"] == {"chunk_token_num": 300}
|
||||
assert resolved["chunk_preset_id"] == "general"
|
||||
assert resolved["chunk_engine_version"] == CHUNK_ENGINE_VERSION
|
||||
assert len(resolved) == 3
|
||||
|
||||
|
||||
def test_qa_chunking_from_markdown_headings() -> None:
|
||||
content = """
|
||||
# 问题一
|
||||
这是答案一。
|
||||
|
||||
## 子问题
|
||||
这是答案二。
|
||||
""".strip()
|
||||
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_1",
|
||||
filename="faq.md",
|
||||
processing_params={"chunk_preset_id": "qa", "chunk_parser_config": {}},
|
||||
)
|
||||
|
||||
assert len(chunks) >= 1
|
||||
assert "问题:" in chunks[0]["content"]
|
||||
assert "回答:" in chunks[0]["content"]
|
||||
|
||||
|
||||
def test_chunk_records_include_reserved_position_fields() -> None:
|
||||
content = "第一段内容。\n\n第二段内容。"
|
||||
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_pos",
|
||||
filename="pos.md",
|
||||
processing_params={
|
||||
"chunk_preset_id": "separator",
|
||||
"chunk_parser_config": {"delimiter": "\\n\\n"},
|
||||
},
|
||||
)
|
||||
|
||||
assert chunks[0]["start_char_pos"] == 0
|
||||
assert chunks[0]["end_char_pos"] == len("第一段内容。")
|
||||
assert chunks[0]["start_token_pos"] is None
|
||||
assert chunks[0]["end_token_pos"] is None
|
||||
assert "start_char_pos" in chunks[1]
|
||||
|
||||
|
||||
def test_book_chunking_hierarchical_merge() -> None:
|
||||
content = """
|
||||
第一章 总则
|
||||
第一节 适用范围
|
||||
本规范适用于测试场景。
|
||||
第二节 基本原则
|
||||
应当遵循最小改动原则。
|
||||
""".strip()
|
||||
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_2",
|
||||
filename="book.txt",
|
||||
processing_params={"chunk_preset_id": "book", "chunk_parser_config": {"chunk_token_num": 256}},
|
||||
)
|
||||
|
||||
assert len(chunks) >= 1
|
||||
assert any("第一章" in ck["content"] for ck in chunks)
|
||||
|
||||
|
||||
def test_book_chunking_should_apply_overlength_protection() -> None:
|
||||
content = "\n".join(
|
||||
[
|
||||
"第一章 总则",
|
||||
"第一节 适用范围",
|
||||
"超长正文" * 1200,
|
||||
"第二节 基本原则",
|
||||
"应当遵循最小改动原则。",
|
||||
]
|
||||
)
|
||||
max_chunk_tokens = 180
|
||||
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_book_long",
|
||||
filename="book.txt",
|
||||
processing_params={
|
||||
"chunk_preset_id": "book",
|
||||
"chunk_parser_config": {"chunk_token_num": max_chunk_tokens, "delimiter": "\\n"},
|
||||
},
|
||||
)
|
||||
|
||||
assert len(chunks) > 1
|
||||
assert max(count_tokens(ck["content"]) for ck in chunks) <= max_chunk_tokens
|
||||
|
||||
|
||||
def test_split_sentences_chinese_should_keep_quote_boundary() -> None:
|
||||
text = '他说:“你好。”然后问:“你在吗?”最后结束!'
|
||||
sentences = split_sentences_chinese(text)
|
||||
|
||||
assert sentences == ["他说:“你好。”", "然后问:“你在吗?”", "最后结束!"]
|
||||
|
||||
|
||||
def test_markdown_heading_has_higher_weight_in_bullet_category() -> None:
|
||||
sections = [
|
||||
"# 3.2 个人所得项目及计税、申报方式概括",
|
||||
"一、关于季节工、临时工等费用税前扣除问题,以下规定继续执行。",
|
||||
"二、根据现行规定,补贴收入应并入工资薪金所得。",
|
||||
"(一)从超出国家规定比例支付的补贴,不属于免税福利费。",
|
||||
]
|
||||
|
||||
# 命中 markdown 标题模式(BULLET_PATTERN 下标 4)时,应该优先选中该组。
|
||||
assert bullets_category(sections) == 4
|
||||
|
||||
|
||||
def test_mid_sentence_bullet_marker_should_not_be_treated_as_heading() -> None:
|
||||
sections = [
|
||||
"根据前述规则:一、这里是句中枚举,不是章节标题,不能被当成层级。",
|
||||
"延续上文:(二)这里同样是正文中的枚举表达,不是独立标题。",
|
||||
"## 3.4 交通补贴的个税处理",
|
||||
]
|
||||
assert bullets_category(sections) == 4
|
||||
|
||||
|
||||
def test_chunk_preset_options_include_description() -> None:
|
||||
options = get_chunk_preset_options()
|
||||
assert [option["value"] for option in options] == list(CHUNK_PRESETS)
|
||||
assert {option["value"] for option in options} == CHUNK_PRESET_IDS
|
||||
assert all(isinstance(option.get("description"), str) and option["description"] for option in options)
|
||||
|
||||
|
||||
def test_chunk_preset_defaults_only_include_strategy_specific_fields() -> None:
|
||||
for preset_id in CHUNK_PRESET_IDS:
|
||||
assert get_default_chunk_parser_config(preset_id) == {}
|
||||
|
||||
|
||||
def test_laws_chunking_should_apply_overlength_protection() -> None:
|
||||
lines = ["#### 中华人民共和国企业所得税法实施条例", "##### 微信扫一扫:分享"]
|
||||
lines.extend(
|
||||
[f"第{i}条 企业所得税法实施细则说明,适用于测试场景,确保条文长度足够用于验证分块策略。" for i in range(1, 260)]
|
||||
)
|
||||
content = "\n".join(lines)
|
||||
|
||||
max_chunk_tokens = 180
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_laws_long",
|
||||
filename="laws.docx",
|
||||
processing_params={
|
||||
"chunk_preset_id": "laws",
|
||||
"chunk_parser_config": {
|
||||
"chunk_token_num": max_chunk_tokens,
|
||||
"overlapped_percent": 20,
|
||||
"delimiter": "\\n",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert len(chunks) > 1
|
||||
assert max(count_tokens(ck["content"]) for ck in chunks) <= max_chunk_tokens
|
||||
|
||||
|
||||
def test_laws_chunking_should_prefer_sentence_boundary_split() -> None:
|
||||
line = "第一条 企业所得税法实施细则用于测试分块语义边界。"
|
||||
content = line * 120
|
||||
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_laws_sentence",
|
||||
filename="laws.docx",
|
||||
processing_params={
|
||||
"chunk_preset_id": "laws",
|
||||
"chunk_parser_config": {
|
||||
"chunk_token_num": 120,
|
||||
"overlapped_percent": 0,
|
||||
"delimiter": "\\n",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert len(chunks) > 1
|
||||
for ck in chunks:
|
||||
text = ck["content"].strip()
|
||||
assert text
|
||||
assert count_tokens(text) <= 120
|
||||
|
||||
|
||||
def test_laws_chunking_should_prefer_article_level_before_item_level() -> None:
|
||||
content = """
|
||||
第六章 特别纳税调整
|
||||
第一百零六条 企业所得税法第三十八条规定的可以指定扣缴义务人的情形,包括:
|
||||
(一)在资金、经营、购销等方面存在直接或者间接的控制关系;
|
||||
(二)可以代表企业实施其他具有约束力的行为。
|
||||
第一百零七条 税务机关可以依法核定应纳税所得额。
|
||||
""".strip()
|
||||
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_laws_article",
|
||||
filename="laws.docx",
|
||||
processing_params={
|
||||
"chunk_preset_id": "laws",
|
||||
"chunk_parser_config": {
|
||||
"chunk_token_num": 1000,
|
||||
"overlapped_percent": 0,
|
||||
"delimiter": "\\n",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
# 只要条下款项没有被拆成独立碎片,即可满足“条级优先”的目标。
|
||||
target_chunks = [ck["content"] for ck in chunks if "第一百零六条" in ck["content"]]
|
||||
assert target_chunks
|
||||
assert any("(一)" in chunk and "(二)" in chunk for chunk in target_chunks)
|
||||
|
||||
|
||||
def test_laws_markdown_articles_should_not_collapse_into_chapter_chunk() -> None:
|
||||
content = """
|
||||
## 第一章 总则
|
||||
- **第一条** 为了规范担保活动,保障债权实现,制定本法。
|
||||
- **第二条** 在借贷活动中,当事人可以依法设定担保。
|
||||
- **第三条** 担保活动应当遵循平等、自愿、公平和诚实信用原则。
|
||||
""".strip()
|
||||
|
||||
chunks = chunk_markdown(
|
||||
markdown_content=content,
|
||||
file_id="file_laws_markdown_article",
|
||||
filename="laws.md",
|
||||
processing_params={
|
||||
"chunk_preset_id": "laws",
|
||||
"chunk_parser_config": {
|
||||
"chunk_token_num": 120,
|
||||
"overlapped_percent": 0,
|
||||
"delimiter": "\\n",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
first_article_chunks = [ck["content"] for ck in chunks if "第一条" in ck["content"]]
|
||||
assert first_article_chunks
|
||||
# 条级切分时,第一条与第二条不应被合并到同一块。
|
||||
assert all("第二条" not in chunk for chunk in first_article_chunks)
|
||||
assert max(count_tokens(ck["content"]) for ck in chunks) <= 120
|
||||
|
||||
|
||||
def test_sanitize_processing_params_should_drop_non_persistent_fields() -> None:
|
||||
sanitized = sanitize_processing_params(
|
||||
{
|
||||
"chunk_preset_id": "general",
|
||||
"chunk_parser_config": {"chunk_token_num": 300},
|
||||
"ocr_engine": "mineru_ocr",
|
||||
"ocr_engine_config": {},
|
||||
"auto_index": True,
|
||||
"content_hashes": {"a.md": "hash-a"},
|
||||
"enable_ocr": "mineru_ocr",
|
||||
"_preprocessed_map": {"a.md": {"path": "/tmp/a.md"}},
|
||||
}
|
||||
)
|
||||
|
||||
assert sanitized == {
|
||||
"chunk_preset_id": "general",
|
||||
"chunk_parser_config": {"chunk_token_num": 300},
|
||||
"ocr_engine": "mineru_ocr",
|
||||
"ocr_engine_config": {},
|
||||
}
|
||||
|
||||
|
||||
def test_resolve_processing_params_keeps_ocr_fields_and_chunk_params() -> None:
|
||||
resolved = resolve_processing_params(
|
||||
kb_additional_params={
|
||||
"chunk_preset_id": "book",
|
||||
"chunk_parser_config": {"delimiter": "\n", "chunk_token_num": 300},
|
||||
},
|
||||
file_processing_params={
|
||||
"ocr_engine": "mineru_ocr",
|
||||
"ocr_engine_config": {"backend": "pipeline"},
|
||||
"chunk_preset_id": "qa",
|
||||
"chunk_parser_config": {"overlapped_percent": 10},
|
||||
"content_hashes": {"a.md": "hash-a"},
|
||||
},
|
||||
request_params={
|
||||
"auto_index": True,
|
||||
"chunk_preset_id": "laws",
|
||||
"chunk_parser_config": {"chunk_token_num": 666},
|
||||
},
|
||||
)
|
||||
|
||||
assert resolved["ocr_engine"] == "mineru_ocr"
|
||||
assert resolved["ocr_engine_config"] == {"backend": "pipeline"}
|
||||
assert resolved["chunk_preset_id"] == "laws"
|
||||
assert resolved["chunk_parser_config"] == {
|
||||
"delimiter": "\n",
|
||||
"chunk_token_num": 666,
|
||||
"overlapped_percent": 10,
|
||||
}
|
||||
assert "content_hashes" not in resolved
|
||||
assert "enable_ocr" not in resolved
|
||||
assert "auto_index" not in resolved
|
||||
|
||||
|
||||
def test_resolve_processing_params_defaults_ocr_fields() -> None:
|
||||
resolved = resolve_processing_params(
|
||||
kb_additional_params={},
|
||||
file_processing_params={"ocr_engine_config": "invalid", "enable_ocr": "mineru_ocr"},
|
||||
)
|
||||
|
||||
assert resolved["ocr_engine"] == "rapid_ocr"
|
||||
assert resolved["ocr_engine_config"] == {}
|
||||
assert "enable_ocr" not in resolved
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.repositories.agent_repository import (
|
||||
AgentRepository,
|
||||
DEFAULT_AGENT_DESCRIPTION,
|
||||
DEFAULT_SHARE_CONFIG,
|
||||
GENERAL_PURPOSE_AGENT_DESCRIPTION,
|
||||
GENERAL_PURPOSE_AGENT_NAME,
|
||||
GENERAL_PURPOSE_AGENT_SLUG,
|
||||
SUB_AGENT_BACKEND_ID,
|
||||
user_can_access_agent,
|
||||
user_can_manage_agent,
|
||||
)
|
||||
from yuxi.storage.postgres.models_business import Agent, User
|
||||
|
||||
|
||||
class FakeDb:
|
||||
def __init__(self):
|
||||
self.added = None
|
||||
self.commit = AsyncMock()
|
||||
self.refresh = AsyncMock()
|
||||
|
||||
def add(self, item):
|
||||
self.added = item
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_default_agent_creates_description(monkeypatch):
|
||||
db = FakeDb()
|
||||
repo = AgentRepository(db)
|
||||
|
||||
async def get_by_slug(_slug):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(repo, "get_by_slug", get_by_slug)
|
||||
|
||||
agent = await repo.ensure_default_agent()
|
||||
|
||||
assert agent.description == DEFAULT_AGENT_DESCRIPTION
|
||||
assert agent.config_json == {"context": {}}
|
||||
assert db.added is agent
|
||||
db.commit.assert_awaited_once()
|
||||
db.refresh.assert_awaited_once_with(agent)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_default_agent_backfills_missing_description(monkeypatch):
|
||||
db = FakeDb()
|
||||
repo = AgentRepository(db)
|
||||
agent = SimpleNamespace(
|
||||
share_config=DEFAULT_SHARE_CONFIG.copy(),
|
||||
is_default=True,
|
||||
description=None,
|
||||
updated_by=None,
|
||||
updated_at=None,
|
||||
)
|
||||
|
||||
async def get_by_slug(_slug):
|
||||
return agent
|
||||
|
||||
monkeypatch.setattr(repo, "get_by_slug", get_by_slug)
|
||||
|
||||
result = await repo.ensure_default_agent(created_by="admin")
|
||||
|
||||
assert result is agent
|
||||
assert agent.description == DEFAULT_AGENT_DESCRIPTION
|
||||
assert agent.updated_by == "admin"
|
||||
db.commit.assert_awaited_once()
|
||||
db.refresh.assert_awaited_once_with(agent)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_general_purpose_subagent_creates_empty_config_subagent(monkeypatch):
|
||||
db = FakeDb()
|
||||
repo = AgentRepository(db)
|
||||
|
||||
async def get_by_slug(_slug):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(repo, "get_by_slug", get_by_slug)
|
||||
|
||||
agent = await repo.ensure_general_purpose_subagent(created_by="system")
|
||||
|
||||
assert agent.slug == GENERAL_PURPOSE_AGENT_SLUG
|
||||
assert agent.name == GENERAL_PURPOSE_AGENT_NAME
|
||||
assert agent.description == GENERAL_PURPOSE_AGENT_DESCRIPTION
|
||||
assert agent.backend_id == SUB_AGENT_BACKEND_ID
|
||||
assert agent.is_subagent is True
|
||||
assert agent.is_default is False
|
||||
assert agent.config_json == {"context": {}}
|
||||
assert agent.share_config == DEFAULT_SHARE_CONFIG
|
||||
assert agent.created_by == "system"
|
||||
assert db.added is agent
|
||||
db.commit.assert_awaited_once()
|
||||
db.refresh.assert_awaited_once_with(agent)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_general_purpose_subagent_is_idempotent(monkeypatch):
|
||||
db = FakeDb()
|
||||
repo = AgentRepository(db)
|
||||
existing = SimpleNamespace(slug=GENERAL_PURPOSE_AGENT_SLUG, config_json={"context": {"model": "custom:model"}})
|
||||
|
||||
async def get_by_slug(_slug):
|
||||
return existing
|
||||
|
||||
monkeypatch.setattr(repo, "get_by_slug", get_by_slug)
|
||||
|
||||
agent = await repo.ensure_general_purpose_subagent()
|
||||
|
||||
assert agent is existing
|
||||
assert db.added is None
|
||||
db.commit.assert_not_awaited()
|
||||
db.refresh.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_agent_for_normal_user_forces_private_share(monkeypatch):
|
||||
db = FakeDb()
|
||||
repo = AgentRepository(db)
|
||||
|
||||
async def fake_unique_slug(_slug, _name):
|
||||
return "personal-bot"
|
||||
|
||||
monkeypatch.setattr(repo, "_unique_slug", fake_unique_slug)
|
||||
|
||||
creator = User(username="user", uid="user", password_hash="x", role="user", department_id=1)
|
||||
agent = await repo.create(
|
||||
name="Personal Bot",
|
||||
backend_id="ChatbotAgent",
|
||||
slug="personal-bot",
|
||||
share_config={"access_level": "global", "department_ids": [], "user_uids": []},
|
||||
created_by="user",
|
||||
creator=creator,
|
||||
)
|
||||
|
||||
assert agent.share_config == {"access_level": "user", "department_ids": [], "user_uids": ["user"]}
|
||||
assert db.added is agent
|
||||
|
||||
|
||||
def test_shared_agent_is_accessible_but_not_manageable_for_normal_user():
|
||||
user = User(username="user", uid="user", password_hash="x", role="user", department_id=1)
|
||||
agent = Agent(
|
||||
slug="shared-bot",
|
||||
name="Shared Bot",
|
||||
backend_id="ChatbotAgent",
|
||||
created_by="other",
|
||||
share_config={"access_level": "user", "department_ids": [], "user_uids": ["user"]},
|
||||
)
|
||||
|
||||
assert user_can_access_agent(user, agent) is True
|
||||
assert user_can_manage_agent(user, agent) is False
|
||||
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.repositories.agent_repository import (
|
||||
AgentRepository,
|
||||
DEEP_RESEARCH_AGENT_SLUG,
|
||||
DEFAULT_AGENT_BACKEND_ID,
|
||||
FACT_VERIFIER_AGENT_SLUG,
|
||||
RESEARCH_EXPLORER_AGENT_SLUG,
|
||||
SUB_AGENT_BACKEND_ID,
|
||||
)
|
||||
|
||||
|
||||
class CollectingDb:
|
||||
def __init__(self):
|
||||
self.added: list = []
|
||||
self.commit = AsyncMock()
|
||||
self.refresh = AsyncMock()
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_deep_research_agents_creates_orchestrator_and_subagents(monkeypatch):
|
||||
db = CollectingDb()
|
||||
repo = AgentRepository(db)
|
||||
|
||||
async def get_by_slug(_slug):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(repo, "get_by_slug", get_by_slug)
|
||||
|
||||
await repo.ensure_deep_research_agents()
|
||||
|
||||
created = {agent.slug: agent for agent in db.added}
|
||||
assert set(created) == {
|
||||
DEEP_RESEARCH_AGENT_SLUG,
|
||||
RESEARCH_EXPLORER_AGENT_SLUG,
|
||||
FACT_VERIFIER_AGENT_SLUG,
|
||||
}
|
||||
|
||||
explorer = created[RESEARCH_EXPLORER_AGENT_SLUG]
|
||||
verifier = created[FACT_VERIFIER_AGENT_SLUG]
|
||||
assert explorer.backend_id == SUB_AGENT_BACKEND_ID and explorer.is_subagent is True
|
||||
assert verifier.backend_id == SUB_AGENT_BACKEND_ID and verifier.is_subagent is True
|
||||
|
||||
orchestrator = created[DEEP_RESEARCH_AGENT_SLUG]
|
||||
assert orchestrator.backend_id == DEFAULT_AGENT_BACKEND_ID
|
||||
assert orchestrator.is_subagent is False
|
||||
assert orchestrator.is_default is False
|
||||
context = orchestrator.config_json["context"]
|
||||
assert context["subagents"] == [RESEARCH_EXPLORER_AGENT_SLUG, FACT_VERIFIER_AGENT_SLUG]
|
||||
assert context["skills"] == [DEEP_RESEARCH_AGENT_SLUG]
|
||||
assert context["system_prompt"].strip()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_deep_research_agents_is_idempotent(monkeypatch):
|
||||
db = CollectingDb()
|
||||
repo = AgentRepository(db)
|
||||
|
||||
async def get_by_slug(slug):
|
||||
return SimpleNamespace(slug=slug)
|
||||
|
||||
monkeypatch.setattr(repo, "get_by_slug", get_by_slug)
|
||||
|
||||
await repo.ensure_deep_research_agents()
|
||||
|
||||
assert db.added == []
|
||||
db.commit.assert_not_awaited()
|
||||
@@ -0,0 +1,90 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from yuxi.repositories.agent_run_repository import AgentRunRepository
|
||||
from yuxi.storage.postgres.models_business import AgentRun, Base, Conversation, SubagentThread
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture()
|
||||
async def session():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
yield db
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _seed_subagent_runs(db, *, relation_child_thread_id: str = "child-thread") -> AgentRun:
|
||||
child_run = AgentRun(
|
||||
id="child-run",
|
||||
conversation_thread_id="child-thread",
|
||||
agent_slug="worker",
|
||||
uid="user-1",
|
||||
status="completed",
|
||||
request_id="child-req",
|
||||
conversation_id=20,
|
||||
created_by_run_id="parent-run",
|
||||
subagent_thread_relation_id=77,
|
||||
run_type="subagent",
|
||||
input_payload={},
|
||||
)
|
||||
db.add_all(
|
||||
[
|
||||
Conversation(id=10, thread_id="parent-thread", uid="user-1", agent_id="main", status="active"),
|
||||
Conversation(id=20, thread_id="child-thread", uid="user-1", agent_id="worker", status="subagent"),
|
||||
SubagentThread(
|
||||
id=77,
|
||||
uid="user-1",
|
||||
parent_conversation_id=10,
|
||||
child_conversation_id=20,
|
||||
child_thread_id=relation_child_thread_id,
|
||||
subagent_slug="worker",
|
||||
created_by_run_id="parent-run",
|
||||
),
|
||||
AgentRun(
|
||||
id="parent-run",
|
||||
conversation_thread_id="parent-thread",
|
||||
agent_slug="main",
|
||||
uid="user-1",
|
||||
status="completed",
|
||||
request_id="parent-req",
|
||||
conversation_id=10,
|
||||
run_type="chat",
|
||||
input_payload={},
|
||||
),
|
||||
child_run,
|
||||
]
|
||||
)
|
||||
await db.commit()
|
||||
return child_run
|
||||
|
||||
|
||||
async def test_get_subagent_run_for_creator_returns_child_run(session):
|
||||
child_run = await _seed_subagent_runs(session)
|
||||
|
||||
result = await AgentRunRepository(session).get_subagent_run_for_creator(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
run_id="child-run",
|
||||
)
|
||||
|
||||
assert result is child_run
|
||||
|
||||
|
||||
async def test_get_subagent_run_for_creator_returns_none_for_relation_mismatch(session):
|
||||
await _seed_subagent_runs(session, relation_child_thread_id="other-child-thread")
|
||||
|
||||
result = await AgentRunRepository(session).get_subagent_run_for_creator(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
run_id="child-run",
|
||||
)
|
||||
|
||||
assert result is None
|
||||
@@ -0,0 +1,96 @@
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.repositories import knowledge_chunk_repository as repo_module
|
||||
from yuxi.repositories.knowledge_chunk_repository import KnowledgeChunkRepository, SQL_IN_BATCH_SIZE
|
||||
|
||||
|
||||
def _extract_id_batch(statement) -> list[str]:
|
||||
id_batches = [value for value in statement.compile().params.values() if isinstance(value, list | tuple) and value]
|
||||
assert len(id_batches) == 1
|
||||
return list(id_batches[0])
|
||||
|
||||
|
||||
def test_iter_batches_limits_sql_in_arguments():
|
||||
ids = [f"id-{index}" for index in range(SQL_IN_BATCH_SIZE * 2 + 1)]
|
||||
|
||||
batches = list(KnowledgeChunkRepository._iter_batches(ids))
|
||||
|
||||
assert [len(batch) for batch in batches] == [SQL_IN_BATCH_SIZE, SQL_IN_BATCH_SIZE, 1]
|
||||
assert [item for batch in batches for item in batch] == ids
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_by_file_ids_splits_large_inputs(monkeypatch):
|
||||
file_ids = [f"file-{index}" for index in range(SQL_IN_BATCH_SIZE + 5)]
|
||||
batch_lengths: list[int] = []
|
||||
seen_file_ids: list[str] = []
|
||||
|
||||
class FakeResult:
|
||||
def __init__(self, batch: list[str]):
|
||||
self.batch = batch
|
||||
|
||||
def all(self):
|
||||
return [(file_id, 1) for file_id in self.batch]
|
||||
|
||||
class FakeSession:
|
||||
async def execute(self, statement):
|
||||
batch = _extract_id_batch(statement)
|
||||
batch_lengths.append(len(batch))
|
||||
seen_file_ids.extend(batch)
|
||||
return FakeResult(batch)
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_session_context():
|
||||
yield FakeSession()
|
||||
|
||||
monkeypatch.setattr(repo_module.pg_manager, "get_async_session_context", fake_session_context)
|
||||
|
||||
counts = await KnowledgeChunkRepository().count_by_file_ids(file_ids)
|
||||
|
||||
assert batch_lengths == [SQL_IN_BATCH_SIZE, 5]
|
||||
assert seen_file_ids == file_ids
|
||||
assert counts["file-0"] == 1
|
||||
assert counts[f"file-{SQL_IN_BATCH_SIZE + 4}"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_by_file_ids_splits_large_inputs(monkeypatch):
|
||||
file_ids = [
|
||||
*(f"file-b-{index:05d}" for index in range(SQL_IN_BATCH_SIZE)),
|
||||
*(f"file-a-{index:05d}" for index in range(5)),
|
||||
]
|
||||
batch_lengths: list[int] = []
|
||||
|
||||
class FakeScalarResult:
|
||||
def __init__(self, chunks: list[SimpleNamespace]):
|
||||
self.chunks = chunks
|
||||
|
||||
def all(self):
|
||||
return self.chunks
|
||||
|
||||
class FakeResult:
|
||||
def __init__(self, chunks: list[SimpleNamespace]):
|
||||
self.chunks = chunks
|
||||
|
||||
def scalars(self):
|
||||
return FakeScalarResult(self.chunks)
|
||||
|
||||
class FakeSession:
|
||||
async def execute(self, statement):
|
||||
batch = _extract_id_batch(statement)
|
||||
batch_lengths.append(len(batch))
|
||||
return FakeResult([SimpleNamespace(file_id=file_id, chunk_index=0) for file_id in batch])
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_session_context():
|
||||
yield FakeSession()
|
||||
|
||||
monkeypatch.setattr(repo_module.pg_manager, "get_async_session_context", fake_session_context)
|
||||
|
||||
chunks = await KnowledgeChunkRepository().list_by_file_ids(file_ids)
|
||||
|
||||
assert batch_lengths == [SQL_IN_BATCH_SIZE, 5]
|
||||
assert [chunk.file_id for chunk in chunks] == sorted(file_ids)
|
||||
@@ -0,0 +1,456 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from server.utils.auth_middleware import get_db, get_required_user
|
||||
|
||||
agent_invocation_router_module = importlib.import_module("server.routers.agent_invocation_router")
|
||||
|
||||
|
||||
def _build_app(monkeypatch: pytest.MonkeyPatch, *, authenticated: bool = True) -> TestClient:
|
||||
app = FastAPI()
|
||||
app.include_router(agent_invocation_router_module.agent_invocation_router, prefix="/api")
|
||||
|
||||
async def fake_db():
|
||||
return object()
|
||||
|
||||
app.dependency_overrides[get_db] = fake_db
|
||||
if authenticated:
|
||||
|
||||
async def fake_user():
|
||||
return SimpleNamespace(uid="user-1", role="user", department_id=1)
|
||||
|
||||
app.dependency_overrides[get_required_user] = fake_user
|
||||
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_agent_call_run_requires_authentication(monkeypatch: pytest.MonkeyPatch):
|
||||
client = _build_app(monkeypatch, authenticated=False)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={"agent_slug": "translator", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
)
|
||||
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_agent_eval_run_returns_service_payload(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
async def fake_create_agent_eval_run_view(**kwargs):
|
||||
calls["kwargs"] = kwargs
|
||||
return {
|
||||
"status": "completed",
|
||||
"output": "final answer",
|
||||
"agent_run_id": "run-1",
|
||||
"request_id": "req-1",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
agent_invocation_router_module,
|
||||
"create_agent_eval_run_view",
|
||||
fake_create_agent_eval_run_view,
|
||||
)
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/eval/runs",
|
||||
json={
|
||||
"query": "2+2=?",
|
||||
"agent_slug": " default-chatbot ",
|
||||
"evaluation": {"dataset_name": "dataset-1", "dataset_item_id": "item-1", "ignored": "nope"},
|
||||
"meta": {"request_id": "req-1", "attachment_file_ids": ["file-1"]},
|
||||
"model_spec": "provider:model",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["output"] == "final answer"
|
||||
assert calls["kwargs"]["query"] == "2+2=?"
|
||||
assert calls["kwargs"]["agent_slug"] == " default-chatbot "
|
||||
assert calls["kwargs"]["evaluation"] == {"dataset_name": "dataset-1", "dataset_item_id": "item-1"}
|
||||
assert calls["kwargs"]["meta"] == {"request_id": "req-1", "attachment_file_ids": ["file-1"]}
|
||||
assert calls["kwargs"]["model_spec"] == "provider:model"
|
||||
assert calls["kwargs"]["current_user"].uid == "user-1"
|
||||
|
||||
|
||||
def test_agent_eval_run_rejects_too_long_request_id(monkeypatch: pytest.MonkeyPatch):
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/eval/runs",
|
||||
json={
|
||||
"query": "2+2=?",
|
||||
"agent_slug": "default-chatbot",
|
||||
"meta": {"request_id": "x" * 65},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
assert response.json()["detail"] == "request_id 不能超过 64 个字符"
|
||||
|
||||
|
||||
def test_agent_eval_run_returns_504_when_wait_times_out(monkeypatch: pytest.MonkeyPatch):
|
||||
async def fake_create_agent_eval_run_view(**_kwargs):
|
||||
raise HTTPException(
|
||||
status_code=504,
|
||||
detail={"message": "运行仍在进行中,等待最终结果超时", "run": {"status": "running"}},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
agent_invocation_router_module,
|
||||
"create_agent_eval_run_view",
|
||||
fake_create_agent_eval_run_view,
|
||||
)
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/eval/runs",
|
||||
json={"query": "2+2=?", "agent_slug": "default-chatbot", "meta": {"request_id": "req-1"}},
|
||||
)
|
||||
|
||||
assert response.status_code == 504
|
||||
assert response.json()["detail"]["message"] == "运行仍在进行中,等待最终结果超时"
|
||||
assert response.json()["detail"]["run"]["status"] == "running"
|
||||
|
||||
|
||||
def test_legacy_agent_eval_run_path_is_not_registered(monkeypatch: pytest.MonkeyPatch):
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent/eval/runs",
|
||||
json={"query": "2+2=?", "agent_slug": "default-chatbot"},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_legacy_agent_call_run_path_is_not_registered(monkeypatch: pytest.MonkeyPatch):
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-call/runs",
|
||||
json={"agent_slug": "translator", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_agent_call_run_creates_async_run_and_returns_agent_call_payload(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
async def fake_create_agent_call_run_view(**kwargs):
|
||||
calls["kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"agent_slug": "translator",
|
||||
"thread_id": "thread-1",
|
||||
"status": "pending",
|
||||
"request_id": "req-1",
|
||||
"output": "",
|
||||
"choices": [{"index": 0, "messages": [{"role": "assistant", "content": ""}], "finish_reason": None}],
|
||||
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
agent_invocation_router_module,
|
||||
"create_agent_call_run_view",
|
||||
fake_create_agent_call_run_view,
|
||||
)
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={
|
||||
"agent_slug": " translator ",
|
||||
"messages": [
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": "ignored"},
|
||||
{"role": "user", "content": "Hello"},
|
||||
],
|
||||
"agent_call_meta": {"trace_id": "trace-1"},
|
||||
"thread_id": " thread-1 ",
|
||||
"request_id": " req-1 ",
|
||||
"async_mode": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["run_id"] == "run-1"
|
||||
assert response.json()["choices"][0]["finish_reason"] is None
|
||||
assert calls["kwargs"]["agent_slug"] == " translator "
|
||||
assert calls["kwargs"]["messages"][-1] == {"role": "user", "content": "Hello"}
|
||||
assert calls["kwargs"]["agent_call_meta"] == {"trace_id": "trace-1"}
|
||||
assert calls["kwargs"]["requested_thread_id"] == " thread-1 "
|
||||
assert calls["kwargs"]["request_id"] == " req-1 "
|
||||
assert calls["kwargs"]["async_mode"] is True
|
||||
assert calls["kwargs"]["stream"] is False
|
||||
assert calls["kwargs"]["current_user"].uid == "user-1"
|
||||
|
||||
|
||||
def test_agent_call_run_waits_and_wraps_final_result(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
async def fake_create_agent_call_run_view(**kwargs):
|
||||
calls["kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"agent_slug": "translator",
|
||||
"thread_id": "thread-1",
|
||||
"status": "completed",
|
||||
"request_id": "req-1",
|
||||
"output": "你好",
|
||||
"choices": [{"index": 0, "messages": [{"role": "assistant", "content": "你好"}], "finish_reason": "stop"}],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
agent_invocation_router_module,
|
||||
"create_agent_call_run_view",
|
||||
fake_create_agent_call_run_view,
|
||||
)
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={
|
||||
"agent_slug": "translator",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"request_id": "req-1",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["run_id"] == "run-1"
|
||||
assert response.json()["output"] == "你好"
|
||||
assert response.json()["choices"][0]["messages"] == [{"role": "assistant", "content": "你好"}]
|
||||
assert response.json()["choices"][0]["finish_reason"] == "stop"
|
||||
assert response.json()["usage"] == {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}
|
||||
assert calls["kwargs"]["request_id"] == "req-1"
|
||||
|
||||
|
||||
def test_agent_call_run_returns_504_when_wait_times_out(monkeypatch: pytest.MonkeyPatch):
|
||||
async def fake_create_agent_call_run_view(**_kwargs):
|
||||
raise HTTPException(
|
||||
status_code=504,
|
||||
detail={"message": "运行仍在进行中,等待最终结果超时", "run": {"status": "running"}},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
agent_invocation_router_module,
|
||||
"create_agent_call_run_view",
|
||||
fake_create_agent_call_run_view,
|
||||
)
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={"agent_slug": "translator", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
)
|
||||
|
||||
assert response.status_code == 504
|
||||
assert response.json()["detail"]["message"] == "运行仍在进行中,等待最终结果超时"
|
||||
assert response.json()["detail"]["run"]["status"] == "running"
|
||||
|
||||
|
||||
def test_agent_call_run_rejects_context_override(monkeypatch: pytest.MonkeyPatch):
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={
|
||||
"agent_slug": "translator",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"agent_call_meta": {"context": {"system_prompt": "只回答中文"}},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
assert "agent_call_meta.context 不允许覆盖 Agent context" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_agent_call_run_accepts_openai_text_content_parts(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
async def fake_create_agent_call_run_view(**kwargs):
|
||||
calls["kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"agent_slug": kwargs["agent_slug"],
|
||||
"thread_id": "thread-1",
|
||||
"status": "pending",
|
||||
"request_id": "req-1",
|
||||
"output": "",
|
||||
"choices": [{"index": 0, "messages": [{"role": "assistant", "content": ""}], "finish_reason": None}],
|
||||
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(agent_invocation_router_module, "create_agent_call_run_view", fake_create_agent_call_run_view)
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={
|
||||
"agent_slug": "translator",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert calls["kwargs"]["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]
|
||||
|
||||
|
||||
def test_agent_call_run_accepts_openai_multimodal_content_parts(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
async def fake_create_agent_call_run_view(**kwargs):
|
||||
calls["kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"agent_slug": kwargs["agent_slug"],
|
||||
"thread_id": "thread-1",
|
||||
"status": "pending",
|
||||
"request_id": "req-1",
|
||||
"output": "",
|
||||
"choices": [{"index": 0, "messages": [{"role": "assistant", "content": ""}], "finish_reason": None}],
|
||||
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(agent_invocation_router_module, "create_agent_call_run_view", fake_create_agent_call_run_view)
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={
|
||||
"agent_slug": "translator",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "describe"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,base64-image", "detail": "low"},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert calls["kwargs"]["messages"][0]["content"][1]["image_url"] == {
|
||||
"url": "data:image/png;base64,base64-image",
|
||||
"detail": "low",
|
||||
}
|
||||
|
||||
|
||||
def test_agent_call_run_propagates_agent_not_found(monkeypatch: pytest.MonkeyPatch):
|
||||
async def fake_create_agent_call_run_view(**_kwargs):
|
||||
raise HTTPException(status_code=404, detail="智能体不存在")
|
||||
|
||||
monkeypatch.setattr(agent_invocation_router_module, "create_agent_call_run_view", fake_create_agent_call_run_view)
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={"agent_slug": "missing", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.json()["detail"] == "智能体不存在"
|
||||
|
||||
|
||||
def test_agent_call_run_rejects_invalid_boundary_payload(monkeypatch: pytest.MonkeyPatch):
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={"agent_slug": " ", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["detail"] == "agent_slug 不能为空"
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={"agent_slug": "translator", "messages": [{"role": "user", "content": "Hello"}], "stream": True},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert "stream=true" in response.json()["detail"]
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={"agent_slug": "translator", "messages": []},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["detail"] == "messages 不能为空"
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={"agent_slug": "translator", "messages": [{"role": "assistant", "content": "hello"}]},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["detail"] == "messages 必须包含 user 消息"
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={"agent_slug": "translator", "messages": [{"role": "user", "content": ""}]},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["detail"] == "user message content 必须是非空字符串或多模态数组"
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs",
|
||||
json={
|
||||
"agent_slug": "translator",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"request_id": "x" * 65,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["detail"] == "request_id 不能超过 64 个字符"
|
||||
|
||||
|
||||
def test_agent_call_result_returns_service_payload(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
async def fake_get_agent_call_run_result_view(**kwargs):
|
||||
calls["kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": kwargs["run_id"],
|
||||
"agent_slug": kwargs["agent_slug"],
|
||||
"thread_id": "thread-1",
|
||||
"status": "completed",
|
||||
"request_id": "req-1",
|
||||
"output": "done",
|
||||
"choices": [{"index": 0, "messages": [{"role": "assistant", "content": "done"}], "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
agent_invocation_router_module,
|
||||
"get_agent_call_run_result_view",
|
||||
fake_get_agent_call_run_result_view,
|
||||
)
|
||||
client = _build_app(monkeypatch)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent-invocation/agent-call/runs/result",
|
||||
json={"run_id": "run-1", "agent_slug": "translator"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["output"] == "done"
|
||||
assert calls["kwargs"] == {
|
||||
"run_id": "run-1",
|
||||
"agent_slug": "translator",
|
||||
"current_uid": "user-1",
|
||||
"db": calls["kwargs"]["db"],
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from server.routers.auth_router import delete_user
|
||||
from server.routers.user_router import APIKeyCreate, create_api_key
|
||||
from server.utils.auth_middleware import _verify_api_key
|
||||
from yuxi.repositories import user_repository as user_repository_module
|
||||
from yuxi.repositories.user_repository import UserRepository
|
||||
from yuxi.storage.postgres.models_business import APIKey, Base, Department, User
|
||||
from yuxi.utils.auth_utils import AuthUtils
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
|
||||
|
||||
|
||||
class _ScalarResult:
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def scalar_one_or_none(self):
|
||||
return self.value
|
||||
|
||||
|
||||
class _FakeApiKeySession:
|
||||
def __init__(self, api_key: APIKey):
|
||||
self.api_key = api_key
|
||||
self.execute_calls = 0
|
||||
|
||||
async def execute(self, _statement):
|
||||
self.execute_calls += 1
|
||||
return _ScalarResult(self.api_key)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture()
|
||||
async def session():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
dept_a = Department(name="Dept A")
|
||||
dept_b = Department(name="Dept B")
|
||||
superadmin = User(
|
||||
username="Super Admin",
|
||||
uid="superadmin",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="superadmin",
|
||||
department=dept_a,
|
||||
)
|
||||
dept_b_admin = User(
|
||||
username="Dept B Admin",
|
||||
uid="dept_b_admin",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="admin",
|
||||
department=dept_b,
|
||||
)
|
||||
regular_user = User(
|
||||
username="Regular",
|
||||
uid="regular",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="user",
|
||||
department=dept_a,
|
||||
)
|
||||
deleted_user = User(
|
||||
username="Deleted",
|
||||
uid="deleted",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="user",
|
||||
department=dept_a,
|
||||
is_deleted=1,
|
||||
)
|
||||
db.add_all([dept_a, dept_b, superadmin, dept_b_admin, regular_user, deleted_user])
|
||||
await db.commit()
|
||||
for item in [dept_a, dept_b, superadmin, dept_b_admin, regular_user, deleted_user]:
|
||||
await db.refresh(item)
|
||||
yield {
|
||||
"db": db,
|
||||
"dept_a": dept_a,
|
||||
"dept_b": dept_b,
|
||||
"superadmin": superadmin,
|
||||
"dept_b_admin": dept_b_admin,
|
||||
"regular_user": regular_user,
|
||||
"deleted_user": deleted_user,
|
||||
}
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_api_key_rejects_deleted_bound_user_without_department_or_superadmin_fallback(session):
|
||||
db = session["db"]
|
||||
secret, key_hash, key_prefix = AuthUtils.generate_api_key()
|
||||
api_key = APIKey(
|
||||
key_hash=key_hash,
|
||||
key_prefix=key_prefix,
|
||||
name="deleted user key",
|
||||
user_id=session["deleted_user"].id,
|
||||
department_id=session["dept_b"].id,
|
||||
created_by=str(session["deleted_user"].id),
|
||||
)
|
||||
db.add(api_key)
|
||||
await db.commit()
|
||||
|
||||
user, verified_key = await _verify_api_key(secret, db)
|
||||
|
||||
assert user is None
|
||||
assert verified_key is None
|
||||
|
||||
|
||||
async def test_api_key_without_user_binding_is_rejected_before_department_mapping(session):
|
||||
secret, key_hash, key_prefix = AuthUtils.generate_api_key()
|
||||
api_key = APIKey(
|
||||
key_hash=key_hash,
|
||||
key_prefix=key_prefix,
|
||||
name="department key",
|
||||
user_id=None,
|
||||
department_id=session["dept_b"].id,
|
||||
created_by=str(session["superadmin"].id),
|
||||
)
|
||||
fake_db = _FakeApiKeySession(api_key)
|
||||
|
||||
user, verified_key = await _verify_api_key(secret, fake_db)
|
||||
|
||||
assert user is None
|
||||
assert verified_key is None
|
||||
assert fake_db.execute_calls == 1
|
||||
|
||||
|
||||
async def test_create_api_key_rejects_mismatched_department(session):
|
||||
db = session["db"]
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await create_api_key(
|
||||
APIKeyCreate(name="wrong department", department_id=session["dept_b"].id),
|
||||
current_user=session["regular_user"],
|
||||
db=db,
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 403
|
||||
|
||||
|
||||
async def test_create_api_key_allows_current_user_department(session):
|
||||
db = session["db"]
|
||||
|
||||
response = await create_api_key(
|
||||
APIKeyCreate(name="own department", department_id=session["dept_a"].id),
|
||||
current_user=session["regular_user"],
|
||||
db=db,
|
||||
)
|
||||
|
||||
assert response.api_key.user_id == session["regular_user"].id
|
||||
assert response.api_key.department_id == session["dept_a"].id
|
||||
assert response.secret.startswith(response.api_key.key_prefix)
|
||||
|
||||
|
||||
async def test_delete_user_disables_owned_api_keys(session):
|
||||
db = session["db"]
|
||||
_secret, key_hash, key_prefix = AuthUtils.generate_api_key()
|
||||
api_key = APIKey(
|
||||
key_hash=key_hash,
|
||||
key_prefix=key_prefix,
|
||||
name="owned key",
|
||||
user_id=session["regular_user"].id,
|
||||
created_by=str(session["regular_user"].id),
|
||||
)
|
||||
db.add(api_key)
|
||||
await db.commit()
|
||||
await db.refresh(api_key)
|
||||
|
||||
result = await delete_user(session["regular_user"].id, None, session["superadmin"], db)
|
||||
await db.refresh(api_key)
|
||||
|
||||
assert result["success"] is True
|
||||
assert api_key.is_enabled is False
|
||||
|
||||
|
||||
async def test_user_repository_soft_delete_disables_owned_api_keys(session, monkeypatch):
|
||||
db = session["db"]
|
||||
_secret, key_hash, key_prefix = AuthUtils.generate_api_key()
|
||||
api_key = APIKey(
|
||||
key_hash=key_hash,
|
||||
key_prefix=key_prefix,
|
||||
name="repository owned key",
|
||||
user_id=session["regular_user"].id,
|
||||
created_by=str(session["regular_user"].id),
|
||||
)
|
||||
db.add(api_key)
|
||||
await db.commit()
|
||||
await db.refresh(api_key)
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_session_context():
|
||||
yield db
|
||||
await db.commit()
|
||||
|
||||
monkeypatch.setattr(user_repository_module.pg_manager, "get_async_session_context", fake_session_context)
|
||||
|
||||
assert await UserRepository().soft_delete(session["regular_user"].id) is True
|
||||
await db.refresh(api_key)
|
||||
|
||||
assert api_key.is_enabled is False
|
||||
@@ -0,0 +1,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from fastapi import FastAPI
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from server.routers.auth_router import auth
|
||||
from server.utils.auth_middleware import get_db, get_required_user
|
||||
from yuxi.storage.postgres.models_business import Base, Department, User
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture()
|
||||
async def app_client():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
dept = Department(name="默认部门")
|
||||
user = User(
|
||||
username="Admin",
|
||||
uid="admin",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="superadmin",
|
||||
department=dept,
|
||||
)
|
||||
db.add_all([dept, user])
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(auth, prefix="/api")
|
||||
|
||||
async def override_db():
|
||||
yield db
|
||||
|
||||
async def override_user():
|
||||
return user
|
||||
|
||||
app.dependency_overrides[get_db] = override_db
|
||||
app.dependency_overrides[get_required_user] = override_user
|
||||
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
yield client
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_auth_router_cli_auth_create_approve_and_exchange(app_client):
|
||||
create_response = await app_client.post("/api/auth/cli/sessions", json={})
|
||||
assert create_response.status_code == 200, create_response.text
|
||||
session = create_response.json()
|
||||
assert session["verification_uri"] == "/auth/cli/authorize"
|
||||
|
||||
pending_response = await app_client.post(
|
||||
"/api/auth/cli/sessions/token", json={"device_code": session["device_code"]}
|
||||
)
|
||||
assert pending_response.status_code == 400
|
||||
assert pending_response.json()["detail"]["error"] == "authorization_pending"
|
||||
|
||||
read_response = await app_client.get(f"/api/auth/cli/sessions/{session['user_code']}")
|
||||
assert read_response.status_code == 200
|
||||
assert read_response.json()["status"] == "pending"
|
||||
|
||||
approve_response = await app_client.post(f"/api/auth/cli/sessions/{session['user_code']}/approve")
|
||||
assert approve_response.status_code == 200
|
||||
assert approve_response.json()["status"] == "approved"
|
||||
|
||||
token_response = await app_client.post("/api/auth/cli/sessions/token", json={"device_code": session["device_code"]})
|
||||
assert token_response.status_code == 200, token_response.text
|
||||
token_data = token_response.json()
|
||||
assert token_data["secret"].startswith("yxkey_")
|
||||
assert token_data["user"]["uid"] == "admin"
|
||||
@@ -0,0 +1,169 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from fastapi import HTTPException
|
||||
from fastapi.routing import APIRoute
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from server.routers.dashboard_router import (
|
||||
dashboard,
|
||||
get_all_conversations,
|
||||
get_conversation_detail,
|
||||
get_tool_call_stats,
|
||||
get_user_activity_stats,
|
||||
)
|
||||
from server.utils.auth_middleware import get_superadmin_user
|
||||
from yuxi.storage.postgres.models_business import Base, Conversation, Department, Message, ToolCall, User
|
||||
from yuxi.utils.datetime_utils import utc_now_naive
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture()
|
||||
async def dashboard_session():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
dept_a = Department(name="Dept A")
|
||||
dept_b = Department(name="Dept B")
|
||||
superadmin = User(
|
||||
username="Super Admin",
|
||||
uid="superadmin",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="superadmin",
|
||||
department=dept_a,
|
||||
)
|
||||
admin_a = User(
|
||||
username="Admin A",
|
||||
uid="admin_a",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="admin",
|
||||
department=dept_a,
|
||||
)
|
||||
user_a = User(
|
||||
username="User A",
|
||||
uid="user_a",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="user",
|
||||
department=dept_a,
|
||||
)
|
||||
admin_b = User(
|
||||
username="Admin B",
|
||||
uid="admin_b",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="admin",
|
||||
department=dept_b,
|
||||
)
|
||||
user_b = User(
|
||||
username="User B",
|
||||
uid="user_b",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="user",
|
||||
department=dept_b,
|
||||
)
|
||||
now = utc_now_naive()
|
||||
conversation_a = Conversation(
|
||||
thread_id="thread-a",
|
||||
uid="user_a",
|
||||
agent_id="agent-shared",
|
||||
title="Dept A conversation",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
conversation_b = Conversation(
|
||||
thread_id="thread-b",
|
||||
uid="user_b",
|
||||
agent_id="agent-shared",
|
||||
title="Dept B conversation",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
message_a = Message(conversation=conversation_a, role="assistant", content="A", created_at=now)
|
||||
message_b = Message(conversation=conversation_b, role="assistant", content="B", created_at=now)
|
||||
tool_call_a = ToolCall(message=message_a, tool_name="dept_a_tool", status="success", created_at=now)
|
||||
tool_call_b = ToolCall(message=message_b, tool_name="dept_b_tool", status="success", created_at=now)
|
||||
db.add_all(
|
||||
[
|
||||
dept_a,
|
||||
dept_b,
|
||||
superadmin,
|
||||
admin_a,
|
||||
user_a,
|
||||
admin_b,
|
||||
user_b,
|
||||
conversation_a,
|
||||
conversation_b,
|
||||
message_a,
|
||||
message_b,
|
||||
tool_call_a,
|
||||
tool_call_b,
|
||||
]
|
||||
)
|
||||
await db.commit()
|
||||
for item in [
|
||||
dept_a,
|
||||
dept_b,
|
||||
superadmin,
|
||||
admin_a,
|
||||
user_a,
|
||||
admin_b,
|
||||
user_b,
|
||||
conversation_a,
|
||||
conversation_b,
|
||||
]:
|
||||
await db.refresh(item)
|
||||
yield {"db": db, "superadmin": superadmin, "admin_a": admin_a}
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_dashboard_routes_require_superadmin_dependency():
|
||||
dashboard_routes = [route for route in dashboard.routes if isinstance(route, APIRoute)]
|
||||
|
||||
assert dashboard_routes
|
||||
for route in dashboard_routes:
|
||||
dependency_calls = {dependency.call for dependency in route.dependant.dependencies}
|
||||
assert get_superadmin_user in dependency_calls
|
||||
|
||||
|
||||
async def test_dashboard_dependency_rejects_department_admin(dashboard_session):
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await get_superadmin_user(dashboard_session["admin_a"])
|
||||
|
||||
assert exc.value.status_code == 403
|
||||
|
||||
|
||||
async def test_conversation_list_superadmin_sees_all_departments(dashboard_session):
|
||||
response = await get_all_conversations(db=dashboard_session["db"], current_user=dashboard_session["superadmin"])
|
||||
|
||||
assert {item["thread_id"] for item in response} == {"thread-a", "thread-b"}
|
||||
|
||||
|
||||
async def test_conversation_detail_superadmin_can_view_other_department(dashboard_session):
|
||||
response = await get_conversation_detail(
|
||||
"thread-b",
|
||||
db=dashboard_session["db"],
|
||||
current_user=dashboard_session["superadmin"],
|
||||
)
|
||||
|
||||
assert response["thread_id"] == "thread-b"
|
||||
|
||||
|
||||
async def test_user_activity_stats_superadmin_include_all_departments(dashboard_session):
|
||||
stats = await get_user_activity_stats(db=dashboard_session["db"], current_user=dashboard_session["superadmin"])
|
||||
|
||||
assert stats.total_users == 5
|
||||
assert stats.active_users_24h == 2
|
||||
assert stats.active_users_30d == 2
|
||||
|
||||
|
||||
async def test_tool_stats_superadmin_include_all_departments(dashboard_session):
|
||||
stats = await get_tool_call_stats(db=dashboard_session["db"], current_user=dashboard_session["superadmin"])
|
||||
|
||||
assert stats.total_calls == 2
|
||||
assert stats.successful_calls == 2
|
||||
assert {tool["tool_name"] for tool in stats.most_used_tools} == {"dept_a_tool", "dept_b_tool"}
|
||||
@@ -0,0 +1,641 @@
|
||||
from inspect import signature
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, HTTPException, UploadFile
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from server.routers import knowledge_router
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
class FakeTaskContext:
|
||||
def __init__(self):
|
||||
self.result = None
|
||||
|
||||
async def set_message(self, message: str) -> None:
|
||||
return None
|
||||
|
||||
async def set_progress(self, progress: float, message: str | None = None) -> None:
|
||||
return None
|
||||
|
||||
async def set_result(self, result: dict) -> None:
|
||||
self.result = result
|
||||
|
||||
async def raise_if_cancelled(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
async def test_upload_file_does_not_expose_legacy_allow_jsonl_query():
|
||||
assert "allow_jsonl" not in signature(knowledge_router.upload_file).parameters
|
||||
|
||||
|
||||
async def test_document_file_exists_returns_boolean_for_relative_path(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
captured["ensure"] = (kb_id, operation)
|
||||
|
||||
async def fake_document_file_exists(kb_id: str, filename: str) -> bool:
|
||||
captured["exists"] = (kb_id, filename)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "document_file_exists", fake_document_file_exists)
|
||||
|
||||
result = await knowledge_router.document_file_exists(
|
||||
"kb_1",
|
||||
filename=" google_drive/shared_drives/engineering/playbook.txt ",
|
||||
current_user=SimpleNamespace(uid="user_1"),
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"kb_id": "kb_1",
|
||||
"filename": "google_drive/shared_drives/engineering/playbook.txt",
|
||||
"exists": True,
|
||||
}
|
||||
assert captured == {
|
||||
"ensure": ("kb_1", "文档存在性检查"),
|
||||
"exists": ("kb_1", "google_drive/shared_drives/engineering/playbook.txt"),
|
||||
}
|
||||
|
||||
|
||||
async def test_document_file_exists_route_accepts_filename_with_slashes(monkeypatch):
|
||||
async def fake_admin_user():
|
||||
return SimpleNamespace(uid="user_1")
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
async def fake_document_file_exists(kb_id: str, filename: str) -> bool:
|
||||
assert kb_id == "kb_1"
|
||||
assert filename == "google_drive/shared_drives/engineering/playbook.txt"
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "document_file_exists", fake_document_file_exists)
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(knowledge_router.knowledge, prefix="/api")
|
||||
app.dependency_overrides[knowledge_router.get_admin_user] = fake_admin_user
|
||||
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
response = await client.get(
|
||||
"/api/knowledge/databases/kb_1/documents/exists",
|
||||
params={"filename": "google_drive/shared_drives/engineering/playbook.txt"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {
|
||||
"kb_id": "kb_1",
|
||||
"filename": "google_drive/shared_drives/engineering/playbook.txt",
|
||||
"exists": True,
|
||||
}
|
||||
|
||||
|
||||
async def test_document_file_exists_rejects_blank_filename(monkeypatch):
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.document_file_exists(
|
||||
"kb_1",
|
||||
filename=" ",
|
||||
current_user=SimpleNamespace(uid="user_1"),
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.detail == "filename is required"
|
||||
|
||||
|
||||
async def test_upload_file_rejects_jsonl_uploads():
|
||||
upload = UploadFile(filename="dataset.jsonl", file=BytesIO(b'{"query":"hello"}\n'))
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.upload_file(upload, kb_id=None, current_user=SimpleNamespace(uid="user_1"))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.detail == "Unsupported file type: .jsonl"
|
||||
|
||||
|
||||
async def test_upload_file_rejects_oversized_file(monkeypatch):
|
||||
monkeypatch.setattr(knowledge_router, "MAX_UPLOAD_SIZE_BYTES", 5)
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
|
||||
upload = UploadFile(filename="demo.txt", file=BytesIO(b"123456"))
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.upload_file(upload, kb_id="kb_1", current_user=SimpleNamespace(uid="user_1"))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "100 MB" in exc_info.value.detail
|
||||
|
||||
|
||||
async def test_upload_file_invalid_kb_fails_before_read_or_minio(monkeypatch):
|
||||
calls = {"read": 0, "upload": 0}
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
raise HTTPException(status_code=404, detail=f"知识库 {kb_id} 不存在")
|
||||
|
||||
async def fake_read_upload_with_limit(*_args, **_kwargs) -> bytes:
|
||||
calls["read"] += 1
|
||||
return b"demo"
|
||||
|
||||
async def fake_upload_to_minio(*_args, **_kwargs) -> str:
|
||||
calls["upload"] += 1
|
||||
return "minio://knowledgebases/kb_1/upload/demo.txt"
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router, "read_upload_with_limit", fake_read_upload_with_limit)
|
||||
monkeypatch.setattr(knowledge_router, "aupload_file_to_minio", fake_upload_to_minio)
|
||||
|
||||
upload = UploadFile(filename="demo.txt", file=BytesIO(b"demo"))
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.upload_file(upload, kb_id="missing", current_user=SimpleNamespace(uid="user_1"))
|
||||
|
||||
assert exc_info.value.status_code == 404
|
||||
assert calls == {"read": 0, "upload": 0}
|
||||
|
||||
|
||||
async def test_upload_file_read_only_kb_fails_before_read_or_minio(monkeypatch):
|
||||
calls = {"read": 0, "upload": 0}
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
raise HTTPException(status_code=400, detail="只支持检索,不支持文档上传")
|
||||
|
||||
async def fake_read_upload_with_limit(*_args, **_kwargs) -> bytes:
|
||||
calls["read"] += 1
|
||||
return b"demo"
|
||||
|
||||
async def fake_upload_to_minio(*_args, **_kwargs) -> str:
|
||||
calls["upload"] += 1
|
||||
return "minio://knowledgebases/kb_1/upload/demo.txt"
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router, "read_upload_with_limit", fake_read_upload_with_limit)
|
||||
monkeypatch.setattr(knowledge_router, "aupload_file_to_minio", fake_upload_to_minio)
|
||||
|
||||
upload = UploadFile(filename="demo.txt", file=BytesIO(b"demo"))
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.upload_file(upload, kb_id="readonly", current_user=SimpleNamespace(uid="user_1"))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert calls == {"read": 0, "upload": 0}
|
||||
|
||||
|
||||
async def test_markdown_endpoint_rejects_oversized_file(monkeypatch):
|
||||
monkeypatch.setattr(knowledge_router, "MAX_UPLOAD_SIZE_BYTES", 5)
|
||||
upload = UploadFile(filename="demo.txt", file=BytesIO(b"123456"))
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.mark_it_down(upload, current_user=SimpleNamespace(uid="user_1"))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "100 MB" in exc_info.value.detail
|
||||
|
||||
|
||||
async def test_index_documents_uses_uid_for_operator(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
async def fake_get_database_info(kb_id: str) -> dict:
|
||||
return {"name": "测试知识库"}
|
||||
|
||||
async def fake_index_file(kb_id: str, file_id: str, operator_id: str | None = None, params: dict | None = None):
|
||||
captured["operator_id"] = operator_id
|
||||
return {"file_id": file_id, "status": "indexed"}
|
||||
|
||||
async def fake_enqueue(name: str, task_type: str, payload: dict, coroutine):
|
||||
await coroutine(FakeTaskContext())
|
||||
return SimpleNamespace(id="task_1")
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "get_database_info", fake_get_database_info)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "index_file", fake_index_file)
|
||||
monkeypatch.setattr(knowledge_router.tasker, "enqueue", fake_enqueue)
|
||||
|
||||
result = await knowledge_router.index_documents(
|
||||
"kb_1",
|
||||
["file_1"],
|
||||
params={},
|
||||
current_user=SimpleNamespace(id="numeric-id", uid="uid-user"),
|
||||
)
|
||||
|
||||
assert result["status"] == "queued"
|
||||
assert captured["operator_id"] == "uid-user"
|
||||
|
||||
|
||||
async def test_parse_documents_rejects_oversized_direct_batch():
|
||||
file_ids = [f"file_{index}" for index in range(knowledge_router.MAX_DIRECT_DOCUMENT_ACTION_FILE_IDS + 1)]
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.parse_documents(
|
||||
"kb_1",
|
||||
file_ids,
|
||||
current_user=SimpleNamespace(uid="uid-user"),
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert str(knowledge_router.MAX_DIRECT_DOCUMENT_ACTION_FILE_IDS) in exc_info.value.detail
|
||||
|
||||
|
||||
async def test_parse_pending_documents_enqueues_status_scoped_task(monkeypatch):
|
||||
captured = {"list_calls": [], "parsed": []}
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
captured["ensure"] = (kb_id, operation)
|
||||
|
||||
async def fake_get_database_info(kb_id: str) -> dict:
|
||||
return {"name": "测试知识库", "stats": {"pending_parse_count": 2}}
|
||||
|
||||
async def fake_list_document_file_ids_by_statuses(kb_id: str, *, statuses, after_file_id, limit):
|
||||
captured["list_calls"].append(
|
||||
{"kb_id": kb_id, "statuses": statuses, "after_file_id": after_file_id, "limit": limit}
|
||||
)
|
||||
return ["file_1", "file_2"] if after_file_id is None else []
|
||||
|
||||
async def fake_parse_file(kb_id: str, file_id: str, operator_id: str | None = None):
|
||||
captured["parsed"].append({"kb_id": kb_id, "file_id": file_id, "operator_id": operator_id})
|
||||
return {"file_id": file_id, "status": "parsed"}
|
||||
|
||||
async def fake_enqueue_unique_by_payload(**kwargs):
|
||||
captured["payload"] = kwargs["payload"]
|
||||
captured["payload_match"] = kwargs["payload_match"]
|
||||
captured["statuses"] = kwargs["statuses"]
|
||||
await kwargs["coroutine"](FakeTaskContext())
|
||||
return SimpleNamespace(id="task_1"), True
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "get_database_info", fake_get_database_info)
|
||||
monkeypatch.setattr(
|
||||
knowledge_router.knowledge_base,
|
||||
"list_document_file_ids_by_statuses",
|
||||
fake_list_document_file_ids_by_statuses,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "parse_file", fake_parse_file)
|
||||
monkeypatch.setattr(knowledge_router.tasker, "enqueue_unique_by_payload", fake_enqueue_unique_by_payload)
|
||||
|
||||
result = await knowledge_router.parse_pending_documents(
|
||||
"kb_1",
|
||||
current_user=SimpleNamespace(uid="uid-user"),
|
||||
)
|
||||
|
||||
assert result["status"] == "queued"
|
||||
assert result["task_id"] == "task_1"
|
||||
assert captured["ensure"] == ("kb_1", "文档解析")
|
||||
assert captured["payload_match"] == {"kb_id": "kb_1", "scope": "pending", "action": "parse"}
|
||||
assert captured["statuses"] == knowledge_router.ACTIVE_DOCUMENT_ACTION_TASK_STATUSES
|
||||
assert captured["payload"]["statuses"] == knowledge_router.PENDING_PARSE_STATUSES
|
||||
assert captured["list_calls"] == [
|
||||
{
|
||||
"kb_id": "kb_1",
|
||||
"statuses": knowledge_router.PENDING_PARSE_STATUSES,
|
||||
"after_file_id": None,
|
||||
"limit": knowledge_router.DOCUMENT_ACTION_BATCH_SIZE,
|
||||
},
|
||||
{
|
||||
"kb_id": "kb_1",
|
||||
"statuses": knowledge_router.PENDING_PARSE_STATUSES,
|
||||
"after_file_id": "file_2",
|
||||
"limit": knowledge_router.DOCUMENT_ACTION_BATCH_SIZE,
|
||||
},
|
||||
]
|
||||
assert captured["parsed"] == [
|
||||
{"kb_id": "kb_1", "file_id": "file_1", "operator_id": "uid-user"},
|
||||
{"kb_id": "kb_1", "file_id": "file_2", "operator_id": "uid-user"},
|
||||
]
|
||||
|
||||
|
||||
async def test_index_pending_documents_uses_pending_statuses_and_params(monkeypatch):
|
||||
captured = {"list_calls": [], "updated": [], "indexed": []}
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
captured["ensure"] = (kb_id, operation)
|
||||
|
||||
async def fake_get_database_info(kb_id: str) -> dict:
|
||||
return {"name": "测试知识库", "stats": {"pending_index_count": 2}}
|
||||
|
||||
async def fake_list_document_file_ids_by_statuses(kb_id: str, *, statuses, after_file_id, limit):
|
||||
captured["list_calls"].append(
|
||||
{"kb_id": kb_id, "statuses": statuses, "after_file_id": after_file_id, "limit": limit}
|
||||
)
|
||||
return ["file_1", "file_2"] if after_file_id is None else []
|
||||
|
||||
async def fake_update_file_params(kb_id: str, file_id: str, params: dict, operator_id: str | None = None):
|
||||
captured["updated"].append({"kb_id": kb_id, "file_id": file_id, "params": params, "operator_id": operator_id})
|
||||
|
||||
async def fake_index_file(kb_id: str, file_id: str, operator_id: str | None = None, params: dict | None = None):
|
||||
captured["indexed"].append({"kb_id": kb_id, "file_id": file_id, "operator_id": operator_id, "params": params})
|
||||
return {"file_id": file_id, "status": "indexed"}
|
||||
|
||||
async def fake_enqueue_unique_by_payload(**kwargs):
|
||||
captured["payload"] = kwargs["payload"]
|
||||
captured["payload_match"] = kwargs["payload_match"]
|
||||
await kwargs["coroutine"](FakeTaskContext())
|
||||
return SimpleNamespace(id="task_1"), True
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "get_database_info", fake_get_database_info)
|
||||
monkeypatch.setattr(
|
||||
knowledge_router.knowledge_base,
|
||||
"list_document_file_ids_by_statuses",
|
||||
fake_list_document_file_ids_by_statuses,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "update_file_params", fake_update_file_params)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "index_file", fake_index_file)
|
||||
monkeypatch.setattr(knowledge_router.tasker, "enqueue_unique_by_payload", fake_enqueue_unique_by_payload)
|
||||
|
||||
params = {"chunk_preset_id": "general"}
|
||||
result = await knowledge_router.index_pending_documents(
|
||||
"kb_1",
|
||||
payload=knowledge_router.PendingIndexDocumentsRequest(params=params),
|
||||
current_user=SimpleNamespace(uid="uid-user"),
|
||||
)
|
||||
|
||||
assert result["status"] == "queued"
|
||||
assert captured["ensure"] == ("kb_1", "文档入库")
|
||||
assert captured["payload_match"] == {"kb_id": "kb_1", "scope": "pending", "action": "index"}
|
||||
assert captured["payload"]["statuses"] == knowledge_router.PENDING_INDEX_STATUSES
|
||||
assert captured["payload"]["params"] == params
|
||||
assert captured["list_calls"][0]["statuses"] == knowledge_router.PENDING_INDEX_STATUSES
|
||||
assert captured["updated"] == [
|
||||
{"kb_id": "kb_1", "file_id": "file_1", "params": params, "operator_id": "uid-user"},
|
||||
{"kb_id": "kb_1", "file_id": "file_2", "params": params, "operator_id": "uid-user"},
|
||||
]
|
||||
assert captured["indexed"] == [
|
||||
{"kb_id": "kb_1", "file_id": "file_1", "operator_id": "uid-user", "params": params},
|
||||
{"kb_id": "kb_1", "file_id": "file_2", "operator_id": "uid-user", "params": params},
|
||||
]
|
||||
|
||||
|
||||
async def test_add_documents_auto_index_returns_one_final_result_per_item(monkeypatch):
|
||||
context = FakeTaskContext()
|
||||
item = "minio://knowledgebases/kb_1/upload/demo.txt"
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
async def fake_get_database_info(kb_id: str) -> dict:
|
||||
return {"name": "测试知识库"}
|
||||
|
||||
async def fake_add_file_record(kb_id: str, item_path: str, params: dict, operator_id: str | None = None):
|
||||
return {"file_id": "file_1", "status": "indexing"}
|
||||
|
||||
async def fake_parse_file(kb_id: str, file_id: str, operator_id: str | None = None):
|
||||
return {"file_id": file_id, "status": "parsed"}
|
||||
|
||||
async def fake_update_file_params(kb_id: str, file_id: str, params: dict, operator_id: str | None = None):
|
||||
return None
|
||||
|
||||
async def fake_index_file(kb_id: str, file_id: str, operator_id: str | None = None, params: dict | None = None):
|
||||
return {"file_id": file_id, "status": "indexed"}
|
||||
|
||||
async def fake_enqueue(name: str, task_type: str, payload: dict, coroutine):
|
||||
await coroutine(context)
|
||||
return SimpleNamespace(id="task_1")
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "get_database_info", fake_get_database_info)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "add_file_record", fake_add_file_record)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "parse_file", fake_parse_file)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "update_file_params", fake_update_file_params)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "index_file", fake_index_file)
|
||||
monkeypatch.setattr(knowledge_router.tasker, "enqueue", fake_enqueue)
|
||||
|
||||
result = await knowledge_router.add_documents(
|
||||
"kb_1",
|
||||
[item],
|
||||
params={"content_type": "file", "auto_index": True, "content_hashes": {item: "hash_1"}},
|
||||
current_user=SimpleNamespace(uid="uid-user"),
|
||||
)
|
||||
|
||||
assert result["status"] == "queued"
|
||||
assert context.result["submitted"] == 1
|
||||
assert context.result["failed"] == 0
|
||||
assert context.result["items"] == [{"file_id": "file_1", "status": "indexed"}]
|
||||
|
||||
|
||||
async def test_add_documents_auto_index_treats_error_none_as_success(monkeypatch):
|
||||
"""成功入库的文件元数据会携带 error=None,不应被统计为失败 (#793)。"""
|
||||
context = FakeTaskContext()
|
||||
item = "minio://knowledgebases/kb_1/upload/demo.txt"
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
async def fake_get_database_info(kb_id: str) -> dict:
|
||||
return {"name": "测试知识库"}
|
||||
|
||||
async def fake_add_file_record(kb_id: str, item_path: str, params: dict, operator_id: str | None = None):
|
||||
return {"file_id": "file_1", "status": "indexing"}
|
||||
|
||||
async def fake_parse_file(kb_id: str, file_id: str, operator_id: str | None = None):
|
||||
return {"file_id": file_id, "status": "parsed", "error": None}
|
||||
|
||||
async def fake_update_file_params(kb_id: str, file_id: str, params: dict, operator_id: str | None = None):
|
||||
return None
|
||||
|
||||
async def fake_index_file(kb_id: str, file_id: str, operator_id: str | None = None, params: dict | None = None):
|
||||
return {"file_id": file_id, "status": "indexed", "error": None}
|
||||
|
||||
async def fake_enqueue(name: str, task_type: str, payload: dict, coroutine):
|
||||
await coroutine(context)
|
||||
return SimpleNamespace(id="task_1")
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "get_database_info", fake_get_database_info)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "add_file_record", fake_add_file_record)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "parse_file", fake_parse_file)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "update_file_params", fake_update_file_params)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "index_file", fake_index_file)
|
||||
monkeypatch.setattr(knowledge_router.tasker, "enqueue", fake_enqueue)
|
||||
|
||||
result = await knowledge_router.add_documents(
|
||||
"kb_1",
|
||||
[item],
|
||||
params={"content_type": "file", "auto_index": True, "content_hashes": {item: "hash_1"}},
|
||||
current_user=SimpleNamespace(uid="uid-user"),
|
||||
)
|
||||
|
||||
assert result["status"] == "queued"
|
||||
assert context.result["submitted"] == 1
|
||||
assert context.result["failed"] == 0
|
||||
assert context.result["items"] == [{"file_id": "file_1", "status": "indexed", "error": None}]
|
||||
|
||||
|
||||
async def test_add_uploaded_documents_rejects_empty_items(monkeypatch):
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.add_uploaded_documents(
|
||||
"kb_1",
|
||||
knowledge_router.AddUploadedDocumentsRequest(items=[], params={}),
|
||||
current_user=SimpleNamespace(uid="uid-user"),
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.detail == "items must not be empty"
|
||||
|
||||
|
||||
async def test_add_uploaded_documents_rejects_non_minio_url(monkeypatch):
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.add_uploaded_documents(
|
||||
"kb_1",
|
||||
knowledge_router.AddUploadedDocumentsRequest(
|
||||
items=["https://example.com/demo.txt"],
|
||||
params={"content_hashes": {"https://example.com/demo.txt": "hash_1"}},
|
||||
),
|
||||
current_user=SimpleNamespace(uid="uid-user"),
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.detail == "File source must be a MinIO URL"
|
||||
|
||||
|
||||
async def test_add_uploaded_documents_rejects_missing_content_hash(monkeypatch):
|
||||
item = "minio://knowledgebases/kb_1/upload/demo.txt"
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.add_uploaded_documents(
|
||||
"kb_1",
|
||||
knowledge_router.AddUploadedDocumentsRequest(items=[item], params={}),
|
||||
current_user=SimpleNamespace(uid="uid-user"),
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.detail == f"Missing content_hash for file: {item}"
|
||||
|
||||
|
||||
async def test_add_uploaded_documents_creates_records_without_task(monkeypatch):
|
||||
item = "minio://knowledgebases/kb_1/upload/demo.txt"
|
||||
captured = {}
|
||||
|
||||
async def fake_ensure_database_supports_documents(kb_id: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
async def fake_add_file_record(kb_id: str, item_path: str, params: dict, operator_id: str | None = None):
|
||||
captured["kb_id"] = kb_id
|
||||
captured["item"] = item_path
|
||||
captured["params"] = params
|
||||
captured["operator_id"] = operator_id
|
||||
return {"file_id": "file_1", "status": "uploaded", "filename": "demo.txt"}
|
||||
|
||||
async def fail_enqueue(*_args, **_kwargs):
|
||||
raise AssertionError("documents/add must not enqueue tasker work")
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "add_file_record", fake_add_file_record)
|
||||
monkeypatch.setattr(knowledge_router.tasker, "enqueue", fail_enqueue)
|
||||
|
||||
result = await knowledge_router.add_uploaded_documents(
|
||||
"kb_1",
|
||||
knowledge_router.AddUploadedDocumentsRequest(
|
||||
items=[item],
|
||||
params={
|
||||
"content_hashes": {item: "hash_1"},
|
||||
"file_sizes": {item: 4},
|
||||
"source_paths": {item: "docs/demo.txt"},
|
||||
},
|
||||
),
|
||||
current_user=SimpleNamespace(uid="uid-user"),
|
||||
)
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["added"] == 1
|
||||
assert result["failed"] == 0
|
||||
assert result["items"][0]["file_id"] == "file_1"
|
||||
assert captured == {
|
||||
"kb_id": "kb_1",
|
||||
"item": item,
|
||||
"params": {
|
||||
"content_hashes": {item: "hash_1"},
|
||||
"file_sizes": {item: 4},
|
||||
"source_path": "docs/demo.txt",
|
||||
},
|
||||
"operator_id": "uid-user",
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from server.routers import knowledge_router
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def test_import_workspace_files_uploads_workspace_file_to_minio(tmp_path, monkeypatch):
|
||||
source = tmp_path / "note.md"
|
||||
source.write_text("# workspace note\n", encoding="utf-8")
|
||||
|
||||
async def fake_ensure_database_supports_documents(slug: str, operation: str) -> None:
|
||||
assert slug == "db_1"
|
||||
assert "文档添加" in operation
|
||||
|
||||
async def fake_file_existed_in_db(slug: str, content_hash: str) -> bool:
|
||||
assert slug == "db_1"
|
||||
assert content_hash
|
||||
return False
|
||||
|
||||
async def fake_get_same_name_files(slug: str, filename: str) -> list:
|
||||
assert slug == "db_1"
|
||||
assert filename == "note.md"
|
||||
return []
|
||||
|
||||
async def fake_upload(bucket_name: str, file_name: str, data: bytes) -> str:
|
||||
assert bucket_name == knowledge_router.MinIOClient.KB_BUCKETS["documents"]
|
||||
assert file_name.startswith("db_1/upload/note_")
|
||||
assert data == b"# workspace note\n"
|
||||
return f"http://minio/{bucket_name}/{file_name}"
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router, "resolve_workspace_file_path", lambda **_kwargs: source)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "file_existed_in_db", fake_file_existed_in_db)
|
||||
monkeypatch.setattr(knowledge_router.knowledge_base, "get_same_name_files", fake_get_same_name_files)
|
||||
monkeypatch.setattr(knowledge_router, "aupload_file_to_minio", fake_upload)
|
||||
|
||||
result = await knowledge_router.import_workspace_files(
|
||||
knowledge_router.WorkspaceImportRequest(kb_id="db_1", paths=["/note.md"]),
|
||||
current_user=SimpleNamespace(id="user_1"),
|
||||
)
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert len(result["items"]) == 1
|
||||
item = result["items"][0]
|
||||
assert item["file_path"].startswith(
|
||||
f"http://minio/{knowledge_router.MinIOClient.KB_BUCKETS['documents']}/db_1/upload/note_"
|
||||
)
|
||||
assert item["content_hash"]
|
||||
assert item["filename"] == "note.md"
|
||||
assert item["size"] == len(b"# workspace note\n")
|
||||
assert item["workspace_path"] == "/note.md"
|
||||
|
||||
|
||||
async def test_import_workspace_files_rejects_directory(tmp_path, monkeypatch):
|
||||
async def fake_ensure_database_supports_documents(slug: str, operation: str) -> None:
|
||||
return None
|
||||
|
||||
def fake_resolve_workspace_file_path(**_kwargs):
|
||||
raise HTTPException(status_code=400, detail="当前路径不是文件: /folder")
|
||||
|
||||
monkeypatch.setattr(
|
||||
knowledge_router,
|
||||
"_ensure_database_supports_documents",
|
||||
fake_ensure_database_supports_documents,
|
||||
)
|
||||
monkeypatch.setattr(knowledge_router, "resolve_workspace_file_path", fake_resolve_workspace_file_path)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await knowledge_router.import_workspace_files(
|
||||
knowledge_router.WorkspaceImportRequest(kb_id="db_1", paths=["/folder"]),
|
||||
current_user=SimpleNamespace(id="user_1"),
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "不是文件" in exc_info.value.detail
|
||||
@@ -0,0 +1,166 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from server.routers.mcp_router import mcp
|
||||
from server.utils.auth_middleware import get_admin_user, get_db, get_required_user
|
||||
from yuxi.storage.postgres.models_business import User
|
||||
|
||||
|
||||
def _build_app(*, allow_admin: bool = True) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.include_router(mcp, prefix="/api")
|
||||
|
||||
async def fake_db():
|
||||
return None
|
||||
|
||||
async def fake_admin_user():
|
||||
if not allow_admin:
|
||||
from fastapi import HTTPException
|
||||
|
||||
raise HTTPException(status_code=403, detail="需要管理员权限")
|
||||
return User(
|
||||
username="admin",
|
||||
uid="admin",
|
||||
password_hash="x",
|
||||
role="admin",
|
||||
)
|
||||
|
||||
async def fake_required_user():
|
||||
return User(
|
||||
username="admin" if allow_admin else "user",
|
||||
uid="admin" if allow_admin else "user",
|
||||
password_hash="x",
|
||||
role="admin" if allow_admin else "user",
|
||||
)
|
||||
|
||||
app.dependency_overrides[get_db] = fake_db
|
||||
app.dependency_overrides[get_admin_user] = fake_admin_user
|
||||
app.dependency_overrides[get_required_user] = fake_required_user
|
||||
return app
|
||||
|
||||
|
||||
def test_update_mcp_server_status(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
class DummyServer:
|
||||
def __init__(self, enabled):
|
||||
self.enabled = enabled
|
||||
|
||||
def to_dict(self):
|
||||
return {"name": "demo-mcp", "enabled": self.enabled}
|
||||
|
||||
async def fake_set_server_enabled(db, name, enabled, updated_by=None):
|
||||
captured["name"] = name
|
||||
captured["enabled"] = enabled
|
||||
captured["updated_by"] = updated_by
|
||||
return enabled, DummyServer(enabled)
|
||||
|
||||
monkeypatch.setattr("server.routers.mcp_router.set_server_enabled", fake_set_server_enabled)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.put("/api/system/mcp-servers/demo-mcp/status", json={"enabled": False})
|
||||
assert resp.status_code == 200, resp.text
|
||||
payload = resp.json()
|
||||
assert payload["success"] is True
|
||||
assert payload["enabled"] is False
|
||||
assert payload["data"]["enabled"] is False
|
||||
assert captured == {"name": "demo-mcp", "enabled": False, "updated_by": "admin"}
|
||||
|
||||
|
||||
def test_update_mcp_server_status_not_found(monkeypatch):
|
||||
async def fake_set_server_enabled(db, name, enabled, updated_by=None):
|
||||
raise ValueError(f"Server '{name}' does not exist")
|
||||
|
||||
monkeypatch.setattr("server.routers.mcp_router.set_server_enabled", fake_set_server_enabled)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.put("/api/system/mcp-servers/missing/status", json={"enabled": True})
|
||||
assert resp.status_code == 404, resp.text
|
||||
|
||||
|
||||
def test_get_mcp_servers_normal_user_is_stripped(monkeypatch):
|
||||
class DummyServer:
|
||||
def __init__(self):
|
||||
self.name = "test-mcp"
|
||||
self.description = "test mcp description"
|
||||
self.transport = "stdio"
|
||||
self.url = "http://localhost:8000"
|
||||
self.command = "python"
|
||||
self.args = ["-m", "mcp"]
|
||||
self.env = {"API_KEY": "secret"}
|
||||
self.headers = {"Auth": "Bearer secret"}
|
||||
self.enabled = 1
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"name": self.name,
|
||||
"description": self.description,
|
||||
"transport": self.transport,
|
||||
"url": self.url,
|
||||
"command": self.command,
|
||||
"args": self.args,
|
||||
"env": self.env,
|
||||
"headers": self.headers,
|
||||
"enabled": bool(self.enabled),
|
||||
}
|
||||
|
||||
async def fake_get_all_mcp_servers(db):
|
||||
return [DummyServer()]
|
||||
|
||||
monkeypatch.setattr("server.routers.mcp_router.get_all_mcp_servers", fake_get_all_mcp_servers)
|
||||
|
||||
# 1. 管理员请求,应该返回全部字段
|
||||
client_admin = TestClient(_build_app(allow_admin=True))
|
||||
resp_admin = client_admin.get("/api/system/mcp-servers")
|
||||
assert resp_admin.status_code == 200
|
||||
data_admin = resp_admin.json()["data"][0]
|
||||
assert data_admin["url"] == "http://localhost:8000"
|
||||
assert data_admin["command"] == "python"
|
||||
assert data_admin["env"] == {"API_KEY": "secret"}
|
||||
|
||||
# 2. 普通用户请求,敏感字段及一切非安全白名单字段应该被彻底脱敏
|
||||
client_user = TestClient(_build_app(allow_admin=False))
|
||||
resp_user = client_user.get("/api/system/mcp-servers")
|
||||
assert resp_user.status_code == 200
|
||||
data_user = resp_user.json()["data"][0]
|
||||
assert "url" not in data_user
|
||||
assert "command" not in data_user
|
||||
assert "env" not in data_user
|
||||
assert "headers" not in data_user
|
||||
assert "transport" not in data_user # NOTE: 进一步验证连 transport 等配置层元数据也一并过滤
|
||||
assert data_user["name"] == "test-mcp"
|
||||
assert data_user["description"] == "test mcp description"
|
||||
assert data_user["enabled"] is True
|
||||
|
||||
|
||||
def test_create_mcp_server_rejects_extra_config_fields():
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post(
|
||||
"/api/system/mcp-servers",
|
||||
json={
|
||||
"slug": "demo-mcp",
|
||||
"name": "Demo MCP",
|
||||
"transport": "streamable_http",
|
||||
"url": "https://example.com/mcp",
|
||||
"enabled": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert resp.status_code == 422, resp.text
|
||||
|
||||
|
||||
def test_update_mcp_server_rejects_extra_config_fields():
|
||||
client = TestClient(_build_app())
|
||||
resp = client.put(
|
||||
"/api/system/mcp-servers/demo-mcp",
|
||||
json={
|
||||
"name": "Demo MCP",
|
||||
"transport": "streamable_http",
|
||||
"url": "https://example.com/mcp",
|
||||
"slug": "renamed-mcp",
|
||||
},
|
||||
)
|
||||
|
||||
assert resp.status_code == 422, resp.text
|
||||
@@ -0,0 +1,347 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from server.routers.skill_router import skills, user_skills
|
||||
from server.utils.auth_middleware import get_admin_user, get_db, get_required_user
|
||||
from yuxi.storage.postgres.models_business import Skill, User
|
||||
|
||||
|
||||
def _build_app(*, role: str = "admin") -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.include_router(skills, prefix="/api")
|
||||
app.include_router(user_skills, prefix="/api")
|
||||
|
||||
async def fake_db():
|
||||
return None
|
||||
|
||||
async def fake_required_user():
|
||||
return User(
|
||||
username=role,
|
||||
uid=role,
|
||||
password_hash="x",
|
||||
role=role,
|
||||
department_id=1,
|
||||
)
|
||||
|
||||
async def fake_admin_user():
|
||||
if role not in {"admin", "superadmin"}:
|
||||
raise HTTPException(status_code=403, detail="需要管理员权限")
|
||||
return await fake_required_user()
|
||||
|
||||
app.dependency_overrides[get_db] = fake_db
|
||||
app.dependency_overrides[get_required_user] = fake_required_user
|
||||
app.dependency_overrides[get_admin_user] = fake_admin_user
|
||||
return app
|
||||
|
||||
|
||||
def _skill(
|
||||
slug: str = "demo",
|
||||
*,
|
||||
source_type: str = "upload",
|
||||
created_by: str = "admin",
|
||||
enabled: bool = True,
|
||||
user_uids: list[str] | None = None,
|
||||
) -> Skill:
|
||||
return Skill(
|
||||
slug=slug,
|
||||
name=slug,
|
||||
description="demo skill",
|
||||
source_type=source_type,
|
||||
dir_path=f"skills/{slug}",
|
||||
share_config={"access_level": "user", "department_ids": [], "user_uids": user_uids or [created_by]},
|
||||
enabled=enabled,
|
||||
created_by=created_by,
|
||||
updated_by=created_by,
|
||||
)
|
||||
|
||||
|
||||
def test_list_visible_skills_route_returns_allowed_levels_and_can_manage(monkeypatch):
|
||||
async def fake_list_visible_skills_for_management(_db, user):
|
||||
assert user.uid == "admin"
|
||||
return [_skill()]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"server.routers.skill_router.list_visible_skills_for_management",
|
||||
fake_list_visible_skills_for_management,
|
||||
)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.get("/api/system/skills")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
payload = resp.json()
|
||||
assert payload["success"] is True
|
||||
assert payload["data"][0]["slug"] == "demo"
|
||||
assert payload["data"][0]["can_manage"] is True
|
||||
assert payload["allowed_access_levels"] == ["global", "department", "user"]
|
||||
|
||||
|
||||
def test_list_visible_skills_route_allows_normal_user_readonly_items(monkeypatch):
|
||||
async def fake_list_visible_skills_for_management(_db, user):
|
||||
assert user.uid == "user"
|
||||
return [
|
||||
_skill(slug="owned-disabled", created_by="user", enabled=False),
|
||||
_skill(slug="shared", created_by="other", user_uids=["user"]),
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"server.routers.skill_router.list_visible_skills_for_management",
|
||||
fake_list_visible_skills_for_management,
|
||||
)
|
||||
|
||||
client = TestClient(_build_app(role="user"))
|
||||
resp = client.get("/api/system/skills")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
payload = resp.json()
|
||||
assert payload["success"] is True
|
||||
assert [(item["slug"], item["can_manage"]) for item in payload["data"]] == [
|
||||
("owned-disabled", True),
|
||||
("shared", False),
|
||||
]
|
||||
assert payload["allowed_access_levels"] == ["user"]
|
||||
|
||||
|
||||
def test_list_accessible_skills_route(monkeypatch):
|
||||
async def fake_list_accessible_skills(_db, user):
|
||||
assert user.uid == "user"
|
||||
return [_skill(created_by="user")]
|
||||
|
||||
monkeypatch.setattr("server.routers.skill_router.list_accessible_skills", fake_list_accessible_skills)
|
||||
|
||||
client = TestClient(_build_app(role="user"))
|
||||
resp = client.get("/api/skills/accessible")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
payload = resp.json()
|
||||
assert payload["success"] is True
|
||||
assert payload["data"][0]["slug"] == "demo"
|
||||
assert payload["data"][0]["can_manage"] is True
|
||||
|
||||
|
||||
def test_prepare_skill_upload_route(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_prepare_skill_upload(_db, *, filename, file_bytes, operator):
|
||||
captured["filename"] = filename
|
||||
captured["file_bytes"] = file_bytes.decode("utf-8")
|
||||
captured["operator_uid"] = operator.uid
|
||||
return {"draft_id": "draft-1", "items": [{"slug": "demo", "success": True}]}
|
||||
|
||||
monkeypatch.setattr("server.routers.skill_router.prepare_skill_upload", fake_prepare_skill_upload)
|
||||
|
||||
client = TestClient(_build_app(role="user"))
|
||||
resp = client.post(
|
||||
"/api/skills/import/prepare",
|
||||
files={"file": ("SKILL.md", b"---\nname: demo\ndescription: demo skill\n---\n", "text/markdown")},
|
||||
)
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["data"]["draft_id"] == "draft-1"
|
||||
assert captured == {
|
||||
"filename": "SKILL.md",
|
||||
"file_bytes": "---\nname: demo\ndescription: demo skill\n---\n",
|
||||
"operator_uid": "user",
|
||||
}
|
||||
|
||||
|
||||
def test_remote_skill_prepare_and_confirm_routes(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_prepare_remote_skill_install(_db, *, source, skills, operator):
|
||||
captured["prepare"] = {"source": source, "skills": skills, "operator_uid": operator.uid}
|
||||
return {"draft_id": "draft-remote", "items": [{"slug": "frontend-design", "success": True}]}
|
||||
|
||||
async def fake_confirm_skill_install_draft(_db, *, draft_id, share_config, operator):
|
||||
captured["confirm"] = {"draft_id": draft_id, "share_config": share_config, "operator_uid": operator.uid}
|
||||
return [{"slug": "frontend-design", "success": True}]
|
||||
|
||||
monkeypatch.setattr("server.routers.skill_router.prepare_remote_skill_install", fake_prepare_remote_skill_install)
|
||||
monkeypatch.setattr("server.routers.skill_router.confirm_skill_install_draft", fake_confirm_skill_install_draft)
|
||||
|
||||
client = TestClient(_build_app(role="user"))
|
||||
prepare_resp = client.post(
|
||||
"/api/skills/remote/prepare",
|
||||
json={"source": "anthropics/skills", "skills": ["frontend-design"]},
|
||||
)
|
||||
confirm_resp = client.post(
|
||||
"/api/skills/install-drafts/draft-remote/confirm",
|
||||
json={"share_config": {"access_level": "user", "department_ids": [], "user_uids": ["user"]}},
|
||||
)
|
||||
|
||||
assert prepare_resp.status_code == 200, prepare_resp.text
|
||||
assert confirm_resp.status_code == 200, confirm_resp.text
|
||||
assert captured["prepare"] == {
|
||||
"source": "anthropics/skills",
|
||||
"skills": ["frontend-design"],
|
||||
"operator_uid": "user",
|
||||
}
|
||||
assert captured["confirm"]["draft_id"] == "draft-remote"
|
||||
assert captured["confirm"]["operator_uid"] == "user"
|
||||
|
||||
|
||||
def test_discard_skill_draft_route(monkeypatch):
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
async def fake_discard_skill_install_draft(*, draft_id, operator):
|
||||
captured["draft_id"] = draft_id
|
||||
captured["operator_uid"] = operator.uid
|
||||
|
||||
monkeypatch.setattr("server.routers.skill_router.discard_skill_install_draft", fake_discard_skill_install_draft)
|
||||
|
||||
client = TestClient(_build_app(role="user"))
|
||||
resp = client.delete("/api/skills/install-drafts/draft-1")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert captured == {"draft_id": "draft-1", "operator_uid": "user"}
|
||||
|
||||
|
||||
def test_dependency_options_route_checks_manage_permission(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_get_manageable_skill_or_raise(_db, user, slug):
|
||||
captured["manageable"] = {"slug": slug, "operator_uid": user.uid}
|
||||
return _skill(slug=slug)
|
||||
|
||||
async def fake_get_skill_dependency_options(_db, user, slug=None):
|
||||
captured["options"] = {"slug": slug, "operator_uid": user.uid}
|
||||
return {"tools": [{"slug": "calculator", "name": "Calculator"}], "mcps": ["mcp-a"], "skills": ["other"]}
|
||||
|
||||
monkeypatch.setattr("server.routers.skill_router.get_manageable_skill_or_raise", fake_get_manageable_skill_or_raise)
|
||||
monkeypatch.setattr("server.routers.skill_router.get_skill_dependency_options", fake_get_skill_dependency_options)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.get("/api/system/skills/dependency-options?slug=demo")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["data"]["skills"] == ["other"]
|
||||
assert captured["manageable"] == {"slug": "demo", "operator_uid": "admin"}
|
||||
assert captured["options"] == {"slug": "demo", "operator_uid": "admin"}
|
||||
|
||||
|
||||
def test_skill_tree_and_file_routes_check_management_read_permission(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_get_management_readable_skill_or_raise(_db, user, slug):
|
||||
captured.setdefault("read", []).append({"slug": slug, "operator_uid": user.uid})
|
||||
return _skill(slug=slug, created_by="user", enabled=False)
|
||||
|
||||
async def fake_get_skill_tree(_db, slug):
|
||||
captured["tree_slug"] = slug
|
||||
return [{"name": "SKILL.md", "path": "SKILL.md", "is_dir": False}]
|
||||
|
||||
async def fake_read_skill_file(_db, slug, path):
|
||||
captured["file"] = {"slug": slug, "path": path}
|
||||
return {"path": path, "content": "---\nname: demo\n---\n"}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"server.routers.skill_router.get_management_readable_skill_or_raise",
|
||||
fake_get_management_readable_skill_or_raise,
|
||||
)
|
||||
monkeypatch.setattr("server.routers.skill_router.get_skill_tree", fake_get_skill_tree)
|
||||
monkeypatch.setattr("server.routers.skill_router.read_skill_file", fake_read_skill_file)
|
||||
|
||||
client = TestClient(_build_app(role="user"))
|
||||
tree_resp = client.get("/api/system/skills/demo/tree")
|
||||
file_resp = client.get("/api/system/skills/demo/file?path=SKILL.md")
|
||||
|
||||
assert tree_resp.status_code == 200, tree_resp.text
|
||||
assert file_resp.status_code == 200, file_resp.text
|
||||
assert captured["read"] == [
|
||||
{"slug": "demo", "operator_uid": "user"},
|
||||
{"slug": "demo", "operator_uid": "user"},
|
||||
]
|
||||
assert captured["tree_slug"] == "demo"
|
||||
assert captured["file"] == {"slug": "demo", "path": "SKILL.md"}
|
||||
|
||||
|
||||
def test_skill_export_route_still_checks_manage_permission(monkeypatch, tmp_path):
|
||||
captured: dict[str, object] = {}
|
||||
export_path = tmp_path / "demo.zip"
|
||||
export_path.write_bytes(b"zip")
|
||||
|
||||
async def fake_get_manageable_skill_or_raise(_db, user, slug):
|
||||
captured["manageable"] = {"slug": slug, "operator_uid": user.uid}
|
||||
return _skill(slug=slug)
|
||||
|
||||
async def fake_export_skill_zip(_db, slug):
|
||||
captured["export_slug"] = slug
|
||||
return str(export_path), "demo.zip"
|
||||
|
||||
monkeypatch.setattr("server.routers.skill_router.get_manageable_skill_or_raise", fake_get_manageable_skill_or_raise)
|
||||
monkeypatch.setattr("server.routers.skill_router.export_skill_zip", fake_export_skill_zip)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.get("/api/system/skills/demo/export")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert captured["manageable"] == {"slug": "demo", "operator_uid": "admin"}
|
||||
assert captured["export_slug"] == "demo"
|
||||
|
||||
|
||||
def test_update_skill_dependencies_route_passes_operator(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_update_skill_dependencies(
|
||||
_db,
|
||||
*,
|
||||
slug,
|
||||
tool_dependencies,
|
||||
mcp_dependencies,
|
||||
skill_dependencies,
|
||||
operator,
|
||||
):
|
||||
captured["slug"] = slug
|
||||
captured["tool_dependencies"] = tool_dependencies
|
||||
captured["mcp_dependencies"] = mcp_dependencies
|
||||
captured["skill_dependencies"] = skill_dependencies
|
||||
captured["operator_uid"] = operator.uid
|
||||
return _skill(slug=slug)
|
||||
|
||||
monkeypatch.setattr("server.routers.skill_router.update_skill_dependencies", fake_update_skill_dependencies)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.put(
|
||||
"/api/system/skills/demo/dependencies",
|
||||
json={
|
||||
"tool_dependencies": ["calculator"],
|
||||
"mcp_dependencies": ["mcp-a"],
|
||||
"skill_dependencies": ["other-skill"],
|
||||
},
|
||||
)
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert captured == {
|
||||
"slug": "demo",
|
||||
"tool_dependencies": ["calculator"],
|
||||
"mcp_dependencies": ["mcp-a"],
|
||||
"skill_dependencies": ["other-skill"],
|
||||
"operator_uid": "admin",
|
||||
}
|
||||
|
||||
|
||||
def test_builtin_routes_require_admin():
|
||||
client = TestClient(_build_app(role="user"))
|
||||
|
||||
resp = client.get("/api/system/skills/builtin")
|
||||
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
def test_sync_builtin_skills_route(monkeypatch):
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
async def fake_init_builtin_skills(_db, *, created_by):
|
||||
captured["created_by"] = created_by
|
||||
return [_skill(slug="builtin-demo", source_type="builtin")]
|
||||
|
||||
monkeypatch.setattr("server.routers.skill_router.init_builtin_skills", fake_init_builtin_skills)
|
||||
|
||||
client = TestClient(_build_app())
|
||||
resp = client.post("/api/system/skills/builtin/sync")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["data"][0]["slug"] == "builtin-demo"
|
||||
assert captured == {"created_by": "admin"}
|
||||
@@ -0,0 +1,204 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from types import SimpleNamespace
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from server.utils.auth_middleware import get_admin_user, get_db, get_required_user
|
||||
|
||||
agent_router_module = importlib.import_module("server.routers.agent_router")
|
||||
|
||||
|
||||
def _user(role: str = "admin"):
|
||||
uid = "admin" if role in {"admin", "superadmin"} else "user"
|
||||
return SimpleNamespace(uid=uid, role=role, department_id=1)
|
||||
|
||||
|
||||
def _agent(slug: str, *, backend_id: str = "ChatbotAgent", is_subagent: bool = False):
|
||||
return SimpleNamespace(
|
||||
id=slug,
|
||||
slug=slug,
|
||||
name=slug,
|
||||
backend_id=backend_id,
|
||||
description="",
|
||||
icon=None,
|
||||
pics=[],
|
||||
config_json={},
|
||||
share_config={"access_level": "user", "user_uids": ["admin"]},
|
||||
is_default=False,
|
||||
is_subagent=is_subagent,
|
||||
can_manage=True,
|
||||
)
|
||||
|
||||
|
||||
class _FakeAgentManager:
|
||||
def get_agent(self, backend_id: str):
|
||||
if backend_id in {"ChatbotAgent", "SubAgentBackend"}:
|
||||
return SimpleNamespace(context_schema=None)
|
||||
return None
|
||||
|
||||
|
||||
class _ListRepo:
|
||||
items = [
|
||||
_agent("chatbot", backend_id="ChatbotAgent"),
|
||||
_agent("worker", backend_id="SubAgentBackend", is_subagent=True),
|
||||
]
|
||||
include_subagent_definition_calls: list[bool] = []
|
||||
get_definition_calls: list[str] = []
|
||||
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def ensure_default_agent(self):
|
||||
return self.items[0]
|
||||
|
||||
async def list_visible(self, *, user, include_subagent_definitions: bool = False):
|
||||
del user
|
||||
self.include_subagent_definition_calls.append(include_subagent_definitions)
|
||||
if include_subagent_definitions:
|
||||
return self.items
|
||||
return [item for item in self.items if not item.is_subagent]
|
||||
|
||||
async def get_visible_by_slug(self, *, slug, user, kind="main"):
|
||||
del user
|
||||
if kind == "any":
|
||||
self.get_definition_calls.append(slug)
|
||||
return next((item for item in self.items if item.slug == slug), None)
|
||||
if kind == "subagent":
|
||||
return next((item for item in self.items if item.slug == slug and item.is_subagent), None)
|
||||
return next((item for item in self.items if item.slug == slug and not item.is_subagent), None)
|
||||
|
||||
async def serialize(self, item, **_kwargs):
|
||||
return dict(item.__dict__)
|
||||
|
||||
|
||||
class _CreateRepo(_ListRepo):
|
||||
created_payload = None
|
||||
|
||||
async def create(self, **kwargs):
|
||||
type(self).created_payload = kwargs
|
||||
return _agent(kwargs["slug"], backend_id=kwargs["backend_id"], is_subagent=kwargs["is_subagent"])
|
||||
|
||||
|
||||
class _RejectingCreateRepo(_ListRepo):
|
||||
async def create(self, **_kwargs):
|
||||
raise ValueError("SubAgentBackend 与 is_subagent 必须保持一致")
|
||||
|
||||
|
||||
def _build_app(monkeypatch, repo_cls, *, role: str = "admin") -> TestClient:
|
||||
monkeypatch.setattr(agent_router_module, "agent_manager", _FakeAgentManager())
|
||||
monkeypatch.setattr(agent_router_module, "AgentRepository", repo_cls)
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(agent_router_module.agent_router, prefix="/api")
|
||||
|
||||
async def fake_db():
|
||||
return None
|
||||
|
||||
async def fake_user():
|
||||
return _user(role)
|
||||
|
||||
app.dependency_overrides[get_db] = fake_db
|
||||
app.dependency_overrides[get_required_user] = fake_user
|
||||
app.dependency_overrides[get_admin_user] = fake_user
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_agent_list_excludes_subagents_by_default(monkeypatch):
|
||||
_ListRepo.include_subagent_definition_calls = []
|
||||
client = _build_app(monkeypatch, _ListRepo)
|
||||
|
||||
response = client.get("/api/agent")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
payload = response.json()
|
||||
assert [agent["slug"] for agent in payload["agents"]] == ["chatbot"]
|
||||
assert _ListRepo.include_subagent_definition_calls == [False]
|
||||
|
||||
|
||||
def test_agent_management_list_can_include_subagents(monkeypatch):
|
||||
_ListRepo.include_subagent_definition_calls = []
|
||||
client = _build_app(monkeypatch, _ListRepo)
|
||||
|
||||
response = client.get("/api/agent?include_subagents=true")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
payload = response.json()
|
||||
assert [agent["slug"] for agent in payload["agents"]] == ["chatbot", "worker"]
|
||||
assert payload["agents"][1]["is_subagent"] is True
|
||||
assert _ListRepo.include_subagent_definition_calls == [True]
|
||||
|
||||
|
||||
def test_agent_detail_can_load_subagent_definition(monkeypatch):
|
||||
_ListRepo.get_definition_calls = []
|
||||
client = _build_app(monkeypatch, _ListRepo)
|
||||
|
||||
response = client.get("/api/agent/worker")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["agent"]["slug"] == "worker"
|
||||
assert response.json()["agent"]["is_subagent"] is True
|
||||
assert _ListRepo.get_definition_calls == ["worker"]
|
||||
|
||||
|
||||
def test_normal_user_can_create_agent(monkeypatch):
|
||||
_CreateRepo.created_payload = None
|
||||
client = _build_app(monkeypatch, _CreateRepo, role="user")
|
||||
|
||||
response = client.post(
|
||||
"/api/agent",
|
||||
json={
|
||||
"name": "Personal Bot",
|
||||
"slug": "personal-bot",
|
||||
"backend_id": "ChatbotAgent",
|
||||
"share_config": {"access_level": "global", "department_ids": [], "user_uids": []},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _CreateRepo.created_payload["creator"].uid == "user"
|
||||
assert _CreateRepo.created_payload["creator"].role == "user"
|
||||
assert _CreateRepo.created_payload["share_config"] == {
|
||||
"access_level": "global",
|
||||
"department_ids": [],
|
||||
"user_uids": [],
|
||||
}
|
||||
|
||||
|
||||
def test_create_subagent_backend_agent_sets_subagent_flag(monkeypatch):
|
||||
_CreateRepo.created_payload = None
|
||||
client = _build_app(monkeypatch, _CreateRepo)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent",
|
||||
json={
|
||||
"name": "Worker",
|
||||
"slug": "worker",
|
||||
"backend_id": "SubAgentBackend",
|
||||
"is_subagent": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _CreateRepo.created_payload["backend_id"] == "SubAgentBackend"
|
||||
assert _CreateRepo.created_payload["is_subagent"] is True
|
||||
assert response.json()["agent"]["is_subagent"] is True
|
||||
|
||||
|
||||
def test_create_subagent_backend_rejects_mismatched_flag(monkeypatch):
|
||||
client = _build_app(monkeypatch, _RejectingCreateRepo)
|
||||
|
||||
response = client.post(
|
||||
"/api/agent",
|
||||
json={
|
||||
"name": "Worker",
|
||||
"slug": "worker",
|
||||
"backend_id": "SubAgentBackend",
|
||||
"is_subagent": False,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
assert "is_subagent" in response.json()["detail"]
|
||||
@@ -0,0 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from server.routers.system_router import system
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_discovery_endpoint_is_public(monkeypatch):
|
||||
monkeypatch.setattr("server.routers.system_router.get_version", lambda: "0.7.1.dev0")
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(system, prefix="/api")
|
||||
response = TestClient(app).get("/api/system/discovery")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["name"] == "Yuxi"
|
||||
assert payload["version"] == "0.7.1.dev0"
|
||||
assert payload["api_prefix"] == "/api"
|
||||
assert payload["capabilities"]["cli"]["browser_login"] is True
|
||||
assert payload["capabilities"]["cli"]["api_key_auth"] is True
|
||||
assert payload["capabilities"]["cli"]["kb_upload"] is True
|
||||
assert payload["endpoints"]["cli_auth_sessions"] == "/api/auth/cli/sessions"
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from server.routers.user_router import get_logged_in_user, get_user_config, update_user_config
|
||||
from yuxi.config import UserConfigSchema
|
||||
from yuxi.storage.postgres.models_business import Base, Department, User
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture()
|
||||
async def session():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
department = Department(name="User Config Dept")
|
||||
user_a = User(
|
||||
username="User A",
|
||||
uid="user_a",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="user",
|
||||
department=department,
|
||||
)
|
||||
user_b = User(
|
||||
username="User B",
|
||||
uid="user_b",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="user",
|
||||
department=department,
|
||||
)
|
||||
db.add_all([department, user_a, user_b])
|
||||
await db.commit()
|
||||
for user in [user_a, user_b]:
|
||||
await db.refresh(user)
|
||||
yield db, user_a, user_b
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_user_config_routes_scope_to_current_user(session):
|
||||
db, user_a, user_b = session
|
||||
|
||||
own_config = await update_user_config(
|
||||
UserConfigSchema(enable_memory=True),
|
||||
current_user=user_a,
|
||||
db=db,
|
||||
)
|
||||
other_config = await get_user_config(current_user=user_b, db=db)
|
||||
|
||||
assert own_config["enable_memory"] is True
|
||||
assert other_config["enable_memory"] is False
|
||||
|
||||
|
||||
async def test_user_config_allows_logged_in_user_without_department():
|
||||
user = User(
|
||||
username="No Dept User",
|
||||
uid="no_dept_user",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="user",
|
||||
)
|
||||
|
||||
assert await get_logged_in_user(user) is user
|
||||
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from server.routers import workspace_router
|
||||
from server.routers.workspace_router import workspace
|
||||
from server.utils.auth_middleware import get_required_user
|
||||
from yuxi.storage.postgres.models_business import User
|
||||
|
||||
|
||||
class SupportsDocuments:
|
||||
supports_documents = True
|
||||
|
||||
|
||||
class NoDocuments:
|
||||
supports_documents = False
|
||||
|
||||
|
||||
class FakeKnowledgeBase:
|
||||
def __init__(self, *, supports: bool = True):
|
||||
self.supports = supports
|
||||
self.list_calls = []
|
||||
|
||||
async def check_accessible(self, _user, _kb_id):
|
||||
return True
|
||||
|
||||
async def get_database_info(self, kb_id):
|
||||
return {
|
||||
"kb_id": kb_id,
|
||||
"name": "知识库",
|
||||
"kb_type": "milvus" if self.supports else "dify",
|
||||
}
|
||||
|
||||
async def list_document_files(self, **kwargs):
|
||||
self.list_calls.append(kwargs)
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
"file_id": "__virtual_folder__:root:资料/",
|
||||
"filename": "资料",
|
||||
"status": "done",
|
||||
"is_folder": True,
|
||||
"parent_id": None,
|
||||
"is_virtual_folder": True,
|
||||
"path_prefix": "资料/",
|
||||
"created_at": None,
|
||||
"updated_at": None,
|
||||
"file_size": 0,
|
||||
},
|
||||
{
|
||||
"file_id": "file_1",
|
||||
"filename": "alpha.pdf",
|
||||
"status": "indexed",
|
||||
"is_folder": False,
|
||||
"parent_id": None,
|
||||
"is_virtual_folder": False,
|
||||
"path_prefix": None,
|
||||
"created_at": "2026-06-20T00:00:00Z",
|
||||
"updated_at": "2026-06-20T01:00:00Z",
|
||||
"file_size": 2048,
|
||||
"has_original_file": True,
|
||||
"has_parsed_markdown": True,
|
||||
},
|
||||
],
|
||||
"page": kwargs["page"],
|
||||
"page_size": kwargs["page_size"],
|
||||
"total": 2,
|
||||
"has_more": False,
|
||||
"parent_id": kwargs["parent_id"],
|
||||
"path_prefix": kwargs["path_prefix"] or "",
|
||||
}
|
||||
|
||||
|
||||
def _build_client(monkeypatch, fake_kb: FakeKnowledgeBase, kb_class) -> TestClient:
|
||||
app = FastAPI()
|
||||
app.include_router(workspace, prefix="/api")
|
||||
|
||||
async def fake_required_user():
|
||||
return User(username="user", uid="user", password_hash="x", role="user", department_id=1)
|
||||
|
||||
app.dependency_overrides[get_required_user] = fake_required_user
|
||||
monkeypatch.setattr(workspace_router, "knowledge_base", fake_kb)
|
||||
monkeypatch.setattr(
|
||||
workspace_router.KnowledgeBaseFactory,
|
||||
"get_kb_class",
|
||||
staticmethod(lambda _kb_type: kb_class),
|
||||
)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_workspace_knowledge_tree_uses_paginated_document_listing(monkeypatch):
|
||||
fake_kb = FakeKnowledgeBase()
|
||||
client = _build_client(monkeypatch, fake_kb, SupportsDocuments)
|
||||
|
||||
response = client.get(
|
||||
"/api/workspace/knowledge/tree",
|
||||
params={
|
||||
"kb_id": "kb_1",
|
||||
"parent_id": "folder_1",
|
||||
"path_prefix": "资料/",
|
||||
"page": 2,
|
||||
"page_size": 100,
|
||||
"files_only": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert fake_kb.list_calls == [
|
||||
{
|
||||
"kb_id": "kb_1",
|
||||
"parent_id": "folder_1",
|
||||
"path_prefix": "资料/",
|
||||
"page": 2,
|
||||
"page_size": 100,
|
||||
"recursive": False,
|
||||
"files_only": True,
|
||||
"include_stats": False,
|
||||
}
|
||||
]
|
||||
payload = response.json()
|
||||
assert payload["page"] == 2
|
||||
assert payload["total"] == 2
|
||||
assert payload["entries"][0]["is_virtual_folder"] is True
|
||||
assert payload["entries"][0]["path"] == "/knowledge/kb_1/virtual/%E8%B5%84%E6%96%99%2F"
|
||||
assert payload["entries"][1]["path"] == "/knowledge/kb_1/file/file_1"
|
||||
assert payload["entries"][1]["size"] == 2048
|
||||
assert payload["entries"][1]["modified_at"] == "2026-06-20T01:00:00Z"
|
||||
|
||||
|
||||
def test_workspace_knowledge_tree_rejects_non_document_kb(monkeypatch):
|
||||
client = _build_client(monkeypatch, FakeKnowledgeBase(supports=False), NoDocuments)
|
||||
|
||||
response = client.get("/api/workspace/knowledge/tree", params={"kb_id": "kb_1"})
|
||||
|
||||
assert response.status_code == 501
|
||||
assert "不支持文件浏览" in response.json()["detail"]
|
||||
@@ -0,0 +1,70 @@
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from server.main import _build_cors_options, _parse_cors_origins
|
||||
|
||||
|
||||
def _client_for_origins(origins: list[str]) -> TestClient:
|
||||
app = FastAPI()
|
||||
app.add_middleware(CORSMiddleware, **_build_cors_options(origins))
|
||||
|
||||
@app.get("/ping")
|
||||
async def ping():
|
||||
return {"ok": True}
|
||||
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _preflight(client: TestClient, origin: str):
|
||||
return client.options(
|
||||
"/ping",
|
||||
headers={
|
||||
"Origin": origin,
|
||||
"Access-Control-Request-Method": "POST",
|
||||
"Access-Control-Request-Headers": "authorization,last-event-id",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_default_development_cors_origins(monkeypatch):
|
||||
monkeypatch.delenv("YUXI_CORS_ORIGINS", raising=False)
|
||||
monkeypatch.setenv("YUXI_ENV", "development")
|
||||
|
||||
assert _parse_cors_origins() == ["http://localhost:5173", "http://127.0.0.1:5173"]
|
||||
|
||||
|
||||
def test_production_does_not_default_to_wildcard(monkeypatch):
|
||||
monkeypatch.delenv("YUXI_CORS_ORIGINS", raising=False)
|
||||
monkeypatch.setenv("YUXI_ENV", "production")
|
||||
|
||||
assert _parse_cors_origins() == []
|
||||
|
||||
|
||||
def test_cors_allows_configured_origin_with_credentials():
|
||||
client = _client_for_origins(["http://localhost:5173"])
|
||||
|
||||
response = _preflight(client, "http://localhost:5173")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["access-control-allow-origin"] == "http://localhost:5173"
|
||||
assert response.headers["access-control-allow-credentials"] == "true"
|
||||
|
||||
|
||||
def test_cors_rejects_unconfigured_origin():
|
||||
client = _client_for_origins(["http://localhost:5173"])
|
||||
|
||||
response = _preflight(client, "https://evil.example")
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "access-control-allow-origin" not in response.headers
|
||||
|
||||
|
||||
def test_wildcard_cors_disables_credentials():
|
||||
client = _client_for_origins(["*"])
|
||||
|
||||
response = _preflight(client, "https://any.example")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["access-control-allow-origin"] == "*"
|
||||
assert "access-control-allow-credentials" not in response.headers
|
||||
@@ -0,0 +1,56 @@
|
||||
import pytest
|
||||
|
||||
from server.routers import model_provider_router
|
||||
from server.routers.model_provider_router import ModelProviderPayload
|
||||
|
||||
|
||||
def test_model_provider_payload_accepts_embedding_and_rerank_urls():
|
||||
payload = ModelProviderPayload(
|
||||
provider_id="mixed-provider",
|
||||
display_name="Mixed Provider",
|
||||
base_url="https://api.example.com/v1",
|
||||
embedding_base_url="https://api.example.com/v1/embeddings",
|
||||
rerank_base_url="https://api.example.com/v1/rerank",
|
||||
capabilities=["chat", "embedding", "rerank"],
|
||||
)
|
||||
|
||||
data = payload.model_dump(exclude_none=True)
|
||||
|
||||
assert data["embedding_base_url"] == "https://api.example.com/v1/embeddings"
|
||||
assert data["rerank_base_url"] == "https://api.example.com/v1/rerank"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_provider_commits_before_refreshing_cache(monkeypatch):
|
||||
calls = []
|
||||
|
||||
class Db:
|
||||
async def commit(self):
|
||||
calls.append("commit")
|
||||
|
||||
class User:
|
||||
username = "admin"
|
||||
|
||||
class Provider:
|
||||
def to_dict(self):
|
||||
return {"provider_id": "alibaba"}
|
||||
|
||||
async def fake_update_provider_config(db, provider_id, data, username):
|
||||
calls.append("update")
|
||||
return Provider()
|
||||
|
||||
async def fake_refresh_model_cache():
|
||||
calls.append("refresh")
|
||||
|
||||
monkeypatch.setattr(model_provider_router, "update_provider_config", fake_update_provider_config)
|
||||
monkeypatch.setattr(model_provider_router, "_refresh_model_cache", fake_refresh_model_cache)
|
||||
|
||||
result = await model_provider_router.update_provider(
|
||||
"alibaba",
|
||||
ModelProviderPayload(enabled_models=[]),
|
||||
current_user=User(),
|
||||
db=Db(),
|
||||
)
|
||||
|
||||
assert result == {"success": True, "data": {"provider_id": "alibaba"}}
|
||||
assert calls == ["update", "commit", "refresh"]
|
||||
@@ -0,0 +1,212 @@
|
||||
from yuxi.agents.backends.sandbox import (
|
||||
VIRTUAL_PATH_PREFIX,
|
||||
ensure_thread_dirs,
|
||||
sandbox_outputs_dir,
|
||||
sandbox_uploads_dir,
|
||||
)
|
||||
from yuxi.agents.buildin.chatbot.state import merge_subagent_runs
|
||||
from yuxi.agents.state import merge_artifacts
|
||||
from yuxi.agents.toolkits.buildin.tools import _normalize_presented_artifact_path
|
||||
from yuxi.services.chat_service import extract_agent_state
|
||||
from yuxi.utils.paths import CONVERSATION_HISTORY_DIR_NAME, LARGE_TOOL_RESULTS_DIR_NAME
|
||||
|
||||
|
||||
def _runtime_with_thread(thread_id: str, uid: str = "user-1"):
|
||||
context = type("RuntimeContext", (), {"thread_id": thread_id, "uid": uid})()
|
||||
return type("RuntimeStub", (), {"context": context})()
|
||||
|
||||
|
||||
def test_merge_artifacts_deduplicates_and_preserves_order():
|
||||
assert merge_artifacts(
|
||||
["/home/gem/user-data/outputs/a.md"],
|
||||
["/home/gem/user-data/outputs/a.md", "/home/gem/user-data/outputs/b.md"],
|
||||
) == [
|
||||
"/home/gem/user-data/outputs/a.md",
|
||||
"/home/gem/user-data/outputs/b.md",
|
||||
]
|
||||
|
||||
|
||||
def test_merge_subagent_runs_does_not_merge_entries_without_run_id():
|
||||
assert merge_subagent_runs(
|
||||
[{"id": "run-1", "status": "completed"}],
|
||||
[
|
||||
{"id": "run-1", "status": "failed", "error": "boom"},
|
||||
{"id": "run-2", "status": "completed"},
|
||||
],
|
||||
) == [
|
||||
{"id": "run-1", "status": "completed"},
|
||||
{"id": "run-1", "status": "failed", "error": "boom"},
|
||||
{"id": "run-2", "status": "completed"},
|
||||
]
|
||||
|
||||
|
||||
def test_merge_subagent_runs_updates_existing_run_by_run_id():
|
||||
assert merge_subagent_runs(
|
||||
[
|
||||
{
|
||||
"id": "tool-1",
|
||||
"run_id": "agent-run-1",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "running",
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"id": "tool-1",
|
||||
"run_id": "agent-run-1",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "completed",
|
||||
}
|
||||
],
|
||||
) == [
|
||||
{
|
||||
"id": "tool-1",
|
||||
"run_id": "agent-run-1",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "completed",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_merge_subagent_runs_keeps_continuation_run_history():
|
||||
assert merge_subagent_runs(
|
||||
[
|
||||
{
|
||||
"id": "tool-old",
|
||||
"run_id": "agent-run-old",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "completed",
|
||||
"completed_at": "2026-06-20T01:00:00Z",
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"id": "tool-new",
|
||||
"run_id": "agent-run-new",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "pending",
|
||||
}
|
||||
],
|
||||
) == [
|
||||
{
|
||||
"id": "tool-old",
|
||||
"run_id": "agent-run-old",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "completed",
|
||||
"completed_at": "2026-06-20T01:00:00Z",
|
||||
},
|
||||
{
|
||||
"id": "tool-new",
|
||||
"run_id": "agent-run-new",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "pending",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_merge_subagent_runs_does_not_merge_different_run_ids_by_state_id():
|
||||
assert merge_subagent_runs(
|
||||
[
|
||||
{
|
||||
"id": "tool-1",
|
||||
"run_id": "agent-run-old",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "completed",
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"id": "tool-1",
|
||||
"run_id": "agent-run-new",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "pending",
|
||||
}
|
||||
],
|
||||
) == [
|
||||
{
|
||||
"id": "tool-1",
|
||||
"run_id": "agent-run-old",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "completed",
|
||||
},
|
||||
{
|
||||
"id": "tool-1",
|
||||
"run_id": "agent-run-new",
|
||||
"child_thread_id": "child-thread",
|
||||
"status": "pending",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_normalize_presented_artifact_path_accepts_host_path():
|
||||
thread_id = "artifacts-host-path"
|
||||
ensure_thread_dirs(thread_id, "user-1")
|
||||
output_file = sandbox_outputs_dir(thread_id) / "report.md"
|
||||
output_file.write_text("# demo", encoding="utf-8")
|
||||
|
||||
normalized = _normalize_presented_artifact_path(str(output_file), _runtime_with_thread(thread_id))
|
||||
|
||||
assert normalized == f"{VIRTUAL_PATH_PREFIX}/outputs/report.md"
|
||||
|
||||
|
||||
def test_normalize_presented_artifact_path_accepts_virtual_path():
|
||||
thread_id = "artifacts-virtual-path"
|
||||
ensure_thread_dirs(thread_id, "user-1")
|
||||
output_file = sandbox_outputs_dir(thread_id) / "summary.txt"
|
||||
output_file.write_text("demo", encoding="utf-8")
|
||||
|
||||
normalized = _normalize_presented_artifact_path(
|
||||
f"{VIRTUAL_PATH_PREFIX}/outputs/summary.txt",
|
||||
_runtime_with_thread(thread_id),
|
||||
)
|
||||
|
||||
assert normalized == f"{VIRTUAL_PATH_PREFIX}/outputs/summary.txt"
|
||||
|
||||
|
||||
def test_normalize_presented_artifact_path_rejects_non_outputs_path():
|
||||
thread_id = "artifacts-reject-path"
|
||||
ensure_thread_dirs(thread_id, "user-1")
|
||||
upload_file = sandbox_uploads_dir(thread_id) / "note.txt"
|
||||
upload_file.write_text("demo", encoding="utf-8")
|
||||
|
||||
try:
|
||||
_normalize_presented_artifact_path(str(upload_file), _runtime_with_thread(thread_id))
|
||||
except ValueError as exc:
|
||||
assert f"{VIRTUAL_PATH_PREFIX}/outputs/" in str(exc)
|
||||
else:
|
||||
raise AssertionError("expected ValueError for non-outputs file")
|
||||
|
||||
|
||||
def test_normalize_presented_artifact_path_rejects_internal_output_files():
|
||||
thread_id = "artifacts-reject-internal"
|
||||
ensure_thread_dirs(thread_id, "user-1")
|
||||
|
||||
for dir_name in [LARGE_TOOL_RESULTS_DIR_NAME, CONVERSATION_HISTORY_DIR_NAME, "large_tool_history"]:
|
||||
output_file = sandbox_outputs_dir(thread_id) / dir_name / "stage.txt"
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_file.write_text("internal", encoding="utf-8")
|
||||
|
||||
try:
|
||||
_normalize_presented_artifact_path(str(output_file), _runtime_with_thread(thread_id))
|
||||
except ValueError as exc:
|
||||
assert "工具调用阶段文件" in str(exc)
|
||||
else:
|
||||
raise AssertionError(f"expected ValueError for internal output file under {dir_name}")
|
||||
|
||||
|
||||
def test_extract_agent_state_includes_artifacts():
|
||||
state = extract_agent_state(
|
||||
{
|
||||
"todos": [{"content": "done", "status": "completed"}],
|
||||
"files": {"/tmp/demo.txt": {"content": ["x"]}},
|
||||
"artifacts": ["/home/gem/user-data/outputs/demo.txt"],
|
||||
"subagent_runs": [{"id": "tool-1", "status": "completed"}],
|
||||
"token_usage": {"llm_input_tokens": 42},
|
||||
}
|
||||
)
|
||||
|
||||
assert state["todos"] == [{"content": "done", "status": "completed"}]
|
||||
assert state["files"] == {"/tmp/demo.txt": {"content": ["x"]}}
|
||||
assert state["artifacts"] == ["/home/gem/user-data/outputs/demo.txt"]
|
||||
assert state["subagent_runs"] == [{"id": "tool-1", "status": "completed"}]
|
||||
assert state["token_usage"] == {"llm_input_tokens": 42}
|
||||
@@ -0,0 +1,537 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from yuxi.services import agent_invocation_service as svc
|
||||
from yuxi.services.input_message_service import build_chat_input_message
|
||||
|
||||
|
||||
class _NoExistingRunRepo:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_run_by_request_id(self, request_id: str):
|
||||
del request_id
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_agent_invocation_run_creates_invocation_metadata(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
current_user = SimpleNamespace(uid="user-1", role="user")
|
||||
|
||||
class AgentRepo:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_visible_by_slug(self, *, slug: str, user, kind="main"):
|
||||
assert kind == "main"
|
||||
assert user is current_user
|
||||
return SimpleNamespace(slug=slug)
|
||||
|
||||
class ConvRepo:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
calls["thread_lookup"] = thread_id
|
||||
return None
|
||||
|
||||
async def add_conversation(self, *, uid: str, agent_id: str, title: str, thread_id: str, metadata: dict):
|
||||
calls["conversation"] = {
|
||||
"uid": uid,
|
||||
"agent_id": agent_id,
|
||||
"title": title,
|
||||
"thread_id": thread_id,
|
||||
"metadata": metadata,
|
||||
}
|
||||
return SimpleNamespace(id=1, thread_id=thread_id)
|
||||
|
||||
async def fake_create_agent_run_view(**kwargs):
|
||||
calls["run_kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"thread_id": kwargs["thread_id"],
|
||||
"status": "pending",
|
||||
"request_id": kwargs["meta"]["request_id"],
|
||||
}
|
||||
|
||||
monkeypatch.setattr(svc, "AgentRepository", AgentRepo)
|
||||
monkeypatch.setattr(svc, "AgentRunRepository", _NoExistingRunRepo)
|
||||
monkeypatch.setattr(svc, "ConversationRepository", ConvRepo)
|
||||
monkeypatch.setattr(svc, "create_agent_run_view", fake_create_agent_run_view)
|
||||
|
||||
result = await svc.create_agent_invocation_run_view(
|
||||
agent_slug="translator",
|
||||
input_message=build_chat_input_message("Hello World"),
|
||||
invocation_metadata={"source": "agent_call", "agent_invocation_meta": {"trace_id": "trace-1"}},
|
||||
requested_thread_id="thread-1",
|
||||
request_id="req-1",
|
||||
model_spec="provider:model",
|
||||
current_user=current_user,
|
||||
db=object(),
|
||||
conversation_title="Agent Call Run",
|
||||
)
|
||||
|
||||
assert calls["conversation"]["metadata"] == {
|
||||
"source": "agent_call",
|
||||
"agent_invocation_meta": {"trace_id": "trace-1"},
|
||||
}
|
||||
assert calls["run_kwargs"]["input_message"].content == "Hello World"
|
||||
assert calls["run_kwargs"]["model_spec"] == "provider:model"
|
||||
assert calls["run_kwargs"]["meta"] == {
|
||||
"request_id": "req-1",
|
||||
"source": "agent_call",
|
||||
"agent_invocation_meta": {"trace_id": "trace-1"},
|
||||
}
|
||||
assert result["run_id"] == "run-1"
|
||||
assert result["thread_id"] == "thread-1"
|
||||
assert result["status"] == "pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_agent_call_run_does_not_commit_conversation_before_run_creation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
calls: dict[str, object] = {}
|
||||
current_user = SimpleNamespace(uid="user-1", role="user")
|
||||
|
||||
class Db:
|
||||
async def commit(self):
|
||||
calls["committed"] = True
|
||||
|
||||
class AgentRepo:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_visible_by_slug(self, *, slug: str, user, kind="main"):
|
||||
del user, kind
|
||||
return SimpleNamespace(slug=slug)
|
||||
|
||||
class ConvRepo:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
calls["thread_lookup"] = thread_id
|
||||
return None
|
||||
|
||||
async def add_conversation(self, *, uid: str, agent_id: str, title: str, thread_id: str, metadata: dict):
|
||||
calls["conversation"] = {
|
||||
"uid": uid,
|
||||
"agent_id": agent_id,
|
||||
"title": title,
|
||||
"thread_id": thread_id,
|
||||
"metadata": metadata,
|
||||
}
|
||||
return SimpleNamespace(id=1, thread_id=thread_id)
|
||||
|
||||
async def create_conversation(self, **_kwargs):
|
||||
raise AssertionError("agent-call conversation must not be committed before run creation")
|
||||
|
||||
async def fake_create_agent_run_view(**_kwargs):
|
||||
raise HTTPException(status_code=422, detail="未找到可用聊天模型: 'missing:model'")
|
||||
|
||||
monkeypatch.setattr(svc, "AgentRepository", AgentRepo)
|
||||
monkeypatch.setattr(svc, "AgentRunRepository", _NoExistingRunRepo)
|
||||
monkeypatch.setattr(svc, "ConversationRepository", ConvRepo)
|
||||
monkeypatch.setattr(svc, "create_agent_run_view", fake_create_agent_run_view)
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await svc.create_agent_invocation_run_view(
|
||||
agent_slug="translator",
|
||||
input_message=build_chat_input_message("Hello"),
|
||||
invocation_metadata={"source": "agent_call"},
|
||||
requested_thread_id="",
|
||||
request_id="req-1",
|
||||
model_spec="missing:model",
|
||||
current_user=current_user,
|
||||
db=Db(),
|
||||
conversation_title="Agent Call Run",
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 422
|
||||
assert "conversation" in calls
|
||||
assert calls.get("committed") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_agent_call_run_parses_openai_content_and_returns_async_payload(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
calls: dict[str, object] = {}
|
||||
current_user = SimpleNamespace(uid="user-1", role="user")
|
||||
|
||||
async def fake_create_agent_invocation_run_view(**kwargs):
|
||||
calls["run_kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"thread_id": kwargs["requested_thread_id"],
|
||||
"status": "pending",
|
||||
"request_id": kwargs["request_id"],
|
||||
}
|
||||
|
||||
async def fail_await_agent_run_result(**_kwargs):
|
||||
raise AssertionError("async_mode must not wait for run result")
|
||||
|
||||
monkeypatch.setattr(svc, "create_agent_invocation_run_view", fake_create_agent_invocation_run_view)
|
||||
monkeypatch.setattr(svc, "await_agent_run_result", fail_await_agent_run_result)
|
||||
|
||||
result = await svc.create_agent_call_run_view(
|
||||
agent_slug=" translator ",
|
||||
messages=[
|
||||
{"role": "assistant", "content": "ignored"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hello"}]},
|
||||
],
|
||||
agent_call_meta={"trace_id": "trace-1"},
|
||||
requested_thread_id=" thread-1 ",
|
||||
request_id=" req-1 ",
|
||||
model_spec="provider:model",
|
||||
async_mode=True,
|
||||
stream=False,
|
||||
current_user=current_user,
|
||||
db=object(),
|
||||
)
|
||||
|
||||
assert result["run_id"] == "run-1"
|
||||
assert result["choices"][0]["finish_reason"] is None
|
||||
assert calls["run_kwargs"]["agent_slug"] == "translator"
|
||||
assert calls["run_kwargs"]["input_message"].content == "hello"
|
||||
assert calls["run_kwargs"]["input_message"].message_type == "text"
|
||||
assert calls["run_kwargs"]["requested_thread_id"] == "thread-1"
|
||||
assert calls["run_kwargs"]["request_id"] == "req-1"
|
||||
assert calls["run_kwargs"]["invocation_metadata"] == {
|
||||
"source": "agent_call",
|
||||
"agent_invocation_meta": {"trace_id": "trace-1"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_agent_call_run_waits_and_wraps_final_result(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
current_user = SimpleNamespace(uid="user-1", role="user")
|
||||
|
||||
async def fake_create_agent_invocation_run_view(**kwargs):
|
||||
calls["run_kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"thread_id": "thread-1",
|
||||
"status": "pending",
|
||||
"request_id": kwargs["request_id"],
|
||||
}
|
||||
|
||||
async def fake_await_agent_run_result(*, run_id: str, current_uid: str):
|
||||
calls["await_kwargs"] = {"run_id": run_id, "current_uid": current_uid}
|
||||
return {
|
||||
"status": "completed",
|
||||
"output": "你好",
|
||||
"agent_slug": "translator",
|
||||
"thread_id": "thread-1",
|
||||
"agent_run_id": run_id,
|
||||
"request_id": "req-1",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(svc, "create_agent_invocation_run_view", fake_create_agent_invocation_run_view)
|
||||
monkeypatch.setattr(svc, "await_agent_run_result", fake_await_agent_run_result)
|
||||
|
||||
result = await svc.create_agent_call_run_view(
|
||||
agent_slug="translator",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
agent_call_meta={},
|
||||
requested_thread_id=None,
|
||||
request_id="req-1",
|
||||
model_spec=None,
|
||||
async_mode=False,
|
||||
stream=False,
|
||||
current_user=current_user,
|
||||
db=object(),
|
||||
)
|
||||
|
||||
assert result["run_id"] == "run-1"
|
||||
assert result["output"] == "你好"
|
||||
assert result["choices"][0]["messages"] == [{"role": "assistant", "content": "你好"}]
|
||||
assert result["choices"][0]["finish_reason"] == "stop"
|
||||
assert result["usage"] == {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}
|
||||
assert calls["await_kwargs"] == {"run_id": "run-1", "current_uid": "user-1"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_agent_eval_run_leaves_thread_resolution_to_invocation_helper(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
calls: dict[str, object] = {}
|
||||
current_user = SimpleNamespace(uid="user-1", role="user")
|
||||
|
||||
async def fake_create_agent_invocation_run_view(**kwargs):
|
||||
calls["run_kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"thread_id": "existing-or-new-thread",
|
||||
"status": "pending",
|
||||
"request_id": kwargs["request_id"],
|
||||
}
|
||||
|
||||
async def fake_await_agent_run_result(*, run_id: str, current_uid: str):
|
||||
calls["await_kwargs"] = {"run_id": run_id, "current_uid": current_uid}
|
||||
return {"status": "completed", "agent_run_id": run_id, "request_id": "eval-req", "output": "ok"}
|
||||
|
||||
monkeypatch.setattr(svc, "create_agent_invocation_run_view", fake_create_agent_invocation_run_view)
|
||||
monkeypatch.setattr(svc, "await_agent_run_result", fake_await_agent_run_result)
|
||||
|
||||
result = await svc.create_agent_eval_run_view(
|
||||
query="question",
|
||||
agent_slug="default-chatbot",
|
||||
evaluation={"dataset_name": "dataset"},
|
||||
meta={"request_id": "eval-req"},
|
||||
image_content=None,
|
||||
model_spec=None,
|
||||
current_user=current_user,
|
||||
db=object(),
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert calls["run_kwargs"]["requested_thread_id"] == ""
|
||||
assert calls["run_kwargs"]["request_id"] == "eval-req"
|
||||
assert calls["await_kwargs"] == {"run_id": "run-1", "current_uid": "user-1"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_agent_eval_run_adds_trajectory_summary_when_requested(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
current_user = SimpleNamespace(uid="user-1", role="user")
|
||||
|
||||
async def fake_create_agent_invocation_run_view(**kwargs):
|
||||
calls["run_kwargs"] = kwargs
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"thread_id": "thread-1",
|
||||
"status": "pending",
|
||||
"request_id": kwargs["request_id"],
|
||||
}
|
||||
|
||||
async def fake_await_agent_run_result(*, run_id: str, current_uid: str):
|
||||
return {
|
||||
"status": "completed",
|
||||
"agent_run_id": run_id,
|
||||
"request_id": "eval-req",
|
||||
"output": "ok",
|
||||
"langfuse_trace_id": "trace-1",
|
||||
}
|
||||
|
||||
async def fake_load_trajectory_summary(run_id: str):
|
||||
calls["trajectory_run_id"] = run_id
|
||||
return {"tool_call_count": 1, "tools": [{"name": "web_search", "call_count": 1, "error_count": 0}]}
|
||||
|
||||
monkeypatch.setattr(svc, "create_agent_invocation_run_view", fake_create_agent_invocation_run_view)
|
||||
monkeypatch.setattr(svc, "await_agent_run_result", fake_await_agent_run_result)
|
||||
monkeypatch.setattr(svc, "_load_trajectory_summary", fake_load_trajectory_summary)
|
||||
|
||||
result = await svc.create_agent_eval_run_view(
|
||||
query="question",
|
||||
agent_slug="default-chatbot",
|
||||
evaluation={},
|
||||
meta={"request_id": "eval-req"},
|
||||
image_content=None,
|
||||
model_spec=None,
|
||||
current_user=current_user,
|
||||
db=object(),
|
||||
include_trajectory_summary=True,
|
||||
)
|
||||
|
||||
assert calls["trajectory_run_id"] == "run-1"
|
||||
assert result["trajectory_summary"] == {
|
||||
"tool_call_count": 1,
|
||||
"tools": [{"name": "web_search", "call_count": 1, "error_count": 0}],
|
||||
"langfuse_trace_id": "trace-1",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_agent_eval_run_ignores_trajectory_summary_load_errors(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
current_user = SimpleNamespace(uid="user-1", role="user")
|
||||
|
||||
async def fake_create_agent_invocation_run_view(**kwargs):
|
||||
return {
|
||||
"run_id": "run-1",
|
||||
"thread_id": "thread-1",
|
||||
"status": "pending",
|
||||
"request_id": kwargs["request_id"],
|
||||
}
|
||||
|
||||
async def fake_await_agent_run_result(*, run_id: str, current_uid: str):
|
||||
del current_uid
|
||||
return {"status": "completed", "agent_run_id": run_id, "request_id": "eval-req", "output": "ok"}
|
||||
|
||||
async def fake_load_trajectory_summary(run_id: str):
|
||||
calls["trajectory_run_id"] = run_id
|
||||
raise RuntimeError("redis unavailable")
|
||||
|
||||
monkeypatch.setattr(svc, "create_agent_invocation_run_view", fake_create_agent_invocation_run_view)
|
||||
monkeypatch.setattr(svc, "await_agent_run_result", fake_await_agent_run_result)
|
||||
monkeypatch.setattr(svc, "_load_trajectory_summary", fake_load_trajectory_summary)
|
||||
|
||||
result = await svc.create_agent_eval_run_view(
|
||||
query="question",
|
||||
agent_slug="default-chatbot",
|
||||
evaluation={},
|
||||
meta={"request_id": "eval-req"},
|
||||
image_content=None,
|
||||
model_spec=None,
|
||||
current_user=current_user,
|
||||
db=object(),
|
||||
include_trajectory_summary=True,
|
||||
)
|
||||
|
||||
assert calls["trajectory_run_id"] == "run-1"
|
||||
assert result == {"status": "completed", "agent_run_id": "run-1", "request_id": "eval-req", "output": "ok"}
|
||||
|
||||
|
||||
def test_build_trajectory_summary_counts_tools_interrupts_and_errors():
|
||||
summary = svc._build_trajectory_summary(
|
||||
[
|
||||
{
|
||||
"seq": "1-0",
|
||||
"event_type": "messages",
|
||||
"payload": {
|
||||
"payload": {
|
||||
"items": [
|
||||
{
|
||||
"stream_event": {
|
||||
"type": "tool_call",
|
||||
"tool_call_id": "call-search",
|
||||
"name": "web_search",
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"seq": "2-0",
|
||||
"event_type": "error",
|
||||
"payload": {
|
||||
"payload": {
|
||||
"chunk": {
|
||||
"event": {
|
||||
"data": {
|
||||
"event": "tool-finished",
|
||||
"tool_call_id": "call-search",
|
||||
"tool_name": "web_search",
|
||||
"error": "timeout",
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
{"seq": "3-0", "event_type": "interrupt", "payload": {"payload": {"reason": "human_approval"}}},
|
||||
]
|
||||
)
|
||||
|
||||
assert summary == {
|
||||
"schema_version": 1,
|
||||
"source": "run_events",
|
||||
"event_count": 3,
|
||||
"events_truncated": False,
|
||||
"event_range": {"first_seq": "1-0", "last_seq": "3-0"},
|
||||
"tool_call_count": 1,
|
||||
"tool_error_count": 1,
|
||||
"interrupt_count": 1,
|
||||
"tools": [{"name": "web_search", "call_count": 1, "error_count": 1}],
|
||||
}
|
||||
|
||||
|
||||
def test_build_trajectory_summary_counts_human_interrupt_once_with_end_event():
|
||||
interrupt_chunk = {
|
||||
"status": "human_approval_required",
|
||||
"message": "approve?",
|
||||
}
|
||||
|
||||
summary = svc._build_trajectory_summary(
|
||||
[
|
||||
{
|
||||
"seq": "1-0",
|
||||
"event_type": "interrupt",
|
||||
"payload": {"payload": {"reason": "human_approval", "chunk": interrupt_chunk}},
|
||||
},
|
||||
{
|
||||
"seq": "2-0",
|
||||
"event_type": "end",
|
||||
"payload": {"payload": {"status": "interrupted", "chunk": interrupt_chunk}},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert summary["interrupt_count"] == 1
|
||||
|
||||
|
||||
def test_build_trajectory_summary_matches_no_id_tool_finish_to_start():
|
||||
summary = svc._build_trajectory_summary(
|
||||
[
|
||||
{
|
||||
"seq": None,
|
||||
"event_type": "messages",
|
||||
"payload": {
|
||||
"payload": {
|
||||
"items": [
|
||||
{
|
||||
"stream_event": {
|
||||
"type": "tool_call",
|
||||
"name": "read_file",
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"seq": None,
|
||||
"event_type": "error",
|
||||
"payload": {
|
||||
"payload": {
|
||||
"chunk": {
|
||||
"event": {
|
||||
"data": {
|
||||
"event": "tool-finished",
|
||||
"tool_name": "read_file",
|
||||
"error": "file not found",
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert summary["event_range"] == {"first_seq": None, "last_seq": None}
|
||||
assert summary["tool_call_count"] == 1
|
||||
assert summary["tool_error_count"] == 1
|
||||
assert summary["tools"] == [{"name": "read_file", "call_count": 1, "error_count": 1}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_trajectory_summary_reads_run_events_with_limit(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
async def fake_list_run_stream_events(run_id: str, *, after_seq: str, limit: int):
|
||||
calls["run_id"] = run_id
|
||||
calls["after_seq"] = after_seq
|
||||
calls["limit"] = limit
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(svc, "list_run_stream_events", fake_list_run_stream_events)
|
||||
|
||||
summary = await svc._load_trajectory_summary("run-1")
|
||||
|
||||
assert calls == {
|
||||
"run_id": "run-1",
|
||||
"after_seq": "0-0",
|
||||
"limit": svc.TRAJECTORY_SUMMARY_EVENT_LIMIT,
|
||||
}
|
||||
assert summary["event_count"] == 0
|
||||
assert summary["tools"] == []
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from yuxi.services.auth_service import (
|
||||
CLIAuthError,
|
||||
approve_cli_auth_session,
|
||||
create_cli_auth_session,
|
||||
exchange_cli_auth_token,
|
||||
get_cli_auth_session_for_user,
|
||||
)
|
||||
from yuxi.storage.postgres.models_business import Base, Department, User
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture()
|
||||
async def session():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
dept = Department(name="默认部门")
|
||||
user = User(
|
||||
username="Admin",
|
||||
uid="admin",
|
||||
password_hash="$argon2id$placeholder",
|
||||
role="superadmin",
|
||||
department=dept,
|
||||
)
|
||||
db.add_all([dept, user])
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
yield db, user
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_cli_auth_session_pending_then_exchange(session):
|
||||
db, user = session
|
||||
auth_session, device_code = await create_cli_auth_session(db)
|
||||
|
||||
assert device_code.startswith("yxcli_")
|
||||
assert auth_session.user_code
|
||||
|
||||
with pytest.raises(CLIAuthError) as pending:
|
||||
await exchange_cli_auth_token(db, device_code)
|
||||
assert pending.value.code == "authorization_pending"
|
||||
|
||||
loaded = await get_cli_auth_session_for_user(db, auth_session.user_code)
|
||||
assert loaded.status == "pending"
|
||||
|
||||
await approve_cli_auth_session(db, auth_session.user_code, user)
|
||||
token_data = await exchange_cli_auth_token(db, device_code)
|
||||
|
||||
assert token_data["secret"].startswith("yxkey_")
|
||||
assert token_data["api_key"]["user_id"] == user.id
|
||||
assert token_data["user"]["uid"] == "admin"
|
||||
|
||||
|
||||
async def test_cli_auth_session_token_exchange_is_single_use(session):
|
||||
db, user = session
|
||||
auth_session, device_code = await create_cli_auth_session(db)
|
||||
|
||||
await approve_cli_auth_session(db, auth_session.user_code, user)
|
||||
await exchange_cli_auth_token(db, device_code)
|
||||
|
||||
with pytest.raises(CLIAuthError) as consumed:
|
||||
await exchange_cli_auth_token(db, device_code)
|
||||
assert consumed.value.code == "already_consumed"
|
||||
|
||||
|
||||
async def test_cli_auth_session_rejects_unknown_user_code(session):
|
||||
db, _user = session
|
||||
|
||||
with pytest.raises(CLIAuthError) as missing:
|
||||
await get_cli_auth_session_for_user(db, "NOPE-NOPE")
|
||||
assert missing.value.code == "not_found"
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.base import BaseAgent
|
||||
|
||||
|
||||
class _FakeGraph:
|
||||
def __init__(self):
|
||||
self.last_stream_config = None
|
||||
self.last_invoke_config = None
|
||||
|
||||
async def astream(self, payload, *, stream_mode, context, config):
|
||||
self.last_stream_config = config
|
||||
yield SimpleNamespace(model_dump=lambda: {"type": "ai"}), {"node": "llm"}
|
||||
|
||||
async def ainvoke(self, payload, *, context, config):
|
||||
self.last_invoke_config = config
|
||||
return {"messages": []}
|
||||
|
||||
|
||||
class _TestAgent(BaseAgent):
|
||||
name = "test_agent"
|
||||
description = "test"
|
||||
|
||||
async def get_graph(self, **kwargs):
|
||||
if getattr(self, "_graph", None) is None:
|
||||
self._graph = _FakeGraph()
|
||||
return self._graph
|
||||
|
||||
|
||||
_TestAgent.__module__ = "yuxi.agents.tests.fake"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base_agent_stream_messages_passes_callbacks_metadata_and_tags():
|
||||
agent = _TestAgent()
|
||||
|
||||
items = []
|
||||
async for item in agent.stream_messages(
|
||||
["hello"],
|
||||
input_context={"uid": "user-1", "thread_id": "thread-1"},
|
||||
callbacks=["handler-1"],
|
||||
metadata={"langfuse_user_id": "user-1"},
|
||||
tags=["yuxi"],
|
||||
):
|
||||
items.append(item)
|
||||
|
||||
graph = await agent.get_graph()
|
||||
assert len(items) == 1
|
||||
assert graph.last_stream_config == {
|
||||
"configurable": {"thread_id": "thread-1", "uid": "user-1"},
|
||||
"recursion_limit": 300,
|
||||
"callbacks": ["handler-1"],
|
||||
"metadata": {"langfuse_user_id": "user-1"},
|
||||
"tags": ["yuxi"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base_agent_invoke_messages_passes_callbacks_metadata_and_tags():
|
||||
agent = _TestAgent()
|
||||
|
||||
await agent.invoke_messages(
|
||||
["hello"],
|
||||
input_context={"uid": "user-1", "thread_id": "thread-1"},
|
||||
callbacks=["handler-1"],
|
||||
metadata={"langfuse_user_id": "user-1"},
|
||||
tags=["yuxi"],
|
||||
)
|
||||
|
||||
graph = await agent.get_graph()
|
||||
assert graph.last_invoke_config == {
|
||||
"configurable": {"thread_id": "thread-1", "uid": "user-1"},
|
||||
"recursion_limit": 300,
|
||||
"callbacks": ["handler-1"],
|
||||
"metadata": {"langfuse_user_id": "user-1"},
|
||||
"tags": ["yuxi"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base_agent_uses_configured_max_execution_steps():
|
||||
agent = _TestAgent()
|
||||
|
||||
await agent.invoke_messages(
|
||||
["hello"],
|
||||
input_context={"uid": "user-1", "thread_id": "thread-1", "max_execution_steps": 42},
|
||||
)
|
||||
|
||||
graph = await agent.get_graph()
|
||||
assert graph.last_invoke_config["recursion_limit"] == 42
|
||||
@@ -0,0 +1,529 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from langchain.messages import AIMessageChunk, HumanMessage
|
||||
|
||||
from yuxi.services import chat_service as svc
|
||||
from yuxi.services.input_message_service import build_chat_input_message
|
||||
|
||||
|
||||
async def _fake_normalize_agent_context_config(context, **_kwargs):
|
||||
return dict(context or {})
|
||||
|
||||
|
||||
class _FakeContext:
|
||||
def __init__(self):
|
||||
self.thread_id = ""
|
||||
self.uid = ""
|
||||
self.temperature = None
|
||||
|
||||
def update(self, data: dict):
|
||||
for key, value in data.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
|
||||
class _FakeConvRepo:
|
||||
def __init__(self, _db):
|
||||
self.saved_messages: list[dict] = []
|
||||
self.conversations: dict[str, SimpleNamespace] = {}
|
||||
|
||||
def _conversation(self, thread_id: str) -> SimpleNamespace:
|
||||
return self.conversations.setdefault(
|
||||
thread_id,
|
||||
SimpleNamespace(
|
||||
id=1,
|
||||
uid="user-1",
|
||||
agent_id="test-agent",
|
||||
thread_id=thread_id,
|
||||
status="active",
|
||||
extra_metadata={},
|
||||
),
|
||||
)
|
||||
|
||||
async def add_message_by_thread_id(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
role: str,
|
||||
content: str,
|
||||
message_type: str = "text",
|
||||
extra_metadata: dict | None = None,
|
||||
image_content: str | None = None,
|
||||
run_id: str | None = None,
|
||||
request_id: str | None = None,
|
||||
):
|
||||
self.saved_messages.append(
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"role": role,
|
||||
"content": content,
|
||||
"message_type": message_type,
|
||||
"extra_metadata": extra_metadata,
|
||||
"image_content": image_content,
|
||||
"run_id": run_id,
|
||||
"request_id": request_id,
|
||||
}
|
||||
)
|
||||
return SimpleNamespace(id=1)
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
return self._conversation(thread_id)
|
||||
|
||||
async def create_conversation(self, *, uid: str, agent_id: str, thread_id: str, metadata: dict | None = None):
|
||||
conversation = SimpleNamespace(
|
||||
id=1,
|
||||
uid=uid,
|
||||
agent_id=agent_id,
|
||||
thread_id=thread_id,
|
||||
status="active",
|
||||
extra_metadata=metadata or {},
|
||||
)
|
||||
self.conversations[thread_id] = conversation
|
||||
return conversation
|
||||
|
||||
async def get_attachments_by_request_id(self, conversation_id: int, request_id: str):
|
||||
return []
|
||||
|
||||
async def bind_attachments_to_request(self, conversation_id: int, request_id: str, file_ids: list[str]):
|
||||
return []
|
||||
|
||||
|
||||
def test_build_langfuse_run_context_reads_evaluation_from_invocation_meta(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
def fake_build_run_context(**kwargs):
|
||||
calls.update(kwargs)
|
||||
return SimpleNamespace(metadata=kwargs.get("extra_metadata") or {}, tags=kwargs.get("extra_tags") or [])
|
||||
|
||||
monkeypatch.setattr(svc, "build_run_context", fake_build_run_context)
|
||||
|
||||
result = svc._build_langfuse_run_context(
|
||||
current_user=SimpleNamespace(id=1, uid="user-1", username="alice", department_id=7),
|
||||
thread_id="thread-1",
|
||||
agent_id="agent-a",
|
||||
request_id="req-1",
|
||||
operation="agent_chat_stream",
|
||||
meta={
|
||||
"source": "agent_evaluation",
|
||||
"agent_invocation_meta": {
|
||||
"evaluation": {
|
||||
"dataset_name": "dataset-a",
|
||||
"dataset_item_id": "item-1",
|
||||
"experiment_name": "exp-1",
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert result.metadata == {
|
||||
"source": "agent_evaluation",
|
||||
"feature": "agent_evaluation",
|
||||
"evaluation_dataset_name": "dataset-a",
|
||||
"evaluation_dataset_item_id": "item-1",
|
||||
"evaluation_experiment_name": "exp-1",
|
||||
}
|
||||
assert result.tags == ["agent_evaluation", "dataset:dataset-a", "experiment:exp-1"]
|
||||
assert "evaluation" not in result.metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_agent_chat_passes_langfuse_callbacks_and_persists_trace_info(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
class FakeAgent:
|
||||
context_schema = _FakeContext
|
||||
|
||||
async def stream_messages_with_state(self, messages, input_context=None, **kwargs):
|
||||
calls["stream_messages"] = messages
|
||||
calls["stream_input_context"] = input_context
|
||||
calls["stream_kwargs"] = kwargs
|
||||
yield "messages", (AIMessageChunk(content="hello"), {"node": "llm"})
|
||||
|
||||
async def get_graph(self, *, context=None):
|
||||
class FakeGraph:
|
||||
async def aget_state(self, config):
|
||||
return SimpleNamespace(values={"messages": [], "files": {}, "artifacts": []})
|
||||
|
||||
return FakeGraph()
|
||||
|
||||
async def fake_resolve_agent_runtime(**_kwargs):
|
||||
return SimpleNamespace(slug="test-agent", backend_id="ChatbotAgent"), FakeAgent(), {"temperature": 0.1}
|
||||
|
||||
async def fake_save_messages_from_langgraph_state(
|
||||
*, agent_instance, thread_id, conv_repo, config_dict, context, trace_info, run_id=None, request_id=None
|
||||
):
|
||||
calls["saved_state"] = {
|
||||
"thread_id": thread_id,
|
||||
"config_dict": config_dict,
|
||||
"context": context,
|
||||
"trace_info": trace_info,
|
||||
"run_id": run_id,
|
||||
"request_id": request_id,
|
||||
}
|
||||
|
||||
async def fake_guard_check(_content):
|
||||
return False
|
||||
|
||||
async def fake_guard_check_with_keywords(_content):
|
||||
return False
|
||||
|
||||
async def fake_interrupts(agent, langgraph_config, make_chunk, meta, thread_id, context):
|
||||
if False:
|
||||
yield None
|
||||
return
|
||||
|
||||
monkeypatch.setattr(svc, "_resolve_agent_runtime", fake_resolve_agent_runtime)
|
||||
monkeypatch.setattr(svc, "normalize_agent_context_config", _fake_normalize_agent_context_config)
|
||||
monkeypatch.setattr(svc, "ConversationRepository", _FakeConvRepo)
|
||||
monkeypatch.setattr(svc, "save_messages_from_langgraph_state", fake_save_messages_from_langgraph_state)
|
||||
monkeypatch.setattr(svc.content_guard, "check", fake_guard_check)
|
||||
monkeypatch.setattr(svc.content_guard, "check_with_keywords", fake_guard_check_with_keywords)
|
||||
monkeypatch.setattr(svc, "check_and_handle_interrupts", fake_interrupts)
|
||||
monkeypatch.setattr(
|
||||
svc,
|
||||
"_build_langfuse_run_context",
|
||||
lambda **kwargs: SimpleNamespace(
|
||||
callbacks=["handler-1"],
|
||||
metadata={"langfuse_user_id": kwargs["current_user"].uid, "langfuse_session_id": kwargs["thread_id"]},
|
||||
tags=["yuxi", "chat"],
|
||||
trace_id="trace-seeded",
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
svc,
|
||||
"get_trace_info",
|
||||
lambda _run_context: {
|
||||
"langfuse_trace_id": "trace-runtime",
|
||||
"langfuse_session_id": "thread-1",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(svc, "flush_langfuse", lambda: calls.setdefault("flushed", True))
|
||||
|
||||
chunks = []
|
||||
async for chunk in svc.stream_agent_chat(
|
||||
agent_slug="test-agent",
|
||||
thread_id="thread-1",
|
||||
meta={"request_id": "req-1"},
|
||||
input_message=build_chat_input_message("hello"),
|
||||
current_user=SimpleNamespace(id=1, uid="user-1", role="user", department_id="dept-1"),
|
||||
db=object(),
|
||||
):
|
||||
chunks.append(json.loads(chunk.decode("utf-8")))
|
||||
|
||||
assert calls["stream_input_context"] == {
|
||||
"temperature": 0.1,
|
||||
"uid": "user-1",
|
||||
"thread_id": "thread-1",
|
||||
"run_id": None,
|
||||
"request_id": "req-1",
|
||||
}
|
||||
assert calls["stream_kwargs"] == {
|
||||
"callbacks": ["handler-1"],
|
||||
"metadata": {"langfuse_user_id": "user-1", "langfuse_session_id": "thread-1"},
|
||||
"tags": ["yuxi", "chat"],
|
||||
}
|
||||
assert calls["saved_state"]["trace_info"] == {
|
||||
"langfuse_trace_id": "trace-runtime",
|
||||
"langfuse_session_id": "thread-1",
|
||||
}
|
||||
assert calls["saved_state"]["context"].thread_id == "thread-1"
|
||||
assert calls["saved_state"]["context"].uid == "user-1"
|
||||
assert calls["saved_state"]["context"].temperature == 0.1
|
||||
assert chunks[-1]["status"] == "finished"
|
||||
assert calls["flushed"] is True
|
||||
assert isinstance(calls["stream_messages"][0], HumanMessage)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_agent_chat_maps_raw_protocol_events_to_yuxi_stream_events(monkeypatch: pytest.MonkeyPatch):
|
||||
class FakeGraph:
|
||||
async def aget_state(self, _config):
|
||||
return SimpleNamespace(values={"messages": [], "files": {}, "artifacts": []})
|
||||
|
||||
class FakeAgent:
|
||||
context_schema = _FakeContext
|
||||
|
||||
async def stream_messages_with_state(self, messages, input_context=None, **kwargs):
|
||||
del messages, input_context, kwargs
|
||||
metadata = {"run_id": "run-1"}
|
||||
yield "messages", ({"event": "message-start", "id": "msg-1", "role": "ai"}, metadata)
|
||||
yield "messages", ({"event": "content-block-start", "index": 0, "content": {"type": "text"}}, metadata)
|
||||
yield (
|
||||
"messages",
|
||||
(
|
||||
{"event": "content-block-delta", "index": 0, "delta": {"type": "text-delta", "text": "hello"}},
|
||||
metadata,
|
||||
),
|
||||
)
|
||||
yield (
|
||||
"messages",
|
||||
(
|
||||
{
|
||||
"event": "content-block-delta",
|
||||
"index": 1,
|
||||
"delta": {
|
||||
"type": "block-delta",
|
||||
"fields": {
|
||||
"type": "tool_call_chunk",
|
||||
"id": "call-1",
|
||||
"name": "task",
|
||||
"args": '{"description":"do',
|
||||
"index": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
)
|
||||
yield (
|
||||
"messages",
|
||||
(
|
||||
{
|
||||
"event": "content-block-finish",
|
||||
"index": 1,
|
||||
"content": {
|
||||
"type": "tool_call",
|
||||
"id": "call-1",
|
||||
"name": "task",
|
||||
"args": {"description": "do work", "subagent_slug": "worker"},
|
||||
},
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
)
|
||||
yield "messages", ({"event": "message-finish", "usage": {}}, metadata)
|
||||
|
||||
async def stream_messages(self, messages, input_context=None, **kwargs):
|
||||
raise AssertionError("stream_messages fallback should not be used")
|
||||
|
||||
async def get_graph(self, *, context=None):
|
||||
return FakeGraph()
|
||||
|
||||
async def fake_resolve_agent_runtime(**_kwargs):
|
||||
return SimpleNamespace(slug="test-agent", backend_id="ChatbotAgent"), FakeAgent(), {}
|
||||
|
||||
async def fake_save_messages_from_langgraph_state(
|
||||
*, agent_instance, thread_id, conv_repo, config_dict, context, trace_info, run_id=None, request_id=None
|
||||
):
|
||||
del agent_instance, thread_id, conv_repo, config_dict, context, trace_info, run_id, request_id
|
||||
return None
|
||||
|
||||
async def fake_guard_check(_content):
|
||||
return False
|
||||
|
||||
async def fake_guard_check_with_keywords(_content):
|
||||
return False
|
||||
|
||||
async def fake_interrupts(agent, langgraph_config, make_chunk, meta, thread_id, context):
|
||||
if False:
|
||||
yield None
|
||||
return
|
||||
|
||||
monkeypatch.setattr(svc, "_resolve_agent_runtime", fake_resolve_agent_runtime)
|
||||
monkeypatch.setattr(svc, "normalize_agent_context_config", _fake_normalize_agent_context_config)
|
||||
monkeypatch.setattr(svc, "ConversationRepository", _FakeConvRepo)
|
||||
monkeypatch.setattr(svc, "save_messages_from_langgraph_state", fake_save_messages_from_langgraph_state)
|
||||
monkeypatch.setattr(svc.content_guard, "check", fake_guard_check)
|
||||
monkeypatch.setattr(svc.content_guard, "check_with_keywords", fake_guard_check_with_keywords)
|
||||
monkeypatch.setattr(svc, "check_and_handle_interrupts", fake_interrupts)
|
||||
monkeypatch.setattr(
|
||||
svc,
|
||||
"_build_langfuse_run_context",
|
||||
lambda **kwargs: SimpleNamespace(callbacks=[], metadata={}, tags=[], trace_id=None),
|
||||
)
|
||||
monkeypatch.setattr(svc, "get_trace_info", lambda _run_context: {})
|
||||
monkeypatch.setattr(svc, "flush_langfuse", lambda: None)
|
||||
|
||||
chunks = []
|
||||
async for chunk in svc.stream_agent_chat(
|
||||
agent_slug="test-agent",
|
||||
thread_id="thread-1",
|
||||
meta={"request_id": "req-1"},
|
||||
input_message=build_chat_input_message("hello"),
|
||||
current_user=SimpleNamespace(id=1, uid="user-1", role="user", department_id="dept-1"),
|
||||
db=object(),
|
||||
):
|
||||
chunks.append(json.loads(chunk.decode("utf-8")))
|
||||
|
||||
loading_chunks = [chunk for chunk in chunks if chunk.get("status") == "loading"]
|
||||
assert [chunk["stream_event"]["type"] for chunk in loading_chunks] == ["message_delta", "tool_call"]
|
||||
assert loading_chunks[0]["response"] == "hello"
|
||||
assert loading_chunks[0]["stream_event"] == {
|
||||
"type": "message_delta",
|
||||
"message_id": "msg-1",
|
||||
"thread_id": "thread-1",
|
||||
"namespace": [],
|
||||
"content": "hello",
|
||||
}
|
||||
assert loading_chunks[1]["response"] == ""
|
||||
assert loading_chunks[1]["stream_event"] == {
|
||||
"type": "tool_call",
|
||||
"message_id": "msg-1",
|
||||
"tool_call_id": "call-1",
|
||||
"name": "task",
|
||||
"args": {"description": "do work", "subagent_slug": "worker"},
|
||||
"index": 1,
|
||||
"thread_id": "thread-1",
|
||||
"namespace": [],
|
||||
}
|
||||
assert all("msg" not in chunk for chunk in loading_chunks)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_agent_chat_emits_realtime_agent_state_from_values(monkeypatch: pytest.MonkeyPatch):
|
||||
class FakeGraph:
|
||||
async def aget_state(self, _config):
|
||||
return SimpleNamespace(values={"todos": [{"content": "done", "status": "completed"}]})
|
||||
|
||||
class FakeAgent:
|
||||
context_schema = _FakeContext
|
||||
|
||||
async def stream_messages_with_state(self, messages, input_context=None, **kwargs):
|
||||
yield "values", {"messages": [], "todos": [{"content": "step 1", "status": "pending"}]}
|
||||
yield "values", {"messages": [], "todos": [{"content": "step 1", "status": "in_progress"}]}
|
||||
yield "values", {"messages": [], "todos": [{"content": "step 1", "status": "in_progress"}]}
|
||||
yield "messages", (AIMessageChunk(content="hello"), {"node": "llm"})
|
||||
|
||||
async def stream_messages(self, messages, input_context=None, **kwargs):
|
||||
raise AssertionError("stream_messages fallback should not be used")
|
||||
|
||||
async def get_graph(self, *, context=None):
|
||||
return FakeGraph()
|
||||
|
||||
async def fake_resolve_agent_runtime(**_kwargs):
|
||||
return SimpleNamespace(slug="test-agent", backend_id="ChatbotAgent"), FakeAgent(), {}
|
||||
|
||||
async def fake_save_messages_from_langgraph_state(
|
||||
*, agent_instance, thread_id, conv_repo, config_dict, context, trace_info, run_id=None, request_id=None
|
||||
):
|
||||
del agent_instance, thread_id, conv_repo, config_dict, context, trace_info, run_id, request_id
|
||||
return None
|
||||
|
||||
async def fake_guard_check(_content):
|
||||
return False
|
||||
|
||||
async def fake_guard_check_with_keywords(_content):
|
||||
return False
|
||||
|
||||
async def fake_interrupts(agent, langgraph_config, make_chunk, meta, thread_id, context):
|
||||
if False:
|
||||
yield None
|
||||
return
|
||||
|
||||
monkeypatch.setattr(svc, "_resolve_agent_runtime", fake_resolve_agent_runtime)
|
||||
monkeypatch.setattr(svc, "normalize_agent_context_config", _fake_normalize_agent_context_config)
|
||||
monkeypatch.setattr(svc, "ConversationRepository", _FakeConvRepo)
|
||||
monkeypatch.setattr(svc, "save_messages_from_langgraph_state", fake_save_messages_from_langgraph_state)
|
||||
monkeypatch.setattr(svc.content_guard, "check", fake_guard_check)
|
||||
monkeypatch.setattr(svc.content_guard, "check_with_keywords", fake_guard_check_with_keywords)
|
||||
monkeypatch.setattr(svc, "check_and_handle_interrupts", fake_interrupts)
|
||||
monkeypatch.setattr(
|
||||
svc,
|
||||
"_build_langfuse_run_context",
|
||||
lambda **kwargs: SimpleNamespace(callbacks=[], metadata={}, tags=[], trace_id=None),
|
||||
)
|
||||
monkeypatch.setattr(svc, "get_trace_info", lambda _run_context: {})
|
||||
monkeypatch.setattr(svc, "flush_langfuse", lambda: None)
|
||||
|
||||
chunks = []
|
||||
async for chunk in svc.stream_agent_chat(
|
||||
agent_slug="test-agent",
|
||||
thread_id="thread-1",
|
||||
meta={"request_id": "req-1"},
|
||||
input_message=build_chat_input_message("hello"),
|
||||
current_user=SimpleNamespace(id=1, uid="user-1", role="user", department_id="dept-1"),
|
||||
db=object(),
|
||||
):
|
||||
chunks.append(json.loads(chunk.decode("utf-8")))
|
||||
|
||||
agent_state_chunks = [chunk for chunk in chunks if chunk.get("status") == "agent_state"]
|
||||
assert len(agent_state_chunks) == 3
|
||||
assert agent_state_chunks[0]["agent_state"]["todos"][0]["status"] == "pending"
|
||||
assert agent_state_chunks[1]["agent_state"]["todos"][0]["status"] == "in_progress"
|
||||
assert agent_state_chunks[2]["agent_state"]["todos"][0]["status"] == "completed"
|
||||
assert all("agent_slug" in chunk.get("meta", {}) for chunk in chunks if isinstance(chunk.get("meta"), dict))
|
||||
assert all("agent_id" not in chunk.get("meta", {}) for chunk in chunks if isinstance(chunk.get("meta"), dict))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_agent_chat_maps_custom_compression_event_to_context_compression_chunk(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
class FakeGraph:
|
||||
async def aget_state(self, _config):
|
||||
return SimpleNamespace(values={"messages": []})
|
||||
|
||||
class FakeAgent:
|
||||
context_schema = _FakeContext
|
||||
|
||||
async def stream_messages_with_state(self, messages, input_context=None, **kwargs):
|
||||
yield "custom", {"type": "yuxi.context_compression", "status": "started"}
|
||||
yield "messages", (AIMessageChunk(content="hi"), {"node": "llm"})
|
||||
yield "custom", {
|
||||
"type": "yuxi.context_compression",
|
||||
"status": "completed",
|
||||
"cutoff_index": 5,
|
||||
"file_path": "/conv/x.md",
|
||||
}
|
||||
|
||||
async def stream_messages(self, messages, input_context=None, **kwargs):
|
||||
raise AssertionError("stream_messages fallback should not be used")
|
||||
|
||||
async def get_graph(self, *, context=None):
|
||||
return FakeGraph()
|
||||
|
||||
async def fake_resolve_agent_runtime(**_kwargs):
|
||||
return SimpleNamespace(slug="test-agent", backend_id="ChatbotAgent"), FakeAgent(), {}
|
||||
|
||||
async def fake_save_messages_from_langgraph_state(
|
||||
*, agent_instance, thread_id, conv_repo, config_dict, context, trace_info, run_id=None, request_id=None
|
||||
):
|
||||
del agent_instance, thread_id, conv_repo, config_dict, context, trace_info, run_id, request_id
|
||||
return None
|
||||
|
||||
async def fake_guard_check(_content):
|
||||
return False
|
||||
|
||||
async def fake_guard_check_with_keywords(_content):
|
||||
return False
|
||||
|
||||
async def fake_interrupts(agent, langgraph_config, make_chunk, meta, thread_id, context):
|
||||
if False:
|
||||
yield None
|
||||
return
|
||||
|
||||
monkeypatch.setattr(svc, "_resolve_agent_runtime", fake_resolve_agent_runtime)
|
||||
monkeypatch.setattr(svc, "normalize_agent_context_config", _fake_normalize_agent_context_config)
|
||||
monkeypatch.setattr(svc, "ConversationRepository", _FakeConvRepo)
|
||||
monkeypatch.setattr(svc, "save_messages_from_langgraph_state", fake_save_messages_from_langgraph_state)
|
||||
monkeypatch.setattr(svc.content_guard, "check", fake_guard_check)
|
||||
monkeypatch.setattr(svc.content_guard, "check_with_keywords", fake_guard_check_with_keywords)
|
||||
monkeypatch.setattr(svc, "check_and_handle_interrupts", fake_interrupts)
|
||||
monkeypatch.setattr(
|
||||
svc,
|
||||
"_build_langfuse_run_context",
|
||||
lambda **kwargs: SimpleNamespace(callbacks=[], metadata={}, tags=[], trace_id=None),
|
||||
)
|
||||
monkeypatch.setattr(svc, "get_trace_info", lambda _run_context: {})
|
||||
monkeypatch.setattr(svc, "flush_langfuse", lambda: None)
|
||||
|
||||
chunks = []
|
||||
async for chunk in svc.stream_agent_chat(
|
||||
agent_slug="test-agent",
|
||||
thread_id="thread-1",
|
||||
meta={"request_id": "req-1"},
|
||||
input_message=build_chat_input_message("hello"),
|
||||
current_user=SimpleNamespace(id=1, uid="user-1", role="user", department_id="dept-1"),
|
||||
db=object(),
|
||||
):
|
||||
chunks.append(json.loads(chunk.decode("utf-8")))
|
||||
|
||||
compression_chunks = [chunk for chunk in chunks if chunk.get("status") == "context_compression"]
|
||||
assert len(compression_chunks) == 2
|
||||
assert compression_chunks[0]["compression"]["status"] == "started"
|
||||
assert compression_chunks[1]["compression"]["status"] == "completed"
|
||||
assert compression_chunks[1]["compression"]["cutoff_index"] == 5
|
||||
assert compression_chunks[1]["compression"]["file_path"] == "/conv/x.md"
|
||||
@@ -0,0 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from yuxi.services import chat_service as svc
|
||||
|
||||
|
||||
def test_apply_model_override_sets_model_from_meta():
|
||||
input_context = {"model": "agent-default"}
|
||||
svc._apply_model_override(input_context, {"model_spec": "user-pick"})
|
||||
assert input_context["model"] == "user-pick"
|
||||
|
||||
|
||||
def test_apply_model_override_noop_without_model_spec():
|
||||
input_context = {"model": "agent-default"}
|
||||
svc._apply_model_override(input_context, {"request_id": "r1"})
|
||||
svc._apply_model_override(input_context, None)
|
||||
assert input_context["model"] == "agent-default"
|
||||
@@ -0,0 +1,584 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from langchain.messages import AIMessage, HumanMessage
|
||||
|
||||
from yuxi.agents import context as agent_context
|
||||
from yuxi.services import chat_service as svc
|
||||
|
||||
|
||||
def _empty_agents_prompt(_thread_id: str, _uid: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
async def _fake_normalize_agent_context_config(context, **_kwargs):
|
||||
return dict(context or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_agent_runtime_includes_subagents_only_when_requested(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
class FakeAgentRepository:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_visible_by_slug(self, *, slug: str, user, kind="main"):
|
||||
del user
|
||||
assert slug == "worker"
|
||||
calls.append(kind)
|
||||
if kind == "subagent":
|
||||
return SimpleNamespace(slug="worker", backend_id="SubAgentBackend", config_json={"context": {}})
|
||||
return None
|
||||
|
||||
class FakeConversationRepository:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
return SimpleNamespace(uid="user-1", agent_id="worker", thread_id=thread_id, status="subagent")
|
||||
|
||||
monkeypatch.setattr(svc, "AgentRepository", FakeAgentRepository)
|
||||
monkeypatch.setattr(svc, "ConversationRepository", FakeConversationRepository)
|
||||
monkeypatch.setattr(svc, "normalize_agent_context_config", _fake_normalize_agent_context_config)
|
||||
monkeypatch.setattr(
|
||||
svc.agent_manager,
|
||||
"get_agent",
|
||||
lambda backend_id: SimpleNamespace(context_schema=None) if backend_id == "SubAgentBackend" else None,
|
||||
)
|
||||
|
||||
user = SimpleNamespace(uid="user-1")
|
||||
|
||||
with pytest.raises(ValueError, match="智能体不存在或无权限访问"):
|
||||
await svc._resolve_agent_runtime(
|
||||
db=object(),
|
||||
user=user,
|
||||
requested_agent_slug="worker",
|
||||
thread_id="child-thread",
|
||||
)
|
||||
|
||||
agent_item, backend, agent_config = await svc._resolve_agent_runtime(
|
||||
db=object(),
|
||||
user=user,
|
||||
requested_agent_slug="worker",
|
||||
thread_id="child-thread",
|
||||
agent_kind="subagent",
|
||||
)
|
||||
|
||||
assert calls == ["main", "subagent"]
|
||||
assert agent_item.slug == "worker"
|
||||
assert backend.context_schema is None
|
||||
assert agent_config == {}
|
||||
|
||||
|
||||
class _FakeConvRepo:
|
||||
def __init__(self, _db):
|
||||
self.db = _db
|
||||
self.saved_messages: list[dict] = []
|
||||
self.tool_calls: list[dict] = []
|
||||
self.conversations: dict[str, SimpleNamespace] = {}
|
||||
|
||||
def _conversation(self, thread_id: str) -> SimpleNamespace:
|
||||
return self.conversations.setdefault(
|
||||
thread_id,
|
||||
SimpleNamespace(
|
||||
id=1,
|
||||
uid="user-1",
|
||||
agent_id="test-agent",
|
||||
thread_id=thread_id,
|
||||
status="active",
|
||||
extra_metadata={},
|
||||
),
|
||||
)
|
||||
|
||||
async def add_message_by_thread_id(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
role: str,
|
||||
content: str,
|
||||
message_type: str = "text",
|
||||
extra_metadata: dict | None = None,
|
||||
image_content: str | None = None,
|
||||
run_id: str | None = None,
|
||||
request_id: str | None = None,
|
||||
):
|
||||
self.saved_messages.append(
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"role": role,
|
||||
"content": content,
|
||||
"message_type": message_type,
|
||||
"extra_metadata": extra_metadata,
|
||||
"image_content": image_content,
|
||||
"run_id": run_id,
|
||||
"request_id": request_id,
|
||||
}
|
||||
)
|
||||
return SimpleNamespace(id=1)
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
return self._conversation(thread_id)
|
||||
|
||||
async def get_messages_by_thread_id(self, _thread_id: str):
|
||||
return []
|
||||
|
||||
async def add_tool_call(
|
||||
self,
|
||||
*,
|
||||
message_id: int,
|
||||
tool_name: str,
|
||||
tool_input: dict | None = None,
|
||||
status: str = "pending",
|
||||
langgraph_tool_call_id: str | None = None,
|
||||
):
|
||||
self.tool_calls.append(
|
||||
{
|
||||
"message_id": message_id,
|
||||
"tool_name": tool_name,
|
||||
"tool_input": tool_input,
|
||||
"status": status,
|
||||
"langgraph_tool_call_id": langgraph_tool_call_id,
|
||||
}
|
||||
)
|
||||
return SimpleNamespace(id=len(self.tool_calls))
|
||||
|
||||
async def create_conversation(self, *, uid: str, agent_id: str, thread_id: str, metadata: dict | None = None):
|
||||
conversation = SimpleNamespace(
|
||||
id=1,
|
||||
uid=uid,
|
||||
agent_id=agent_id,
|
||||
thread_id=thread_id,
|
||||
status="active",
|
||||
extra_metadata=metadata or {},
|
||||
)
|
||||
self.conversations[thread_id] = conversation
|
||||
return conversation
|
||||
|
||||
async def get_attachments_by_request_id(self, conversation_id: int, request_id: str):
|
||||
return []
|
||||
|
||||
async def bind_attachments_to_request(self, conversation_id: int, request_id: str, file_ids: list[str]):
|
||||
return []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_messages_from_langgraph_state_handles_dict_tool_call_blocks() -> None:
|
||||
class FakeGraph:
|
||||
async def aget_state(self, _config):
|
||||
return SimpleNamespace(
|
||||
values={
|
||||
"messages": [
|
||||
{
|
||||
"id": "ai-tool-call",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"id": "call-task-1",
|
||||
"name": "task",
|
||||
"args": {"description": "write file", "subagent_slug": "worker"},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
class FakeAgent:
|
||||
async def get_graph(self, *, context):
|
||||
assert context is fake_context
|
||||
return FakeGraph()
|
||||
|
||||
conv_repo = _FakeConvRepo(None)
|
||||
fake_context = object()
|
||||
|
||||
await svc.save_messages_from_langgraph_state(
|
||||
agent_instance=FakeAgent(),
|
||||
thread_id="thread-1",
|
||||
conv_repo=conv_repo,
|
||||
config_dict={"configurable": {"thread_id": "thread-1", "uid": "user-1"}},
|
||||
context=fake_context,
|
||||
trace_info=None,
|
||||
)
|
||||
|
||||
assert conv_repo.saved_messages[0]["content"] == ""
|
||||
assert conv_repo.saved_messages[0]["extra_metadata"]["content"][0]["id"] == "call-task-1"
|
||||
assert conv_repo.tool_calls == [
|
||||
{
|
||||
"message_id": 1,
|
||||
"tool_name": "task",
|
||||
"tool_input": {"description": "write file", "subagent_slug": "worker"},
|
||||
"status": "pending",
|
||||
"langgraph_tool_call_id": "call-task-1",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_messages_from_langgraph_state_backfills_run_output_message(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
class FakeDB:
|
||||
def __init__(self):
|
||||
self.commit_count = 0
|
||||
|
||||
async def commit(self):
|
||||
self.commit_count += 1
|
||||
|
||||
class FakeGraph:
|
||||
async def aget_state(self, _config):
|
||||
return SimpleNamespace(values={"messages": [HumanMessage(content="question"), AIMessage(content="answer")]})
|
||||
|
||||
class FakeAgent:
|
||||
async def get_graph(self, *, context):
|
||||
assert context is fake_context
|
||||
return FakeGraph()
|
||||
|
||||
fake_db = FakeDB()
|
||||
conv_repo = _FakeConvRepo(fake_db)
|
||||
fake_context = object()
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class FakeRunRepo:
|
||||
def __init__(self, db):
|
||||
assert db is fake_db
|
||||
|
||||
async def set_output_message(self, run_id: str, message_id: int):
|
||||
captured["run_id"] = run_id
|
||||
captured["message_id"] = message_id
|
||||
|
||||
monkeypatch.setattr(svc, "AgentRunRepository", FakeRunRepo)
|
||||
|
||||
await svc.save_messages_from_langgraph_state(
|
||||
agent_instance=FakeAgent(),
|
||||
thread_id="thread-1",
|
||||
conv_repo=conv_repo,
|
||||
config_dict={"configurable": {"thread_id": "thread-1", "uid": "user-1"}},
|
||||
context=fake_context,
|
||||
trace_info={"langfuse_trace_id": "trace-1"},
|
||||
run_id="run-1",
|
||||
request_id="req-1",
|
||||
)
|
||||
|
||||
assert conv_repo.saved_messages[0]["content"] == "answer"
|
||||
assert conv_repo.saved_messages[0]["run_id"] == "run-1"
|
||||
assert conv_repo.saved_messages[0]["request_id"] == "req-1"
|
||||
assert conv_repo.saved_messages[0]["extra_metadata"]["langfuse_trace_id"] == "trace-1"
|
||||
assert captured == {"run_id": "run-1", "message_id": 1}
|
||||
assert fake_db.commit_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_agent_input_context_merges_workspace_agents_prompt(monkeypatch: pytest.MonkeyPatch):
|
||||
def fake_agents_prompt(_thread_id: str, _uid: str) -> str:
|
||||
return "回答前先读取 AGENTS.md"
|
||||
|
||||
monkeypatch.setattr(agent_context, "_load_workspace_agents_prompt", fake_agents_prompt)
|
||||
|
||||
context = await agent_context.build_agent_input_context(
|
||||
{"system_prompt": "原始系统提示词", "temperature": 0.1},
|
||||
thread_id="thread-1",
|
||||
uid="user-1",
|
||||
)
|
||||
|
||||
assert context["system_prompt"] == "原始系统提示词\n\n用户工作区 agents/AGENTS.md 内容:\n回答前先读取 AGENTS.md"
|
||||
assert context["temperature"] == 0.1
|
||||
assert context["thread_id"] == "thread-1"
|
||||
assert context["uid"] == "user-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_agent_state_view_rejects_async_subagent_without_child_conversation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
child_thread_id = "missing-child-conversation"
|
||||
|
||||
class ConvRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
del thread_id
|
||||
return None
|
||||
|
||||
class RunRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_latest_subagent_run_by_thread_for_user(self, thread_id: str, uid: str):
|
||||
assert thread_id == child_thread_id
|
||||
assert uid == "user-1"
|
||||
return SimpleNamespace(
|
||||
id="child-run",
|
||||
conversation_thread_id=child_thread_id,
|
||||
agent_slug="worker",
|
||||
status="running",
|
||||
created_by_run_id="parent-run",
|
||||
subagent_thread_relation_id=77,
|
||||
input_payload={"runtime": {"tool_call_id": "tool-1"}},
|
||||
)
|
||||
|
||||
async def get_run_for_user(self, run_id: str, uid: str):
|
||||
del run_id, uid
|
||||
raise AssertionError("async subagent state must be loaded through child conversation relation")
|
||||
|
||||
monkeypatch.setattr(svc, "ConversationRepository", ConvRepo)
|
||||
monkeypatch.setattr(svc, "AgentRunRepository", RunRepo)
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await svc.get_agent_state_view(
|
||||
thread_id=child_thread_id,
|
||||
current_user=SimpleNamespace(uid="user-1"),
|
||||
db=object(),
|
||||
include_messages=True,
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_agent_state_view_includes_subagent_thread_relation(monkeypatch: pytest.MonkeyPatch):
|
||||
child_thread_id = "child-thread"
|
||||
|
||||
class ConvRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
if thread_id == child_thread_id:
|
||||
return SimpleNamespace(id=20, uid="user-1", agent_id="worker", status="subagent")
|
||||
return None
|
||||
|
||||
async def get_conversation_by_id(self, conversation_id: int):
|
||||
assert conversation_id == 11
|
||||
return SimpleNamespace(id=11, thread_id="parent-thread", uid="user-1", status="active")
|
||||
|
||||
class AgentRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_by_slug(self, slug: str):
|
||||
assert slug == "worker"
|
||||
return SimpleNamespace(
|
||||
backend_id="SubAgentBackend",
|
||||
config_json={"context": {}},
|
||||
)
|
||||
|
||||
class ThreadRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_by_child_conversation_for_user(self, child_conversation_id: int, uid: str):
|
||||
assert child_conversation_id == 20
|
||||
assert uid == "user-1"
|
||||
return SimpleNamespace(
|
||||
id=77,
|
||||
parent_conversation_id=11,
|
||||
child_conversation_id=20,
|
||||
child_thread_id=child_thread_id,
|
||||
subagent_slug="worker",
|
||||
to_dict=lambda: {
|
||||
"id": 77,
|
||||
"parent_conversation_id": 11,
|
||||
"child_conversation_id": 20,
|
||||
"child_thread_id": child_thread_id,
|
||||
"subagent_slug": "worker",
|
||||
},
|
||||
)
|
||||
|
||||
class RunRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_latest_run_by_thread_for_user(self, thread_id: str, uid: str):
|
||||
assert thread_id == child_thread_id
|
||||
assert uid == "user-1"
|
||||
return SimpleNamespace(input_payload={"model_spec": "provider:run-model"})
|
||||
|
||||
async def get_latest_subagent_run_by_thread_for_user(self, thread_id: str, uid: str):
|
||||
assert thread_id == child_thread_id
|
||||
assert uid == "user-1"
|
||||
return SimpleNamespace(
|
||||
id="child-run",
|
||||
conversation_thread_id=child_thread_id,
|
||||
agent_slug="worker",
|
||||
uid="user-1",
|
||||
status="running",
|
||||
created_by_run_id="parent-run",
|
||||
subagent_thread_relation_id=77,
|
||||
input_payload={
|
||||
"runtime": {
|
||||
"tool_call_id": "tool-1",
|
||||
"subagent_name": "Worker",
|
||||
"description": "do work",
|
||||
},
|
||||
},
|
||||
error_message=None,
|
||||
created_at=None,
|
||||
finished_at=None,
|
||||
to_dict=lambda: {"created_at": "2026-06-21T01:00:00Z", "finished_at": None},
|
||||
)
|
||||
|
||||
class Graph:
|
||||
async def aget_state(self, config):
|
||||
assert config["configurable"]["thread_id"] == child_thread_id
|
||||
return SimpleNamespace(
|
||||
values={
|
||||
"messages": [HumanMessage(content="do work"), AIMessage(content="working")],
|
||||
"artifacts": ["out.txt"],
|
||||
}
|
||||
)
|
||||
|
||||
class Context:
|
||||
def __init__(self, *, thread_id="", uid=""):
|
||||
self.thread_id = thread_id
|
||||
self.uid = uid
|
||||
self.model = ""
|
||||
|
||||
def update(self, data: dict):
|
||||
for key, value in data.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
class Agent:
|
||||
context_schema = Context
|
||||
|
||||
async def get_graph(self, *, context):
|
||||
assert context.thread_id == child_thread_id
|
||||
assert context.uid == "user-1"
|
||||
assert context.model == "provider:run-model"
|
||||
return Graph()
|
||||
|
||||
monkeypatch.setattr(svc, "ConversationRepository", ConvRepo)
|
||||
monkeypatch.setattr(svc, "AgentRepository", AgentRepo)
|
||||
monkeypatch.setattr(svc, "SubagentThreadRepository", ThreadRepo)
|
||||
monkeypatch.setattr(svc, "AgentRunRepository", RunRepo)
|
||||
monkeypatch.setattr(svc, "normalize_agent_context_config", _fake_normalize_agent_context_config)
|
||||
monkeypatch.setattr(svc.agent_manager, "get_agent", lambda backend_id: Agent())
|
||||
|
||||
result = await svc.get_agent_state_view(
|
||||
thread_id=child_thread_id,
|
||||
current_user=SimpleNamespace(uid="user-1"),
|
||||
db=object(),
|
||||
include_messages=True,
|
||||
)
|
||||
|
||||
assert result["parent_thread_id"] == "parent-thread"
|
||||
assert result["subagent_thread"]["id"] == 77
|
||||
assert result["subagent_run"]["run_id"] == "child-run"
|
||||
assert result["agent_state"]["artifacts"] == ["out.txt"]
|
||||
assert [message["type"] for message in result["messages"]] == ["human", "ai"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_agent_state_view_reports_malformed_subagent_run_as_server_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
child_thread_id = "child-thread"
|
||||
|
||||
class ConvRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
assert thread_id == child_thread_id
|
||||
return SimpleNamespace(id=20, uid="user-1", agent_id="worker", status="subagent")
|
||||
|
||||
async def get_conversation_by_id(self, conversation_id: int):
|
||||
assert conversation_id == 11
|
||||
return SimpleNamespace(id=11, thread_id="parent-thread", uid="user-1", status="active")
|
||||
|
||||
class AgentRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_by_slug(self, slug: str):
|
||||
assert slug == "worker"
|
||||
return SimpleNamespace(backend_id="SubAgentBackend", config_json={"context": {}})
|
||||
|
||||
class ThreadRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_by_child_conversation_for_user(self, child_conversation_id: int, uid: str):
|
||||
assert child_conversation_id == 20
|
||||
assert uid == "user-1"
|
||||
return SimpleNamespace(
|
||||
id=77,
|
||||
parent_conversation_id=11,
|
||||
to_dict=lambda: {"id": 77},
|
||||
)
|
||||
|
||||
class RunRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_latest_run_by_thread_for_user(self, thread_id: str, uid: str):
|
||||
assert thread_id == child_thread_id
|
||||
assert uid == "user-1"
|
||||
return None
|
||||
|
||||
async def get_latest_subagent_run_by_thread_for_user(self, thread_id: str, uid: str):
|
||||
assert thread_id == child_thread_id
|
||||
assert uid == "user-1"
|
||||
return SimpleNamespace(
|
||||
id="child-run",
|
||||
conversation_thread_id=child_thread_id,
|
||||
agent_slug="worker",
|
||||
status="running",
|
||||
input_payload={"runtime": {}},
|
||||
)
|
||||
|
||||
class Graph:
|
||||
async def aget_state(self, _config):
|
||||
return SimpleNamespace(values={})
|
||||
|
||||
class Context:
|
||||
def __init__(self, *, thread_id="", uid=""):
|
||||
self.thread_id = thread_id
|
||||
self.uid = uid
|
||||
|
||||
def update(self, data: dict):
|
||||
for key, value in data.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
class Agent:
|
||||
context_schema = Context
|
||||
|
||||
async def get_graph(self, *, context):
|
||||
assert context.thread_id == child_thread_id
|
||||
assert context.uid == "user-1"
|
||||
return Graph()
|
||||
|
||||
monkeypatch.setattr(svc, "ConversationRepository", ConvRepo)
|
||||
monkeypatch.setattr(svc, "AgentRepository", AgentRepo)
|
||||
monkeypatch.setattr(svc, "SubagentThreadRepository", ThreadRepo)
|
||||
monkeypatch.setattr(svc, "AgentRunRepository", RunRepo)
|
||||
monkeypatch.setattr(svc, "normalize_agent_context_config", _fake_normalize_agent_context_config)
|
||||
monkeypatch.setattr(svc.agent_manager, "get_agent", lambda _backend_id: Agent())
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await svc.get_agent_state_view(
|
||||
thread_id=child_thread_id,
|
||||
current_user=SimpleNamespace(uid="user-1"),
|
||||
db=object(),
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 500
|
||||
assert exc.value.detail == "子智能体运行记录格式异常"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_agent_input_context_keeps_prompt_when_workspace_agents_prompt_empty(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(agent_context, "_load_workspace_agents_prompt", _empty_agents_prompt)
|
||||
|
||||
context = await agent_context.build_agent_input_context(
|
||||
{"system_prompt": "原始系统提示词"},
|
||||
thread_id="thread-1",
|
||||
uid="user-1",
|
||||
)
|
||||
|
||||
assert context["system_prompt"] == "原始系统提示词"
|
||||
@@ -0,0 +1,121 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.services import conversation_service as cs
|
||||
|
||||
|
||||
class _DummyUpload:
|
||||
def __init__(self, *, filename: str, content_type: str | None, data: bytes):
|
||||
self.filename = filename
|
||||
self.content_type = content_type
|
||||
self._buffer = io.BytesIO(data)
|
||||
|
||||
async def read(self, size: int = -1) -> bytes:
|
||||
return self._buffer.read(size)
|
||||
|
||||
async def seek(self, offset: int) -> int:
|
||||
return self._buffer.seek(offset)
|
||||
|
||||
|
||||
def test_build_attachment_storage_path_uses_thread_local_uploads_dir(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(cs.app_config, "save_dir", str(tmp_path))
|
||||
|
||||
virtual_path, host_path = cs._build_attachment_storage_path(
|
||||
uid="u-1",
|
||||
thread_id="t-1",
|
||||
file_name="demo.txt",
|
||||
)
|
||||
|
||||
assert virtual_path == "/home/gem/user-data/uploads/attachments/demo.md"
|
||||
assert host_path == tmp_path / "threads" / "t-1" / "user-data" / "uploads" / "attachments" / "demo.md"
|
||||
|
||||
|
||||
def test_serialize_attachment_includes_original_file_fields() -> None:
|
||||
serialized = cs.serialize_attachment(
|
||||
{
|
||||
"file_id": "f-1",
|
||||
"file_name": "demo.txt",
|
||||
"file_type": "text/plain",
|
||||
"file_size": 5,
|
||||
"status": "parsed",
|
||||
"uploaded_at": "2026-03-25T00:00:00+00:00",
|
||||
"path": "/home/gem/user-data/uploads/attachments/demo.md",
|
||||
"artifact_url": "/api/chat/thread/t-1/artifacts/home/gem/user-data/uploads/attachments/demo.md",
|
||||
"original_path": "/home/gem/user-data/uploads/demo.txt",
|
||||
"original_artifact_url": "/api/chat/thread/t-1/artifacts/home/gem/user-data/uploads/demo.txt",
|
||||
"minio_url": None,
|
||||
}
|
||||
)
|
||||
|
||||
assert serialized["path"] == "/home/gem/user-data/uploads/attachments/demo.md"
|
||||
assert serialized["original_path"] == "/home/gem/user-data/uploads/demo.txt"
|
||||
assert serialized["original_artifact_url"] == "/api/chat/thread/t-1/artifacts/home/gem/user-data/uploads/demo.txt"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_materialize_attachment_files_keeps_original_file_when_markdown_conversion_unsupported(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(cs.app_config, "save_dir", str(tmp_path))
|
||||
|
||||
upload = _DummyUpload(filename="demo.pdf", content_type="application/pdf", data=b"%PDF-test")
|
||||
|
||||
result = await cs._materialize_attachment_files(
|
||||
thread_id="t-1",
|
||||
uid="u-1",
|
||||
upload=upload,
|
||||
file_name="demo.pdf",
|
||||
file_content=b"%PDF-test",
|
||||
)
|
||||
|
||||
assert result["status"] == "uploaded"
|
||||
assert result["path"] == "/home/gem/user-data/uploads/demo.pdf"
|
||||
assert result["original_path"] == "/home/gem/user-data/uploads/demo.pdf"
|
||||
assert (
|
||||
tmp_path / "threads" / "t-1" / "user-data" / "uploads" / "demo.pdf"
|
||||
).read_bytes() == b"%PDF-test"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_materialize_attachment_files_writes_markdown_copy_when_conversion_succeeds(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(cs.app_config, "save_dir", str(tmp_path))
|
||||
|
||||
async def _fake_convert(_upload):
|
||||
return cs.ConversionResult(
|
||||
file_id="f-1",
|
||||
file_name="demo.txt",
|
||||
file_type="text/plain",
|
||||
file_size=5,
|
||||
markdown="hello\nworld",
|
||||
truncated=False,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(cs, "_convert_upload_to_markdown", _fake_convert)
|
||||
|
||||
upload = _DummyUpload(filename="demo.txt", content_type="text/plain", data=b"hello")
|
||||
|
||||
result = await cs._materialize_attachment_files(
|
||||
thread_id="t-1",
|
||||
uid="u-1",
|
||||
upload=upload,
|
||||
file_name="demo.txt",
|
||||
file_content=b"hello",
|
||||
)
|
||||
|
||||
assert result["status"] == "parsed"
|
||||
assert result["path"] == "/home/gem/user-data/uploads/attachments/demo.md"
|
||||
assert result["original_path"] == "/home/gem/user-data/uploads/demo.txt"
|
||||
assert result["file_path"] == "/home/gem/user-data/uploads/attachments/demo.md"
|
||||
assert result["markdown"] == "hello\nworld"
|
||||
assert (tmp_path / "threads" / "t-1" / "user-data" / "uploads" / "demo.txt").read_bytes() == b"hello"
|
||||
assert (
|
||||
tmp_path / "threads" / "t-1" / "user-data" / "uploads" / "attachments" / "demo.md"
|
||||
).read_text(encoding="utf-8") == "hello\nworld"
|
||||
@@ -0,0 +1,302 @@
|
||||
"""测试 chat_service 中的 interrupt 相关函数"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.getcwd())
|
||||
|
||||
from yuxi.services.chat_service import (
|
||||
_normalize_interrupt_questions,
|
||||
_build_ask_user_question_payload,
|
||||
_coerce_interrupt_payload,
|
||||
stream_agent_resume,
|
||||
)
|
||||
from yuxi.services import chat_service as svc
|
||||
from yuxi.utils.question_utils import normalize_options
|
||||
|
||||
|
||||
class TestNormalizeInterruptOptions:
|
||||
"""测试 _normalize_interrupt_options 函数"""
|
||||
|
||||
def test_empty_input(self):
|
||||
assert normalize_options(None) == []
|
||||
assert normalize_options([]) == []
|
||||
|
||||
def test_dict_options(self):
|
||||
raw = [
|
||||
{"label": "选项1", "value": "option1"},
|
||||
{"label": "选项2", "value": "option2"},
|
||||
]
|
||||
result = normalize_options(raw)
|
||||
assert len(result) == 2
|
||||
assert result[0] == {"label": "选项1", "value": "option1"}
|
||||
assert result[1] == {"label": "选项2", "value": "option2"}
|
||||
|
||||
def test_string_options(self):
|
||||
raw = ["选项1", "选项2", "选项3"]
|
||||
result = normalize_options(raw)
|
||||
assert len(result) == 3
|
||||
assert result[0] == {"label": "选项1", "value": "选项1"}
|
||||
|
||||
def test_mixed_options(self):
|
||||
raw = [{"label": "选项1", "value": "option1"}, "选项2"]
|
||||
result = normalize_options(raw)
|
||||
assert len(result) == 2
|
||||
assert result[0] == {"label": "选项1", "value": "option1"}
|
||||
assert result[1] == {"label": "选项2", "value": "选项2"}
|
||||
|
||||
def test_invalid_options(self):
|
||||
raw = [{"label": "只有label"}, {}, " "]
|
||||
result = normalize_options(raw)
|
||||
assert len(result) == 1 # 只有有效的选项
|
||||
assert result[0] == {"label": "只有label", "value": "只有label"}
|
||||
|
||||
def test_value_only(self):
|
||||
raw = [{"value": "only_value"}]
|
||||
result = normalize_options(raw)
|
||||
assert len(result) == 1
|
||||
assert result[0] == {"label": "only_value", "value": "only_value"}
|
||||
|
||||
|
||||
class TestBuildAskUserQuestionPayload:
|
||||
"""测试 _build_ask_user_question_payload 函数"""
|
||||
|
||||
def test_basic_questions(self):
|
||||
info = {
|
||||
"questions": [
|
||||
{
|
||||
"question": "请确认是否继续?",
|
||||
"options": [
|
||||
{"label": "确认", "value": "yes"},
|
||||
{"label": "取消", "value": "no"},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
result = _build_ask_user_question_payload(info, "thread-123")
|
||||
|
||||
assert len(result["questions"]) == 1
|
||||
assert result["questions"][0]["question"] == "请确认是否继续?"
|
||||
assert len(result["questions"][0]["options"]) == 2
|
||||
assert result["questions"][0]["options"][0] == {"label": "确认", "value": "yes"}
|
||||
assert result["questions"][0]["options"][1] == {"label": "取消", "value": "no"}
|
||||
assert result["source"] == "interrupt"
|
||||
assert result["thread_id"] == "thread-123"
|
||||
|
||||
def test_questions_with_source(self):
|
||||
info = {
|
||||
"questions": [{"question": "选择一个选项", "options": ["A", "B", "C"]}],
|
||||
"source": "ask_user_question",
|
||||
}
|
||||
result = _build_ask_user_question_payload(info, "thread-456")
|
||||
|
||||
assert result["source"] == "ask_user_question"
|
||||
assert len(result["questions"][0]["options"]) == 3
|
||||
|
||||
def test_multi_select(self):
|
||||
info = {
|
||||
"questions": [
|
||||
{
|
||||
"question": "选择多个",
|
||||
"options": ["A", "B", "C"],
|
||||
"multi_select": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
result = _build_ask_user_question_payload(info, "thread-789")
|
||||
|
||||
assert result["questions"][0]["multi_select"] is True
|
||||
|
||||
def test_disable_allow_other(self):
|
||||
info = {
|
||||
"questions": [{"question": "只能选择", "options": ["A", "B"], "allow_other": False}],
|
||||
}
|
||||
result = _build_ask_user_question_payload(info, "thread-000")
|
||||
|
||||
assert result["questions"][0]["allow_other"] is False
|
||||
|
||||
def test_with_operation(self):
|
||||
info = {
|
||||
"questions": [
|
||||
{
|
||||
"question": "是否执行操作?",
|
||||
"operation": "删除文件",
|
||||
"options": [
|
||||
{"label": "批准", "value": "approve"},
|
||||
{"label": "拒绝", "value": "reject"},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
result = _build_ask_user_question_payload(info, "thread-op")
|
||||
|
||||
assert result["questions"][0]["operation"] == "删除文件"
|
||||
|
||||
def test_default_question_when_questions_missing(self):
|
||||
info = {}
|
||||
result = _build_ask_user_question_payload(info, "thread-no-opt")
|
||||
|
||||
assert len(result["questions"]) == 1
|
||||
assert result["questions"][0]["question"] == "请选择一个选项"
|
||||
assert result["questions"][0]["options"] == []
|
||||
assert result["source"] == "interrupt"
|
||||
|
||||
def test_question_id_generation(self):
|
||||
"""测试 question_id 自动生成"""
|
||||
info = {"questions": [{"question": "测试?"}]}
|
||||
result = _build_ask_user_question_payload(info, "thread-id")
|
||||
|
||||
assert result["questions"][0]["question_id"] != ""
|
||||
assert len(result["questions"][0]["question_id"]) > 0
|
||||
|
||||
|
||||
class TestNormalizeInterruptQuestions:
|
||||
"""测试 _normalize_interrupt_questions 函数"""
|
||||
|
||||
def test_empty_input(self):
|
||||
assert _normalize_interrupt_questions(None) == []
|
||||
assert _normalize_interrupt_questions([]) == []
|
||||
|
||||
def test_normalize_basic_question(self):
|
||||
raw = [{"question": "Q1", "options": ["A", "B"]}]
|
||||
result = _normalize_interrupt_questions(raw)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["question"] == "Q1"
|
||||
assert result[0]["options"][0] == {"label": "A", "value": "A"}
|
||||
assert result[0]["multi_select"] is False
|
||||
assert result[0]["allow_other"] is True
|
||||
|
||||
def test_invalid_question_filtered(self):
|
||||
raw = [{"question": " "}, "Q2", {"question": "有效问题"}]
|
||||
result = _normalize_interrupt_questions(raw)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["question"] == "有效问题"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_agent_resume_init_does_not_render_resume_input():
|
||||
stream = stream_agent_resume(
|
||||
thread_id="thread-1",
|
||||
resume_input={"language": "python"},
|
||||
meta={"request_id": "req-1"},
|
||||
current_user=SimpleNamespace(uid="user-1"),
|
||||
db=object(),
|
||||
)
|
||||
|
||||
first_chunk = json.loads((await stream.__anext__()).decode("utf-8"))
|
||||
await stream.aclose()
|
||||
|
||||
assert first_chunk["status"] == "init"
|
||||
assert "msg" not in first_chunk
|
||||
assert "Resume with input" not in json.dumps(first_chunk, ensure_ascii=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_agent_resume_routes_subagent_chunks(monkeypatch):
|
||||
class FakeContext:
|
||||
def __init__(self):
|
||||
self.thread_id = None
|
||||
self.uid = None
|
||||
|
||||
def update(self, values):
|
||||
for key, value in values.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
def model_dump(self):
|
||||
return {"thread_id": self.thread_id, "uid": self.uid}
|
||||
|
||||
class FakeAgent:
|
||||
context_schema = FakeContext
|
||||
|
||||
async def stream_resume_with_state(self, resume_command, input_context=None, **kwargs):
|
||||
yield (
|
||||
"messages",
|
||||
(
|
||||
{"content": "child token", "id": "msg-child"},
|
||||
{"namespace": ["task:1"], "thread_id": "child-thread"},
|
||||
),
|
||||
)
|
||||
|
||||
async def get_graph(self, context=None):
|
||||
class FakeGraph:
|
||||
async def aget_state(self, _config):
|
||||
return SimpleNamespace(values={})
|
||||
|
||||
return FakeGraph()
|
||||
|
||||
async def fake_resolve_agent_runtime(**_kwargs):
|
||||
return SimpleNamespace(slug="main-agent", backend_id="ChatbotAgent"), FakeAgent(), {}
|
||||
|
||||
async def fake_save_messages_from_langgraph_state(**_kwargs):
|
||||
return None
|
||||
|
||||
async def fake_check_and_handle_interrupts(*_args, **_kwargs):
|
||||
if False:
|
||||
yield None
|
||||
|
||||
async def fake_build_agent_input_context(*_args, **_kwargs):
|
||||
return {"thread_id": "parent-thread", "uid": "user-1"}
|
||||
|
||||
monkeypatch.setattr(svc, "_resolve_agent_runtime", fake_resolve_agent_runtime)
|
||||
monkeypatch.setattr(svc, "build_agent_input_context", fake_build_agent_input_context)
|
||||
monkeypatch.setattr(
|
||||
svc,
|
||||
"_build_langfuse_run_context",
|
||||
lambda **_kwargs: SimpleNamespace(callbacks=[], metadata={}, tags=[]),
|
||||
)
|
||||
monkeypatch.setattr(svc, "check_and_handle_interrupts", fake_check_and_handle_interrupts)
|
||||
monkeypatch.setattr(svc, "save_messages_from_langgraph_state", fake_save_messages_from_langgraph_state)
|
||||
monkeypatch.setattr(svc, "ConversationRepository", lambda _db: object())
|
||||
monkeypatch.setattr(svc, "flush_langfuse", lambda: None)
|
||||
|
||||
stream = stream_agent_resume(
|
||||
thread_id="parent-thread",
|
||||
resume_input={"ok": True},
|
||||
meta={"request_id": "req-1"},
|
||||
current_user=SimpleNamespace(uid="user-1"),
|
||||
db=object(),
|
||||
)
|
||||
|
||||
chunks = []
|
||||
loading = None
|
||||
async for raw in stream:
|
||||
chunk = json.loads(raw.decode("utf-8"))
|
||||
chunks.append(chunk)
|
||||
if chunk.get("status") == "loading":
|
||||
loading = chunk
|
||||
if chunk.get("status") == "finished":
|
||||
break
|
||||
await stream.aclose()
|
||||
|
||||
assert loading is not None
|
||||
assert loading["thread_id"] == "child-thread"
|
||||
assert loading["response"] == "child token"
|
||||
assert loading["stream_event"]["thread_id"] == "child-thread"
|
||||
finished = chunks[-1]
|
||||
assert finished["status"] == "finished"
|
||||
assert finished["meta"]["agent_slug"] == "main-agent"
|
||||
assert "agent_id" not in finished["meta"]
|
||||
|
||||
|
||||
class TestCoerceInterruptPayload:
|
||||
"""测试 _coerce_interrupt_payload 函数"""
|
||||
|
||||
def test_dict_input(self):
|
||||
info = {"question": "test?", "options": ["a", "b"]}
|
||||
result = _coerce_interrupt_payload(info)
|
||||
assert result == info
|
||||
|
||||
def test_string_input(self):
|
||||
info = "just a string"
|
||||
result = _coerce_interrupt_payload(info)
|
||||
assert isinstance(result, dict)
|
||||
|
||||
def test_none_input(self):
|
||||
result = _coerce_interrupt_payload(None)
|
||||
assert isinstance(result, dict)
|
||||
@@ -0,0 +1,161 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.services import chat_service as chat_svc
|
||||
from yuxi.services import conversation_service as svc
|
||||
|
||||
|
||||
def test_tmp_attachment_ocr_methods_use_processor_factory():
|
||||
assert svc.TMP_ATTACHMENT_OCR_METHODS == tuple(svc.DocumentProcessorFactory.get_available_processors())
|
||||
assert "paddleocr_vl_1_6" in svc.TMP_ATTACHMENT_OCR_METHODS
|
||||
|
||||
|
||||
class _DummyUpload:
|
||||
def __init__(self, *, filename: str, content_type: str | None, data: bytes):
|
||||
self.filename = filename
|
||||
self.content_type = content_type
|
||||
self._buffer = io.BytesIO(data)
|
||||
|
||||
async def read(self, size: int = -1) -> bytes:
|
||||
return self._buffer.read(size)
|
||||
|
||||
async def seek(self, offset: int) -> int:
|
||||
return self._buffer.seek(offset)
|
||||
|
||||
|
||||
def test_build_state_files_only_parsed_and_with_content():
|
||||
attachments = [
|
||||
{
|
||||
"status": "parsed",
|
||||
"file_path": "/attachments/a.md",
|
||||
"markdown": "line1\nline2",
|
||||
"uploaded_at": "2026-02-20T00:00:00+00:00",
|
||||
},
|
||||
{
|
||||
"status": "pending",
|
||||
"file_path": "/attachments/b.md",
|
||||
"markdown": "ignored",
|
||||
},
|
||||
{
|
||||
"status": "parsed",
|
||||
"file_path": "/attachments/c.md",
|
||||
"markdown": "",
|
||||
},
|
||||
]
|
||||
|
||||
files = chat_svc._build_state_files(attachments)
|
||||
|
||||
assert list(files.keys()) == ["/attachments/a.md"]
|
||||
assert files["/attachments/a.md"]["content"] == ["line1", "line2"]
|
||||
assert files["/attachments/a.md"]["created_at"] == "2026-02-20T00:00:00+00:00"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_thread_attachment_state_updates_graph(monkeypatch: pytest.MonkeyPatch):
|
||||
captured: dict = {}
|
||||
|
||||
class FakeGraph:
|
||||
async def aupdate_state(self, *, config, values):
|
||||
captured["write_config"] = config
|
||||
captured["write_values"] = values
|
||||
|
||||
class FakeAgent:
|
||||
async def get_graph(self):
|
||||
return FakeGraph()
|
||||
|
||||
monkeypatch.setattr(svc.agent_manager, "get_agent", lambda _agent_id: FakeAgent())
|
||||
|
||||
attachments = [
|
||||
{
|
||||
"status": "parsed",
|
||||
"path": "/home/gem/user-data/uploads/attachments/resume.md",
|
||||
"file_name": "resume.md",
|
||||
"uploaded_at": "2026-02-20T00:00:00+00:00",
|
||||
}
|
||||
]
|
||||
await svc._sync_thread_upload_state(
|
||||
thread_id="thread-1",
|
||||
uid="u1",
|
||||
agent_id="ChatbotAgent",
|
||||
backend_id=None,
|
||||
attachments=attachments,
|
||||
)
|
||||
|
||||
assert captured["write_config"] == {"configurable": {"thread_id": "thread-1", "uid": "u1"}}
|
||||
assert captured["write_values"] == {"uploads": svc._build_state_uploads(attachments)}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_thread_attachment_state_skips_when_agent_missing(monkeypatch: pytest.MonkeyPatch):
|
||||
warnings: list[str] = []
|
||||
fake_logger = SimpleNamespace(
|
||||
warning=lambda message: warnings.append(message),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(svc, "logger", fake_logger)
|
||||
monkeypatch.setattr(svc.agent_manager, "get_agent", lambda _agent_id: None)
|
||||
|
||||
await svc._sync_thread_upload_state(
|
||||
thread_id="thread-1",
|
||||
uid="u1",
|
||||
agent_id="MissingAgent",
|
||||
backend_id=None,
|
||||
attachments=[],
|
||||
)
|
||||
|
||||
assert any("agent not found" in msg for msg in warnings)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_upload_to_markdown_returns_conversion_result(
|
||||
tmp_path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
async def _fake_aparse(source: str, params=None) -> str:
|
||||
return "converted markdown"
|
||||
|
||||
monkeypatch.setattr(svc, "_ensure_workdir", lambda: tmp_path)
|
||||
monkeypatch.setattr(svc.Parser, "aparse", _fake_aparse)
|
||||
|
||||
payload = b"hello attachment"
|
||||
upload = _DummyUpload(filename="note.txt", content_type="text/plain", data=payload)
|
||||
|
||||
result = await svc._convert_upload_to_markdown(upload)
|
||||
|
||||
assert result.file_name == "note.txt"
|
||||
assert result.file_type == "text/plain"
|
||||
assert result.file_size == len(payload)
|
||||
assert result.markdown == "converted markdown"
|
||||
assert result.truncated is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_upload_to_markdown_truncates_content(
|
||||
tmp_path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
async def _fake_aparse(source: str, params=None) -> str:
|
||||
return "x" * (svc.MAX_ATTACHMENT_MARKDOWN_CHARS + 200)
|
||||
|
||||
monkeypatch.setattr(svc, "_ensure_workdir", lambda: tmp_path)
|
||||
monkeypatch.setattr(svc.Parser, "aparse", _fake_aparse)
|
||||
|
||||
upload = _DummyUpload(filename="note.md", content_type="text/markdown", data=b"hello")
|
||||
|
||||
result = await svc._convert_upload_to_markdown(upload)
|
||||
|
||||
assert result.truncated is True
|
||||
assert f"超出 {svc.MAX_ATTACHMENT_MARKDOWN_CHARS} 字符限制" in result.markdown
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_upload_to_markdown_rejects_unsupported_extension(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(svc, "ATTACHMENT_ALLOWED_EXTENSIONS", (".md",))
|
||||
upload = _DummyUpload(filename="note.pdf", content_type="application/pdf", data=b"pdf")
|
||||
|
||||
with pytest.raises(ValueError, match="不支持的文件类型"):
|
||||
await svc._convert_upload_to_markdown(upload)
|
||||
@@ -0,0 +1,105 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.services import feedback_service as svc
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def scalar_one_or_none(self):
|
||||
return self.value
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self, results):
|
||||
self.results = list(results)
|
||||
self.added = []
|
||||
self.committed = False
|
||||
self.rolled_back = False
|
||||
|
||||
async def execute(self, _query):
|
||||
return _FakeResult(self.results.pop(0))
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
async def commit(self):
|
||||
self.committed = True
|
||||
|
||||
async def refresh(self, item):
|
||||
item.id = 9
|
||||
item.created_at = datetime(2026, 1, 2, 3, 4, 5)
|
||||
|
||||
async def rollback(self):
|
||||
self.rolled_back = True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_submit_message_feedback_syncs_langfuse_score(monkeypatch: pytest.MonkeyPatch):
|
||||
message = SimpleNamespace(
|
||||
id=3,
|
||||
conversation_id=7,
|
||||
extra_metadata={"langfuse_trace_id": "trace-1"},
|
||||
)
|
||||
conversation = SimpleNamespace(id=7, uid="user-1")
|
||||
db = _FakeSession([message, conversation, None])
|
||||
calls = []
|
||||
|
||||
monkeypatch.setattr(svc, "submit_user_feedback_score", lambda **kwargs: calls.append(kwargs) or True)
|
||||
|
||||
result = await svc.submit_message_feedback_view(
|
||||
message_id=3,
|
||||
rating="like",
|
||||
reason=None,
|
||||
db=db,
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"id": 9,
|
||||
"message_id": 3,
|
||||
"rating": "like",
|
||||
"reason": None,
|
||||
"created_at": "2026-01-02T03:04:05",
|
||||
}
|
||||
assert db.committed is True
|
||||
assert db.rolled_back is False
|
||||
assert calls == [
|
||||
{
|
||||
"trace_id": "trace-1",
|
||||
"feedback_id": 9,
|
||||
"message_id": 3,
|
||||
"conversation_id": 7,
|
||||
"uid": "user-1",
|
||||
"rating": "like",
|
||||
"reason": None,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_submit_message_feedback_skips_langfuse_without_trace_id(monkeypatch: pytest.MonkeyPatch):
|
||||
message = SimpleNamespace(id=3, conversation_id=7, extra_metadata={})
|
||||
conversation = SimpleNamespace(id=7, uid="user-1")
|
||||
db = _FakeSession([message, conversation, None])
|
||||
calls = []
|
||||
|
||||
monkeypatch.setattr(svc, "submit_user_feedback_score", lambda **kwargs: calls.append(kwargs) or True)
|
||||
|
||||
result = await svc.submit_message_feedback_view(
|
||||
message_id=3,
|
||||
rating="dislike",
|
||||
reason="不相关",
|
||||
db=db,
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
assert result["rating"] == "dislike"
|
||||
assert result["reason"] == "不相关"
|
||||
assert calls == []
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
from docx import Document
|
||||
|
||||
from yuxi.services.file_preview import (
|
||||
MAX_TEXT_PREVIEW_CHARS,
|
||||
detect_preview_type,
|
||||
is_office_pdf_preview_file,
|
||||
render_preview_payload,
|
||||
)
|
||||
|
||||
|
||||
def _build_docx_bytes(text: str) -> bytes:
|
||||
document = Document()
|
||||
document.add_paragraph(text)
|
||||
buffer = BytesIO()
|
||||
document.save(buffer)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def test_detect_preview_type_does_not_treat_docx_as_markdown_preview():
|
||||
preview_type, supported, message = detect_preview_type("demo.docx", _build_docx_bytes("Docx preview"))
|
||||
|
||||
assert preview_type == "unsupported"
|
||||
assert supported is False
|
||||
assert message == "当前文件是二进制文件,暂不支持预览"
|
||||
|
||||
|
||||
def test_render_preview_payload_rejects_docx_without_parsed_markdown():
|
||||
payload = render_preview_payload("demo.docx", _build_docx_bytes("Docx preview text"))
|
||||
|
||||
assert payload["preview_type"] == "unsupported"
|
||||
assert payload["supported"] is False
|
||||
assert payload["content"] is None
|
||||
|
||||
|
||||
def test_render_preview_payload_truncates_long_markdown():
|
||||
payload = render_preview_payload("note.md", ("x" * (MAX_TEXT_PREVIEW_CHARS + 1)).encode("utf-8"))
|
||||
|
||||
assert payload["preview_type"] == "markdown"
|
||||
assert payload["supported"] is True
|
||||
assert payload["truncated"] is True
|
||||
assert payload["limit"] == MAX_TEXT_PREVIEW_CHARS
|
||||
assert len(payload["content"]) == MAX_TEXT_PREVIEW_CHARS
|
||||
|
||||
|
||||
def test_office_pdf_preview_scope_only_includes_docx_and_pptx():
|
||||
assert is_office_pdf_preview_file("demo.docx") is True
|
||||
assert is_office_pdf_preview_file("demo.pptx") is True
|
||||
assert is_office_pdf_preview_file("demo.xlsx") is False
|
||||
assert is_office_pdf_preview_file("demo.doc") is False
|
||||
assert is_office_pdf_preview_file("demo.ppt") is False
|
||||
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from yuxi.services import langfuse_service as svc
|
||||
|
||||
|
||||
class _FakeLangfuseClient:
|
||||
instances = []
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.scores = []
|
||||
self.flush_count = 0
|
||||
self.raise_on_score = False
|
||||
self.__class__.instances.append(self)
|
||||
|
||||
def create_trace_id(self, *, seed: str | None = None) -> str:
|
||||
return f"trace-{seed}"
|
||||
|
||||
def create_score(self, **kwargs) -> None:
|
||||
if self.raise_on_score:
|
||||
raise RuntimeError("score failed")
|
||||
self.scores.append(kwargs)
|
||||
|
||||
def get_trace_url(self, *, trace_id: str | None = None) -> str | None:
|
||||
if trace_id is None:
|
||||
return None
|
||||
return f"https://langfuse.local/trace/{trace_id}"
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flush_count += 1
|
||||
|
||||
|
||||
class _FakeCallbackHandler:
|
||||
def __init__(self, *, public_key=None, trace_context=None):
|
||||
self.public_key = public_key
|
||||
self.trace_context = trace_context
|
||||
self.last_trace_id = None
|
||||
|
||||
|
||||
def test_build_run_context_includes_trace_metadata(monkeypatch):
|
||||
_FakeLangfuseClient.instances.clear()
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test")
|
||||
monkeypatch.setenv("LANGFUSE_BASE_URL", "https://cloud.langfuse.example")
|
||||
monkeypatch.delenv("LANGFUSE_ENABLED", raising=False)
|
||||
monkeypatch.setattr(svc, "Langfuse", _FakeLangfuseClient)
|
||||
monkeypatch.setattr(svc, "CallbackHandler", _FakeCallbackHandler)
|
||||
svc.get_langfuse_client.cache_clear()
|
||||
|
||||
run_context = svc.build_run_context(
|
||||
user_id="user-1",
|
||||
thread_id="thread-1",
|
||||
agent_id="agent-a",
|
||||
request_id="req-1",
|
||||
operation="agent_chat_stream",
|
||||
backend_id="ChatbotAgent",
|
||||
message_type="text",
|
||||
username="alice",
|
||||
login_user_id="alice-login",
|
||||
department_id=7,
|
||||
)
|
||||
|
||||
assert run_context.trace_id == "trace-req-1"
|
||||
assert len(run_context.callbacks) == 1
|
||||
assert run_context.callbacks[0].trace_context == {"trace_id": "trace-req-1"}
|
||||
assert run_context.metadata["langfuse_user_id"] == "user-1"
|
||||
assert run_context.metadata["langfuse_session_id"] == "thread-1"
|
||||
assert run_context.metadata["backend_id"] == "ChatbotAgent"
|
||||
assert run_context.metadata["department_id"] == "7"
|
||||
assert run_context.tags == [
|
||||
"yuxi",
|
||||
"chat",
|
||||
"agent_chat_stream",
|
||||
"agent:agent-a",
|
||||
"message_type:text",
|
||||
]
|
||||
|
||||
|
||||
def test_build_run_context_merges_evaluation_metadata_and_tags(monkeypatch):
|
||||
monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False)
|
||||
monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False)
|
||||
svc.get_langfuse_client.cache_clear()
|
||||
|
||||
run_context = svc.build_run_context(
|
||||
user_id="user-1",
|
||||
thread_id="thread-1",
|
||||
agent_id="agent-a",
|
||||
request_id="req-1",
|
||||
operation="agent_chat_stream",
|
||||
extra_metadata={
|
||||
"source": "agent_evaluation",
|
||||
"feature": "agent_evaluation",
|
||||
"evaluation": {"dataset_name": "agent-eval-smoke"},
|
||||
},
|
||||
extra_tags=["agent_evaluation", "dataset:agent-eval-smoke", "agent_evaluation"],
|
||||
)
|
||||
|
||||
assert run_context.metadata["source"] == "agent_evaluation"
|
||||
assert run_context.metadata["feature"] == "agent_evaluation"
|
||||
assert run_context.metadata["evaluation"] == {"dataset_name": "agent-eval-smoke"}
|
||||
assert run_context.tags == [
|
||||
"yuxi",
|
||||
"chat",
|
||||
"agent_chat_stream",
|
||||
"agent:agent-a",
|
||||
"agent_evaluation",
|
||||
"dataset:agent-eval-smoke",
|
||||
]
|
||||
|
||||
|
||||
def test_get_trace_info_prefers_handler_last_trace_id(monkeypatch):
|
||||
_FakeLangfuseClient.instances.clear()
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test")
|
||||
monkeypatch.setattr(svc, "Langfuse", _FakeLangfuseClient)
|
||||
monkeypatch.setattr(svc, "CallbackHandler", _FakeCallbackHandler)
|
||||
svc.get_langfuse_client.cache_clear()
|
||||
|
||||
run_context = svc.build_run_context(
|
||||
user_id="user-1",
|
||||
thread_id="thread-1",
|
||||
agent_id="agent-a",
|
||||
request_id="req-1",
|
||||
operation="agent_chat_stream",
|
||||
)
|
||||
run_context.callbacks[0].last_trace_id = "trace-runtime"
|
||||
|
||||
trace_info = svc.get_trace_info(run_context)
|
||||
|
||||
assert trace_info == {
|
||||
"langfuse_trace_id": "trace-runtime",
|
||||
"langfuse_user_id": "user-1",
|
||||
"langfuse_session_id": "thread-1",
|
||||
}
|
||||
|
||||
|
||||
async def test_get_trace_url_async_returns_trace_url(monkeypatch):
|
||||
_FakeLangfuseClient.instances.clear()
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test")
|
||||
monkeypatch.setattr(svc, "Langfuse", _FakeLangfuseClient)
|
||||
monkeypatch.setattr(svc, "CallbackHandler", _FakeCallbackHandler)
|
||||
svc.get_langfuse_client.cache_clear()
|
||||
|
||||
run_context = svc.build_run_context(
|
||||
user_id="user-1",
|
||||
thread_id="thread-1",
|
||||
agent_id="agent-a",
|
||||
request_id="req-1",
|
||||
operation="agent_chat_stream",
|
||||
)
|
||||
run_context.callbacks[0].last_trace_id = "trace-runtime"
|
||||
|
||||
trace_url = await svc.get_trace_url_async(run_context)
|
||||
|
||||
assert trace_url == "https://langfuse.local/trace/trace-runtime"
|
||||
|
||||
|
||||
def test_submit_user_feedback_score_creates_boolean_score(monkeypatch):
|
||||
_FakeLangfuseClient.instances.clear()
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test")
|
||||
monkeypatch.setattr(svc, "Langfuse", _FakeLangfuseClient)
|
||||
monkeypatch.setattr(svc, "CallbackHandler", _FakeCallbackHandler)
|
||||
svc.get_langfuse_client.cache_clear()
|
||||
|
||||
created = svc.submit_user_feedback_score(
|
||||
trace_id="trace-1",
|
||||
feedback_id=12,
|
||||
message_id=34,
|
||||
conversation_id=56,
|
||||
uid="user-1",
|
||||
rating="dislike",
|
||||
reason="答案不准确",
|
||||
)
|
||||
|
||||
client = _FakeLangfuseClient.instances[-1]
|
||||
assert created is True
|
||||
assert client.scores == [
|
||||
{
|
||||
"trace_id": "trace-1",
|
||||
"score_id": "yuxi-message-feedback-12",
|
||||
"name": "user-feedback",
|
||||
"value": 0,
|
||||
"data_type": "BOOLEAN",
|
||||
"comment": "答案不准确",
|
||||
"metadata": {
|
||||
"source": "yuxi",
|
||||
"feedback_id": 12,
|
||||
"message_id": 34,
|
||||
"conversation_id": 56,
|
||||
"uid": "user-1",
|
||||
"rating": "dislike",
|
||||
},
|
||||
}
|
||||
]
|
||||
assert client.flush_count == 1
|
||||
|
||||
|
||||
def test_submit_user_feedback_score_returns_false_when_langfuse_fails(monkeypatch):
|
||||
_FakeLangfuseClient.instances.clear()
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test")
|
||||
monkeypatch.setattr(svc, "Langfuse", _FakeLangfuseClient)
|
||||
monkeypatch.setattr(svc, "CallbackHandler", _FakeCallbackHandler)
|
||||
svc.get_langfuse_client.cache_clear()
|
||||
client = svc.get_langfuse_client()
|
||||
client.raise_on_score = True
|
||||
|
||||
created = svc.submit_user_feedback_score(
|
||||
trace_id="trace-1",
|
||||
feedback_id=12,
|
||||
message_id=34,
|
||||
conversation_id=56,
|
||||
uid="user-1",
|
||||
rating="like",
|
||||
)
|
||||
|
||||
assert created is False
|
||||
assert client.flush_count == 0
|
||||
@@ -0,0 +1,222 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from yuxi.agents.mcp import service as mcp_service
|
||||
from yuxi.storage.postgres import manager as postgres_manager
|
||||
from yuxi.storage.postgres.models_business import MCPServer
|
||||
|
||||
|
||||
class _AsyncSessionContext:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def __aenter__(self):
|
||||
return self.db
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def mcp_session():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(MCPServer.__table__.create)
|
||||
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with session_factory() as session:
|
||||
yield session
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, tools):
|
||||
self._tools = tools
|
||||
|
||||
async def get_tools(self):
|
||||
return self._tools
|
||||
|
||||
|
||||
async def test_ensure_builtin_mcp_servers_removes_retired_system_server(monkeypatch, mcp_session):
|
||||
retired_server = MCPServer(
|
||||
slug="sequentialthinking",
|
||||
name="sequentialthinking",
|
||||
description="old builtin",
|
||||
transport="streamable_http",
|
||||
url="https://remote.mcpservers.org/sequentialthinking/mcp",
|
||||
enabled=1,
|
||||
created_by="system",
|
||||
updated_by="system",
|
||||
)
|
||||
mcp_session.add(retired_server)
|
||||
await mcp_session.commit()
|
||||
|
||||
monkeypatch.setattr(
|
||||
postgres_manager.pg_manager,
|
||||
"get_async_session_context",
|
||||
lambda: _AsyncSessionContext(mcp_session),
|
||||
)
|
||||
|
||||
await mcp_service.ensure_builtin_mcp_servers_in_db()
|
||||
|
||||
retired = await mcp_session.scalar(select(MCPServer).where(MCPServer.slug == "sequentialthinking"))
|
||||
chart = await mcp_session.scalar(select(MCPServer).where(MCPServer.slug == "mcp-server-chart"))
|
||||
assert retired is None
|
||||
assert chart is not None
|
||||
|
||||
|
||||
async def test_ensure_builtin_mcp_servers_preserves_user_server_with_retired_slug(monkeypatch, mcp_session):
|
||||
user_server = MCPServer(
|
||||
slug="sequentialthinking",
|
||||
name="用户自定义 MCP",
|
||||
description="user managed",
|
||||
transport="streamable_http",
|
||||
url="https://example.com/mcp",
|
||||
enabled=1,
|
||||
created_by="admin",
|
||||
updated_by="admin",
|
||||
)
|
||||
mcp_session.add(user_server)
|
||||
await mcp_session.commit()
|
||||
|
||||
monkeypatch.setattr(
|
||||
postgres_manager.pg_manager,
|
||||
"get_async_session_context",
|
||||
lambda: _AsyncSessionContext(mcp_session),
|
||||
)
|
||||
|
||||
await mcp_service.ensure_builtin_mcp_servers_in_db()
|
||||
|
||||
server = await mcp_session.scalar(select(MCPServer).where(MCPServer.slug == "sequentialthinking"))
|
||||
assert server is not None
|
||||
assert server.created_by == "admin"
|
||||
|
||||
|
||||
async def test_get_enabled_mcp_tools_loads_latest_config_from_db(monkeypatch):
|
||||
captured: list[dict] = []
|
||||
|
||||
async def fake_get_enabled_mcp_server_config(server_name: str, db=None):
|
||||
del db
|
||||
assert server_name == "demo"
|
||||
return {"transport": "stdio", "command": "demo", "disabled_tools": ["tool_b"]}
|
||||
|
||||
async def fake_get_mcp_tools(server_name: str, additional_servers=None, disabled_tools=None, **kwargs):
|
||||
del kwargs
|
||||
captured.append(
|
||||
{
|
||||
"server_name": server_name,
|
||||
"additional_servers": additional_servers,
|
||||
"disabled_tools": list(disabled_tools or []),
|
||||
}
|
||||
)
|
||||
return ["tool-a"]
|
||||
|
||||
monkeypatch.setattr(mcp_service, "get_enabled_mcp_server_config", fake_get_enabled_mcp_server_config)
|
||||
monkeypatch.setattr(mcp_service, "get_mcp_tools", fake_get_mcp_tools)
|
||||
|
||||
tools = await mcp_service.get_enabled_mcp_tools("demo")
|
||||
|
||||
assert tools == ["tool-a"]
|
||||
assert captured == [
|
||||
{
|
||||
"server_name": "demo",
|
||||
"additional_servers": {
|
||||
"demo": {"transport": "stdio", "command": "demo", "disabled_tools": ["tool_b"]}
|
||||
},
|
||||
"disabled_tools": ["tool_b"],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def test_get_mcp_tools_rebuilds_cache_when_config_hash_changes(monkeypatch):
|
||||
mcp_service.clear_mcp_cache()
|
||||
|
||||
configs = [
|
||||
{"transport": "stdio", "command": "demo-v1", "disabled_tools": []},
|
||||
{"transport": "stdio", "command": "demo-v2", "disabled_tools": []},
|
||||
]
|
||||
build_calls: list[str] = []
|
||||
|
||||
async def fake_get_enabled_mcp_server_config(server_name: str, db=None):
|
||||
del db
|
||||
assert server_name == "demo"
|
||||
return configs[0]
|
||||
|
||||
async def fake_get_mcp_client(server_configs):
|
||||
config = server_configs["demo"]
|
||||
build_calls.append(config["command"])
|
||||
tool = SimpleNamespace(name=f"tool_for_{config['command']}", metadata={})
|
||||
return _FakeClient([tool])
|
||||
|
||||
monkeypatch.setattr(mcp_service, "get_enabled_mcp_server_config", fake_get_enabled_mcp_server_config)
|
||||
monkeypatch.setattr(mcp_service, "get_mcp_client", fake_get_mcp_client)
|
||||
|
||||
tools_v1_first = await mcp_service.get_mcp_tools("demo")
|
||||
tools_v1_second = await mcp_service.get_mcp_tools("demo")
|
||||
|
||||
configs[0] = configs[1]
|
||||
tools_v2 = await mcp_service.get_mcp_tools("demo")
|
||||
|
||||
assert [tool.name for tool in tools_v1_first] == ["tool_for_demo-v1"]
|
||||
assert [tool.name for tool in tools_v1_second] == ["tool_for_demo-v1"]
|
||||
assert [tool.name for tool in tools_v2] == ["tool_for_demo-v2"]
|
||||
assert build_calls == ["demo-v1", "demo-v2"]
|
||||
|
||||
mcp_service.clear_mcp_cache()
|
||||
|
||||
|
||||
async def test_get_tools_from_all_servers_loads_names_from_db_once(monkeypatch):
|
||||
server_configs = {
|
||||
"alpha": {"transport": "stdio", "command": "cmd-a", "disabled_tools": []},
|
||||
"beta": {"transport": "stdio", "command": "cmd-b", "disabled_tools": []},
|
||||
}
|
||||
calls: list[tuple[str, dict[str, dict]]] = []
|
||||
|
||||
async def fake_load_enabled_mcp_server_configs(*, names=None, db=None):
|
||||
del names, db
|
||||
return server_configs
|
||||
|
||||
async def fake_get_mcp_tools(server_name: str, additional_servers=None, **kwargs):
|
||||
del kwargs
|
||||
calls.append((server_name, additional_servers or {}))
|
||||
return [server_name]
|
||||
|
||||
monkeypatch.setattr(mcp_service, "_load_enabled_mcp_server_configs", fake_load_enabled_mcp_server_configs)
|
||||
monkeypatch.setattr(mcp_service, "get_mcp_tools", fake_get_mcp_tools)
|
||||
|
||||
tools = await mcp_service.get_tools_from_all_servers()
|
||||
|
||||
assert tools == ["alpha", "beta"]
|
||||
assert calls == [
|
||||
("alpha", server_configs),
|
||||
("beta", server_configs),
|
||||
]
|
||||
|
||||
|
||||
async def test_get_mcp_tools_sets_handle_tool_error(monkeypatch):
|
||||
mcp_service.clear_mcp_cache()
|
||||
|
||||
config = {"transport": "stdio", "command": "demo-tool", "disabled_tools": []}
|
||||
|
||||
async def fake_get_enabled_mcp_server_config(server_name: str, db=None):
|
||||
del db
|
||||
return config
|
||||
|
||||
async def fake_get_mcp_client(server_configs):
|
||||
tool = SimpleNamespace(name="demo_tool", metadata={})
|
||||
return _FakeClient([tool])
|
||||
|
||||
monkeypatch.setattr(mcp_service, "get_enabled_mcp_server_config", fake_get_enabled_mcp_server_config)
|
||||
monkeypatch.setattr(mcp_service, "get_mcp_client", fake_get_mcp_client)
|
||||
|
||||
tools = await mcp_service.get_mcp_tools("demo")
|
||||
assert len(tools) == 1
|
||||
assert tools[0].handle_tool_error is True
|
||||
|
||||
mcp_service.clear_mcp_cache()
|
||||
@@ -0,0 +1,249 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
|
||||
import ormsgpack
|
||||
import yuxi.services.mention_search_service as mention_service
|
||||
|
||||
|
||||
class _FakeRedis:
|
||||
def __init__(self):
|
||||
self.data: dict[str, str] = {}
|
||||
self.expire_calls: dict[str, int] = {}
|
||||
self.delete_calls: list[str] = []
|
||||
|
||||
async def get(self, key: str) -> str | None:
|
||||
return self.data.get(key)
|
||||
|
||||
async def set(self, key: str, value: str, ex: int | None = None) -> None:
|
||||
self.data[key] = value
|
||||
if ex is not None:
|
||||
self.expire_calls[key] = ex
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
self.delete_calls.append(key)
|
||||
self.data.pop(key, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_sandbox_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
# 创建模拟的工作区、上传、输出目录
|
||||
workspace_dir = tmp_path / "shared" / "user_1" / "workspace"
|
||||
uploads_dir = tmp_path / "threads" / "thread_1" / "user-data" / "uploads"
|
||||
outputs_dir = tmp_path / "threads" / "thread_1" / "user-data" / "outputs"
|
||||
|
||||
workspace_dir.mkdir(parents=True, exist_ok=True)
|
||||
uploads_dir.mkdir(parents=True, exist_ok=True)
|
||||
outputs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Mock sandbox_paths 的函数
|
||||
monkeypatch.setattr(mention_service, "sandbox_workspace_dir", lambda t, u: workspace_dir)
|
||||
monkeypatch.setattr(mention_service, "sandbox_uploads_dir", lambda t: uploads_dir)
|
||||
monkeypatch.setattr(mention_service, "sandbox_outputs_dir", lambda t: outputs_dir)
|
||||
|
||||
return {
|
||||
"workspace": workspace_dir,
|
||||
"uploads": uploads_dir,
|
||||
"outputs": outputs_dir,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_redis(monkeypatch: pytest.MonkeyPatch) -> _FakeRedis:
|
||||
redis = _FakeRedis()
|
||||
|
||||
async def mock_get_redis():
|
||||
return redis
|
||||
|
||||
monkeypatch.setattr(mention_service, "get_redis_client", mock_get_redis)
|
||||
return redis
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_pruned_files_and_exclude_dirs(mock_sandbox_paths):
|
||||
workspace = mock_sandbox_paths["workspace"]
|
||||
|
||||
# 创建常规文件
|
||||
(workspace / "main.py").write_text("print('hello')")
|
||||
(workspace / "utils.py").write_text("def run(): pass")
|
||||
|
||||
# 创建被排除的目录和文件
|
||||
git_dir = workspace / ".git"
|
||||
git_dir.mkdir()
|
||||
(git_dir / "config").write_text("[core]")
|
||||
|
||||
node_modules = workspace / "node_modules"
|
||||
node_modules.mkdir()
|
||||
(node_modules / "express.js").write_text("module.exports = {}")
|
||||
|
||||
# 扫描
|
||||
results = mention_service._scan_pruned_files(workspace, 100)
|
||||
|
||||
# 校验
|
||||
files = {name for name, _ in results}
|
||||
assert "main.py" in files
|
||||
assert "utils.py" in files
|
||||
assert "config" not in files
|
||||
assert "express.js" not in files
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_depth_protection(mock_sandbox_paths):
|
||||
workspace = mock_sandbox_paths["workspace"]
|
||||
|
||||
# 创建超深的文件树路径:超过 15 层
|
||||
deep_dir = workspace
|
||||
for i in range(18):
|
||||
deep_dir = deep_dir / f"dir_{i}"
|
||||
|
||||
deep_dir.mkdir(parents=True, exist_ok=True)
|
||||
(deep_dir / "deep_file.py").write_text("deep")
|
||||
|
||||
# 扫描
|
||||
results = mention_service._scan_pruned_files(workspace, 100)
|
||||
files = {name for name, _ in results}
|
||||
|
||||
# 深度限制应成功剪枝拦截该超深文件
|
||||
assert "deep_file.py" not in files
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_width_limit(mock_sandbox_paths):
|
||||
workspace = mock_sandbox_paths["workspace"]
|
||||
|
||||
# 创建 600 个扁平的小文件
|
||||
for i in range(600):
|
||||
(workspace / f"file_{i}.py").write_text(str(i))
|
||||
|
||||
# 扫描,设置 max_entries = 1000(看看单目录 500 的宽度限额是否起作用)
|
||||
results = mention_service._scan_pruned_files(workspace, 1000)
|
||||
|
||||
# 限制单目录 MAX_ENTRIES_PER_DIR = 500 熔断
|
||||
assert len(results) == 500
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mention_cache_lifecycle_with_ormsgpack(mock_sandbox_paths, fake_redis):
|
||||
workspace = mock_sandbox_paths["workspace"]
|
||||
uploads = mock_sandbox_paths["uploads"]
|
||||
|
||||
(workspace / "main.py").write_text("main")
|
||||
(uploads / "data.csv").write_text("csv")
|
||||
|
||||
# 1. 首次查询:构建缓存并存入 Redis
|
||||
index_1 = await mention_service.get_or_build_file_index("thread_1", "user_1")
|
||||
assert len(index_1) == 2
|
||||
|
||||
# 验证 Redis 中已按 workspace/thread 分别缓存
|
||||
workspace_redis_key = f"{mention_service.WORKSPACE_CACHE_PREFIX}user_1"
|
||||
thread_redis_key = f"{mention_service.THREAD_CACHE_PREFIX}thread_1"
|
||||
cached_workspace = fake_redis.data.get(workspace_redis_key)
|
||||
cached_thread = fake_redis.data.get(thread_redis_key)
|
||||
assert cached_workspace is not None
|
||||
assert cached_thread is not None
|
||||
|
||||
# 反序列化校验
|
||||
workspace_entries = ormsgpack.unpackb(base64.b64decode(cached_workspace))
|
||||
thread_entries = ormsgpack.unpackb(base64.b64decode(cached_thread))
|
||||
assert len(workspace_entries) == 1
|
||||
assert len(thread_entries) == 1
|
||||
|
||||
# 2. 修改磁盘文件,但在 TTL 内应仍然走 Redis 缓存,内容不更新
|
||||
(workspace / "new_file.py").write_text("new")
|
||||
index_2 = await mention_service.get_or_build_file_index("thread_1", "user_1")
|
||||
assert len(index_2) == 2 # 仍然命中缓存,没有扫描出 new_file.py
|
||||
|
||||
# 3. 清理 workspace 缓存后重新读取,应成功更新磁盘扫描内容
|
||||
await mention_service.invalidate_workspace_mention_cache("user_1")
|
||||
assert fake_redis.data.get(workspace_redis_key) is None
|
||||
|
||||
index_3 = await mention_service.get_or_build_file_index("thread_1", "user_1")
|
||||
assert len(index_3) == 3
|
||||
assert any(name == "new_file.py" for name, _, source in index_3 if source == "workspace")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_workspace_without_thread_id(mock_sandbox_paths, fake_redis):
|
||||
workspace = mock_sandbox_paths["workspace"]
|
||||
uploads = mock_sandbox_paths["uploads"]
|
||||
(workspace / "guide.md").write_text("workspace")
|
||||
(uploads / "guide.csv").write_text("thread")
|
||||
|
||||
results = await mention_service.search_mention_files_in_index(None, "user_1", "guide")
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["name"] == "guide.md"
|
||||
assert results[0]["path"] == "/home/gem/user-data/workspace/guide.md"
|
||||
assert results[0]["source"] == "workspace"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_thread_source_before_workspace(mock_sandbox_paths, fake_redis):
|
||||
workspace = mock_sandbox_paths["workspace"]
|
||||
uploads = mock_sandbox_paths["uploads"]
|
||||
(workspace / "report.md").write_text("workspace")
|
||||
(uploads / "report.md").write_text("thread")
|
||||
|
||||
results = await mention_service.search_mention_files_in_index("thread_1", "user_1", "report")
|
||||
|
||||
assert len(results) == 2
|
||||
assert results[0]["path"] == "/home/gem/user-data/uploads/report.md"
|
||||
assert results[0]["source"] == "thread"
|
||||
assert results[1]["path"] == "/home/gem/user-data/workspace/report.md"
|
||||
assert results[1]["source"] == "workspace"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_mention_files_in_index(mock_sandbox_paths, fake_redis):
|
||||
workspace = mock_sandbox_paths["workspace"]
|
||||
(workspace / "agent_config.json").write_text("config")
|
||||
(workspace / "main.py").write_text("main")
|
||||
|
||||
# 搜索匹配测试
|
||||
results = await mention_service.search_mention_files_in_index("thread_1", "user_1", "config")
|
||||
assert len(results) == 1
|
||||
assert results[0]["name"] == "agent_config.json"
|
||||
assert results[0]["path"] == "/home/gem/user-data/workspace/agent_config.json"
|
||||
assert results[0]["is_dir"] is False
|
||||
assert results[0]["source"] == "workspace"
|
||||
|
||||
# 大小写不敏感匹配
|
||||
results_case = await mention_service.search_mention_files_in_index("thread_1", "user_1", "MAIN")
|
||||
assert len(results_case) == 1
|
||||
assert results_case[0]["name"] == "main.py"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_mention_directories_and_weighted_ranking(mock_sandbox_paths, fake_redis):
|
||||
workspace = mock_sandbox_paths["workspace"]
|
||||
|
||||
# 1. 创建合格的子目录 "test"
|
||||
test_dir = workspace / "test"
|
||||
test_dir.mkdir(exist_ok=True)
|
||||
|
||||
# 2. 在子目录下创建一些包含关键字的文件
|
||||
(test_dir / "test_auth.py").write_text("auth")
|
||||
(test_dir / "conftest.py").write_text("conf") # 文件名不含 test,但路径含 test
|
||||
|
||||
# 3. 搜索 "@test"
|
||||
results = await mention_service.search_mention_files_in_index("thread_1", "user_1", "test")
|
||||
|
||||
# 4. 校验结果
|
||||
# 必须包含 3 个项:目录 "test/",文件 "test_auth.py",文件 "conftest.py" (路径匹配兜底)
|
||||
assert len(results) == 3
|
||||
|
||||
# 5. 校验置顶排序和 is_dir 属性
|
||||
# 由于目录名字 "test" 与搜索词 "test" 100% 完全一致,得分为最高 (1000分),必须排在第 1 位
|
||||
assert results[0]["name"] == "test"
|
||||
assert results[0]["is_dir"] is True
|
||||
assert results[0]["path"] == "/home/gem/user-data/workspace/test/"
|
||||
|
||||
# "test_auth.py" 文件名以 "test" 开头,为前缀匹配 (500分),必须排在第 2 位
|
||||
assert results[1]["name"] == "test_auth.py"
|
||||
assert results[1]["is_dir"] is False
|
||||
|
||||
# "conftest.py" 文件名不含 test,为纯路径匹配兜底 (10分),必须排在最后
|
||||
assert results[2]["name"] == "conftest.py"
|
||||
assert results[2]["is_dir"] is False
|
||||
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import yuxi.models.providers.cache as cache_module
|
||||
from yuxi.models.providers.cache import REDIS_CACHE_KEY, ModelCache, ModelInfo
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeRedis:
|
||||
def __init__(self):
|
||||
self.data: dict[str, str] = {}
|
||||
self.get_calls = 0
|
||||
|
||||
def get(self, key: str) -> str | None:
|
||||
self.get_calls += 1
|
||||
return self.data.get(key)
|
||||
|
||||
def set(self, key: str, value: str) -> bool:
|
||||
self.data[key] = value
|
||||
return True
|
||||
|
||||
|
||||
def _patch_redis(monkeypatch: pytest.MonkeyPatch, redis: _FakeRedis) -> None:
|
||||
@contextmanager
|
||||
def fake_sync_redis_client(*args, **kwargs):
|
||||
del args, kwargs
|
||||
yield redis
|
||||
|
||||
monkeypatch.setattr(cache_module, "sync_redis_client", fake_sync_redis_client)
|
||||
|
||||
|
||||
def test_model_cache_prefers_model_base_url_override(monkeypatch):
|
||||
saved_cache = {}
|
||||
|
||||
class Provider:
|
||||
is_enabled = True
|
||||
provider_id = "alibaba"
|
||||
api_key = "sk-test"
|
||||
api_key_env = None
|
||||
provider_type = "openai"
|
||||
base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
embedding_base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings"
|
||||
rerank_base_url = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
|
||||
headers_json = {}
|
||||
extra_json = {}
|
||||
enabled_models = [
|
||||
{
|
||||
"id": "qwen3-rerank",
|
||||
"type": "rerank",
|
||||
"display_name": "Qwen3 Rerank",
|
||||
"base_url_override": "https://invalid.example/rerank",
|
||||
}
|
||||
]
|
||||
|
||||
cache = ModelCache()
|
||||
monkeypatch.setattr(cache, "_save_cache", lambda data: saved_cache.update(data))
|
||||
|
||||
cache.rebuild([Provider()])
|
||||
|
||||
assert saved_cache["alibaba:qwen3-rerank"].base_url == "https://invalid.example/rerank"
|
||||
|
||||
|
||||
def test_model_cache_loads_from_redis_and_uses_local_ttl(monkeypatch: pytest.MonkeyPatch):
|
||||
redis = _FakeRedis()
|
||||
_patch_redis(monkeypatch, redis)
|
||||
redis.data[REDIS_CACHE_KEY] = json.dumps(
|
||||
{
|
||||
"provider:chat": {
|
||||
"provider_id": "provider",
|
||||
"model_id": "chat",
|
||||
"model_type": "chat",
|
||||
"display_name": "Chat",
|
||||
"api_key": "sk-test",
|
||||
"base_url": "https://example.com/v1",
|
||||
"provider_type": "openai",
|
||||
}
|
||||
}
|
||||
)
|
||||
cache = ModelCache()
|
||||
|
||||
info = cache.get_model_info("provider:chat")
|
||||
cached_info = cache.get_model_info("provider:chat")
|
||||
|
||||
assert info is not None
|
||||
assert cached_info is info
|
||||
assert info.base_url == "https://example.com/v1"
|
||||
assert redis.get_calls == 1
|
||||
|
||||
|
||||
def test_model_cache_save_writes_redis_json(monkeypatch: pytest.MonkeyPatch):
|
||||
redis = _FakeRedis()
|
||||
_patch_redis(monkeypatch, redis)
|
||||
cache = ModelCache()
|
||||
info = ModelInfo(
|
||||
provider_id="provider",
|
||||
model_id="chat",
|
||||
model_type="chat",
|
||||
display_name="Chat",
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
provider_type="openai",
|
||||
)
|
||||
|
||||
cache._save_cache({info.spec: info})
|
||||
|
||||
payload = json.loads(redis.data[REDIS_CACHE_KEY])
|
||||
assert payload[info.spec]["base_url"] == "https://example.com/v1"
|
||||
@@ -0,0 +1,331 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("OPENAI_API_KEY", "test-key")
|
||||
|
||||
from yuxi.models.providers.builtin import BUILTIN_PROVIDERS
|
||||
from yuxi.models.providers.service import (
|
||||
check_credential_status,
|
||||
_normalize_payload,
|
||||
_normalize_remote_model,
|
||||
fetch_remote_models,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_payload_accepts_enabled_chat_model():
|
||||
payload = _normalize_payload(
|
||||
{
|
||||
"provider_id": "openrouter-local",
|
||||
"display_name": "OpenRouter Local",
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
"enabled_models": [{"id": "anthropic/claude-sonnet-4.5", "type": "chat"}],
|
||||
}
|
||||
)
|
||||
|
||||
assert payload["provider_id"] == "openrouter-local"
|
||||
assert payload["provider_type"] == "openai"
|
||||
assert "models_endpoint" not in payload
|
||||
assert "embedding_models_endpoint" not in payload
|
||||
assert payload["enabled_models"][0]["display_name"] == "anthropic/claude-sonnet-4.5"
|
||||
|
||||
|
||||
def test_normalize_payload_accepts_anthropic_provider_type():
|
||||
payload = _normalize_payload(
|
||||
{
|
||||
"provider_id": "xiaomi-token-plan",
|
||||
"display_name": "Xiaomi Token Plan",
|
||||
"provider_type": "anthropic",
|
||||
"base_url": "https://token-plan-cn.xiaomimimo.com/anthropic",
|
||||
"capabilities": ["chat"],
|
||||
"enabled_models": [{"id": "mimo-v2.5-pro", "type": "chat", "source": "manual"}],
|
||||
}
|
||||
)
|
||||
|
||||
assert payload["provider_type"] == "anthropic"
|
||||
assert payload["enabled_models"][0]["id"] == "mimo-v2.5-pro"
|
||||
|
||||
|
||||
def test_normalize_payload_rejects_unknown_enabled_model_type():
|
||||
with pytest.raises(ValueError, match="type 必须是"):
|
||||
_normalize_payload(
|
||||
{
|
||||
"provider_id": "openrouter-local",
|
||||
"display_name": "OpenRouter Local",
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
"enabled_models": [{"id": "unknown-model", "type": "unknown"}],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_payload_allows_embedding_without_dimension():
|
||||
"""embedding 模型的 dimension 是可选字段,不提供也不会报错。"""
|
||||
payload = _normalize_payload(
|
||||
{
|
||||
"provider_id": "embedding-local",
|
||||
"display_name": "Embedding Local",
|
||||
"base_url": "https://example.com/v1",
|
||||
"capabilities": ["embedding"],
|
||||
"embedding_base_url": "https://example.com/v1/embeddings",
|
||||
"enabled_models": [{"id": "text-embedding", "type": "embedding"}],
|
||||
}
|
||||
)
|
||||
assert payload["provider_id"] == "embedding-local"
|
||||
assert payload["enabled_models"][0].get("dimension") is None
|
||||
|
||||
|
||||
def test_normalize_remote_model_preserves_detailed_model_config():
|
||||
model = _normalize_remote_model(
|
||||
{
|
||||
"id": "xiaomi/mimo-v2-omni",
|
||||
"name": "Xiaomi: MiMo-V2-Omni",
|
||||
"context_length": 262144,
|
||||
"architecture": {
|
||||
"input_modalities": ["text", "audio", "image", "video"],
|
||||
"output_modalities": ["text"],
|
||||
},
|
||||
"top_provider": {"max_completion_tokens": 65536},
|
||||
"supported_parameters": ["temperature", "tools"],
|
||||
}
|
||||
)
|
||||
|
||||
assert model["id"] == "xiaomi/mimo-v2-omni"
|
||||
assert model["display_name"] == "Xiaomi: MiMo-V2-Omni"
|
||||
assert model["type"] == "chat"
|
||||
assert model["input_modalities"] == ["text", "audio", "image", "video"]
|
||||
assert model["max_completion_tokens"] == 65536
|
||||
assert model["raw_metadata"]["supported_parameters"] == ["temperature", "tools"]
|
||||
|
||||
|
||||
def test_normalize_remote_model_uses_endpoint_model_type():
|
||||
model = _normalize_remote_model({"id": "BAAI/bge-m3", "object": "model"}, "embedding")
|
||||
|
||||
assert model["id"] == "BAAI/bge-m3"
|
||||
assert model["type"] == "embedding"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_remote_models_loads_embedding_only_when_capability_enabled(monkeypatch):
|
||||
calls = []
|
||||
|
||||
async def fake_fetch(client, provider, headers, endpoint, model_type):
|
||||
calls.append((endpoint, model_type))
|
||||
return [{"id": f"{model_type}-model", "type": model_type}]
|
||||
|
||||
monkeypatch.setattr("yuxi.models.providers.service._fetch_models_from_endpoint", fake_fetch)
|
||||
|
||||
class Provider:
|
||||
base_url = "https://example.com/v1"
|
||||
api_key = None
|
||||
api_key_env = None
|
||||
headers_json = {}
|
||||
capabilities = ["chat", "embedding", "rerank"]
|
||||
models_endpoint = "/models"
|
||||
embedding_models_endpoint = "/embeddings/models"
|
||||
rerank_models_endpoint = None
|
||||
|
||||
models = await fetch_remote_models(Provider())
|
||||
|
||||
assert calls == [("/models", "chat"), ("/embeddings/models", "embedding")]
|
||||
assert [model["type"] for model in models] == ["chat", "embedding"]
|
||||
|
||||
|
||||
def test_normalize_payload_rejects_ollama_provider_type():
|
||||
with pytest.raises(ValueError, match="provider_type 必须是"):
|
||||
_normalize_payload(
|
||||
{
|
||||
"provider_id": "ollama-local",
|
||||
"display_name": "Ollama Local",
|
||||
"provider_type": "ollama",
|
||||
"base_url": "http://localhost:11434",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_builtin_provider_templates_default_to_openai_provider_type():
|
||||
assert len(BUILTIN_PROVIDERS) >= 16
|
||||
provider_types = {
|
||||
_normalize_payload(
|
||||
{
|
||||
"provider_id": provider["provider_id"],
|
||||
"display_name": provider["display_name"],
|
||||
"base_url": provider["base_url"],
|
||||
"provider_type": provider.get("provider_type"),
|
||||
}
|
||||
)["provider_type"]
|
||||
for provider in BUILTIN_PROVIDERS
|
||||
}
|
||||
assert provider_types == {"openai"}
|
||||
assert all("ollama" not in provider["provider_id"] for provider in BUILTIN_PROVIDERS)
|
||||
|
||||
|
||||
def test_builtin_siliconflow_provider_includes_default_runnable_models():
|
||||
provider = next(item for item in BUILTIN_PROVIDERS if item["provider_id"] == "siliconflow-cn")
|
||||
models = {model["id"]: model for model in provider["enabled_models"]}
|
||||
|
||||
assert provider["capabilities"] == ["chat", "embedding", "rerank"]
|
||||
assert provider["embedding_base_url"] == "https://api.siliconflow.cn/v1/embeddings"
|
||||
assert provider["rerank_base_url"] == "https://api.siliconflow.cn/v1/rerank"
|
||||
assert models["Pro/BAAI/bge-m3"]["type"] == "embedding"
|
||||
assert models["Pro/BAAI/bge-m3"]["dimension"] == 1024
|
||||
assert "base_url_override" not in models["Pro/BAAI/bge-m3"]
|
||||
assert models["Pro/BAAI/bge-reranker-v2-m3"]["type"] == "rerank"
|
||||
assert "base_url_override" not in models["Pro/BAAI/bge-reranker-v2-m3"]
|
||||
|
||||
|
||||
def test_builtin_dashscope_provider_includes_default_embedding_and_rerank_models():
|
||||
provider = next(item for item in BUILTIN_PROVIDERS if item["provider_id"] == "alibaba")
|
||||
models = {model["id"]: model for model in provider["enabled_models"]}
|
||||
|
||||
assert provider["capabilities"] == ["chat", "embedding", "rerank"]
|
||||
assert provider["embedding_base_url"] == "https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings"
|
||||
assert provider["rerank_base_url"] == "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
|
||||
assert "embedding_models_endpoint" not in provider
|
||||
assert "rerank_models_endpoint" not in provider
|
||||
assert models["text-embedding-v4"]["type"] == "embedding"
|
||||
assert models["text-embedding-v4"]["dimension"] == 1024
|
||||
assert models["qwen3-rerank"]["type"] == "rerank"
|
||||
|
||||
|
||||
def testcheck_credential_status_disabled_provider_always_ok():
|
||||
"""未启用的 provider 无论凭证如何配置,状态始终为 ok。"""
|
||||
|
||||
class Provider:
|
||||
is_enabled = False
|
||||
api_key = None
|
||||
api_key_env = None
|
||||
|
||||
assert check_credential_status(Provider()) == "ok"
|
||||
|
||||
|
||||
def testcheck_credential_status_direct_api_key_ok():
|
||||
"""直接配置了 api_key 的启用 provider 状态为 ok。"""
|
||||
|
||||
class Provider:
|
||||
is_enabled = True
|
||||
api_key = "sk-test"
|
||||
api_key_env = None
|
||||
|
||||
assert check_credential_status(Provider()) == "ok"
|
||||
|
||||
|
||||
def testcheck_credential_status_env_key_exists_ok(monkeypatch):
|
||||
"""api_key_env 对应的环境变量存在时状态为 ok。"""
|
||||
monkeypatch.setenv("TEST_API_KEY", "exists")
|
||||
|
||||
class Provider:
|
||||
is_enabled = True
|
||||
api_key = None
|
||||
api_key_env = "TEST_API_KEY"
|
||||
|
||||
assert check_credential_status(Provider()) == "ok"
|
||||
|
||||
|
||||
def testcheck_credential_status_env_key_missing_warning(monkeypatch):
|
||||
"""api_key_env 对应的环境变量不存在时状态为 warning。"""
|
||||
monkeypatch.delenv("MISSING_KEY", raising=False)
|
||||
|
||||
class Provider:
|
||||
is_enabled = True
|
||||
api_key = None
|
||||
api_key_env = "MISSING_KEY"
|
||||
|
||||
assert check_credential_status(Provider()) == "warning"
|
||||
|
||||
|
||||
def testcheck_credential_status_both_empty_warning():
|
||||
"""api_key 和 api_key_env 都未配置时状态为 warning。"""
|
||||
|
||||
class Provider:
|
||||
is_enabled = True
|
||||
api_key = None
|
||||
api_key_env = None
|
||||
|
||||
assert check_credential_status(Provider()) == "warning"
|
||||
|
||||
|
||||
# ==================== 手动添加模型 / source 字段 ====================
|
||||
|
||||
|
||||
def test_normalize_payload_default_model_source_is_remote():
|
||||
"""未显式指定 source 时,规范化后默认填入 remote,向后兼容旧数据。"""
|
||||
payload = _normalize_payload(
|
||||
{
|
||||
"provider_id": "openrouter-local",
|
||||
"display_name": "OpenRouter Local",
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
"enabled_models": [{"id": "anthropic/claude-sonnet-4.5", "type": "chat"}],
|
||||
}
|
||||
)
|
||||
|
||||
assert payload["enabled_models"][0]["source"] == "remote"
|
||||
|
||||
|
||||
def test_normalize_payload_accepts_manual_source():
|
||||
"""source=manual 表示管理员手动添加的模型,规范化保留该标签。"""
|
||||
payload = _normalize_payload(
|
||||
{
|
||||
"provider_id": "custom-local",
|
||||
"display_name": "Custom Local",
|
||||
"base_url": "https://example.com/v1",
|
||||
"capabilities": ["chat"],
|
||||
"enabled_models": [{"id": "my-chat-model", "type": "chat", "source": "manual"}],
|
||||
}
|
||||
)
|
||||
|
||||
assert payload["enabled_models"][0]["source"] == "manual"
|
||||
|
||||
|
||||
def test_normalize_payload_rejects_invalid_source():
|
||||
"""source 仅允许 manual 或 remote,其他取值视为非法。"""
|
||||
with pytest.raises(ValueError, match="source 必须是"):
|
||||
_normalize_payload(
|
||||
{
|
||||
"provider_id": "custom-local",
|
||||
"display_name": "Custom Local",
|
||||
"base_url": "https://example.com/v1",
|
||||
"enabled_models": [{"id": "x", "type": "chat", "source": "custom"}],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_payload_rejects_model_type_not_in_capabilities():
|
||||
"""provider 仅声明 chat 能力时,不允许写入 embedding 类型的模型。"""
|
||||
with pytest.raises(ValueError, match="不在 provider 能力"):
|
||||
_normalize_payload(
|
||||
{
|
||||
"provider_id": "chat-only",
|
||||
"display_name": "Chat Only",
|
||||
"base_url": "https://example.com/v1",
|
||||
"capabilities": ["chat"],
|
||||
"enabled_models": [{"id": "rogue-embedding", "type": "embedding", "dimension": 1024}],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_payload_allows_model_type_within_capabilities():
|
||||
"""provider 同时声明 chat + embedding 时,两类模型均可正常写入。"""
|
||||
payload = _normalize_payload(
|
||||
{
|
||||
"provider_id": "multi-cap",
|
||||
"display_name": "Multi Cap",
|
||||
"base_url": "https://example.com/v1",
|
||||
"capabilities": ["chat", "embedding"],
|
||||
"embedding_base_url": "https://example.com/v1/embeddings",
|
||||
"embedding_models_endpoint": "/embeddings/models",
|
||||
"enabled_models": [
|
||||
{"id": "chat-1", "type": "chat", "source": "manual"},
|
||||
{
|
||||
"id": "embed-1",
|
||||
"type": "embedding",
|
||||
"source": "manual",
|
||||
"dimension": 1024,
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
types = [model["type"] for model in payload["enabled_models"]]
|
||||
sources = [model["source"] for model in payload["enabled_models"]]
|
||||
assert types == ["chat", "embedding"]
|
||||
assert sources == ["manual", "manual"]
|
||||
@@ -0,0 +1,495 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from yuxi.agents.models import load_chat_model, resolve_chat_model_spec
|
||||
from yuxi.models.chat import LangChainChatAdapter, select_model
|
||||
from yuxi.models.embed import OtherEmbedding, select_embedding_model
|
||||
from yuxi.models.rerank import OpenAIReranker, get_reranker
|
||||
from yuxi.models.providers.cache import ModelInfo
|
||||
|
||||
|
||||
def _model_info(model_type: str) -> ModelInfo:
|
||||
return ModelInfo(
|
||||
provider_id="test-provider",
|
||||
model_id=f"namespace/{model_type}-model",
|
||||
model_type=model_type,
|
||||
display_name=f"Test {model_type}",
|
||||
api_key="test-key",
|
||||
base_url="https://example.com/v1",
|
||||
provider_type="openai",
|
||||
dimension=1024 if model_type == "embedding" else None,
|
||||
)
|
||||
|
||||
|
||||
def _chat_model_info(provider_id: str, model_id: str, provider_type: str = "openai") -> ModelInfo:
|
||||
return ModelInfo(
|
||||
provider_id=provider_id,
|
||||
model_id=model_id,
|
||||
model_type="chat",
|
||||
display_name=model_id,
|
||||
api_key="test-key",
|
||||
base_url="https://example.com/v1",
|
||||
provider_type=provider_type,
|
||||
)
|
||||
|
||||
|
||||
def _capture_embed_warnings(monkeypatch: pytest.MonkeyPatch) -> list[str]:
|
||||
warnings = []
|
||||
monkeypatch.setattr(
|
||||
"yuxi.models.embed.logger",
|
||||
SimpleNamespace(
|
||||
warning=warnings.append,
|
||||
error=lambda *_args, **_kwargs: None,
|
||||
info=lambda *_args, **_kwargs: None,
|
||||
),
|
||||
)
|
||||
return warnings
|
||||
|
||||
|
||||
def _requests_embedding_response(status_code: int, content: bytes | None = None) -> requests.Response:
|
||||
response = requests.Response()
|
||||
response.status_code = status_code
|
||||
response.url = "https://example.com/v1/embeddings"
|
||||
response._content = content or b'{"error":"temporary error"}'
|
||||
return response
|
||||
|
||||
|
||||
def _httpx_embedding_response(status_code: int, content: str | None = None) -> httpx.Response:
|
||||
request = httpx.Request("POST", "https://example.com/v1/embeddings")
|
||||
return httpx.Response(status_code, request=request, text=content or '{"error":"temporary error"}')
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"selector,args",
|
||||
[
|
||||
(select_model, {"model_spec": "unknown-provider:namespace/model"}),
|
||||
(load_chat_model, {"fully_specified_name": "unknown-provider:namespace/model"}),
|
||||
(select_embedding_model, {"model_id": "unknown-provider:namespace/model"}),
|
||||
(get_reranker, {"model_id": "unknown-provider:namespace/model"}),
|
||||
],
|
||||
)
|
||||
def test_selectors_report_unknown_unconfigured_specs(selector, args):
|
||||
with pytest.raises(ValueError, match="Unknown|未找到模型"):
|
||||
selector(**args)
|
||||
|
||||
|
||||
def test_resolve_chat_model_spec_prefers_explicit_then_fallback_then_default(monkeypatch):
|
||||
monkeypatch.setattr("yuxi.agents.models.sys_config.default_model", "system-default:model")
|
||||
|
||||
assert resolve_chat_model_spec(" explicit:model ", fallback="fallback:model") == "explicit:model"
|
||||
assert resolve_chat_model_spec("", fallback=" fallback:model ") == "fallback:model"
|
||||
assert resolve_chat_model_spec(None, fallback="") == "system-default:model"
|
||||
|
||||
|
||||
def test_resolve_chat_model_spec_rejects_all_empty(monkeypatch):
|
||||
monkeypatch.setattr("yuxi.agents.models.sys_config.default_model", "")
|
||||
|
||||
with pytest.raises(ValueError, match="model spec 不能为空"):
|
||||
resolve_chat_model_spec("", fallback=None)
|
||||
|
||||
|
||||
def test_select_embedding_model_loads_model_from_cache(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"yuxi.models.embed.model_cache.get_model_info",
|
||||
lambda spec: _model_info("embedding") if spec == "test-provider:namespace/embedding-model" else None,
|
||||
)
|
||||
|
||||
model = select_embedding_model("test-provider:namespace/embedding-model")
|
||||
|
||||
assert isinstance(model, OtherEmbedding)
|
||||
assert model.model == "namespace/embedding-model"
|
||||
assert model.dimension == 1024
|
||||
|
||||
|
||||
def test_select_model_wraps_langchain_model_and_expands_model_params(monkeypatch):
|
||||
fake_model = SimpleNamespace()
|
||||
captured = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.models.chat.model_cache.get_model_info",
|
||||
lambda spec: _chat_model_info("test-provider", "namespace/chat-model")
|
||||
if spec == "test-provider:namespace/chat-model"
|
||||
else None,
|
||||
)
|
||||
|
||||
def fake_load_chat_model(spec, **kwargs):
|
||||
captured["spec"] = spec
|
||||
captured["kwargs"] = kwargs
|
||||
return fake_model
|
||||
|
||||
monkeypatch.setattr("yuxi.models.chat.load_chat_model", fake_load_chat_model)
|
||||
|
||||
model = select_model(
|
||||
"test-provider:namespace/chat-model",
|
||||
model_params={"temperature": 0.2},
|
||||
timeout=60.0,
|
||||
)
|
||||
|
||||
assert isinstance(model, LangChainChatAdapter)
|
||||
assert model.model is fake_model
|
||||
assert model.model_name == "namespace/chat-model"
|
||||
assert captured == {
|
||||
"spec": "test-provider:namespace/chat-model",
|
||||
"kwargs": {"temperature": 0.2, "timeout": 60.0},
|
||||
}
|
||||
|
||||
|
||||
def test_select_model_maps_anthropic_max_completion_tokens(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.models.chat.model_cache.get_model_info",
|
||||
lambda spec: _chat_model_info("anthropic", "mimo-v2.5", provider_type="anthropic")
|
||||
if spec == "anthropic:mimo-v2.5"
|
||||
else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.models.chat.load_chat_model",
|
||||
lambda spec, **kwargs: captured.update({"spec": spec, "kwargs": kwargs}) or SimpleNamespace(),
|
||||
)
|
||||
|
||||
select_model("anthropic:mimo-v2.5", model_params={"max_completion_tokens": 123})
|
||||
|
||||
assert captured == {"spec": "anthropic:mimo-v2.5", "kwargs": {"max_tokens": 123}}
|
||||
|
||||
|
||||
def test_load_chat_model_uses_toolcall_chunk_fix_for_openai_compatible(monkeypatch):
|
||||
from yuxi.agents.models import _ToolCallChunkFixChatOpenAI
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.models.model_cache.get_model_info",
|
||||
lambda spec: _chat_model_info("siliconflow-cn", "deepseek-ai/DeepSeek-V4-Flash")
|
||||
if spec == "siliconflow-cn:deepseek-ai/DeepSeek-V4-Flash"
|
||||
else None,
|
||||
)
|
||||
|
||||
model = load_chat_model("siliconflow-cn:deepseek-ai/DeepSeek-V4-Flash")
|
||||
|
||||
# 不再按 provider 禁用流式,改用归一化子类规避 v3 流式累积丢 tool_call 字段的缺陷
|
||||
assert isinstance(model, _ToolCallChunkFixChatOpenAI)
|
||||
assert model.disable_streaming is False
|
||||
|
||||
|
||||
def test_load_chat_model_keeps_non_siliconflow_openai_streaming(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.models.model_cache.get_model_info",
|
||||
lambda spec: _chat_model_info("openai-compatible", "namespace/chat-model")
|
||||
if spec == "openai-compatible:namespace/chat-model"
|
||||
else None,
|
||||
)
|
||||
|
||||
model = load_chat_model("openai-compatible:namespace/chat-model")
|
||||
explicit = load_chat_model("openai-compatible:namespace/chat-model", disable_streaming=True)
|
||||
|
||||
assert model.disable_streaming is False
|
||||
assert explicit.disable_streaming is True
|
||||
|
||||
|
||||
def test_openai_payload_bridges_read_file_image_tool_result_to_user_role():
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from yuxi.agents.models import _ToolCallChunkFixChatOpenAI
|
||||
|
||||
model = _ToolCallChunkFixChatOpenAI(
|
||||
model="namespace/chat-model",
|
||||
api_key="test-key",
|
||||
base_url="https://example.com/v1",
|
||||
)
|
||||
|
||||
payload = model._get_request_payload(
|
||||
[
|
||||
HumanMessage("读一下这张图"),
|
||||
AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "read_file",
|
||||
"args": {"file_path": "/home/gem/user-data/workspace/a.png"},
|
||||
"id": "call_image",
|
||||
}
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
content_blocks=[{"type": "image", "base64": "iVBORw0KGgo=", "mime_type": "image/png"}],
|
||||
name="read_file",
|
||||
tool_call_id="call_image",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
tool_message = payload["messages"][2]
|
||||
image_message = payload["messages"][3]
|
||||
|
||||
assert tool_message["role"] == "tool"
|
||||
assert isinstance(tool_message["content"], str)
|
||||
assert "image_url" not in tool_message["content"]
|
||||
assert image_message == {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Images returned by read_file are attached below. Inspect them when answering.",
|
||||
},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_openai_payload_inserts_tool_image_user_message_after_parallel_tool_block():
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from yuxi.agents.models import _ToolCallChunkFixChatOpenAI
|
||||
|
||||
model = _ToolCallChunkFixChatOpenAI(
|
||||
model="namespace/chat-model",
|
||||
api_key="test-key",
|
||||
base_url="https://example.com/v1",
|
||||
)
|
||||
|
||||
payload = model._get_request_payload(
|
||||
[
|
||||
HumanMessage("读图并列目录"),
|
||||
AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "read_file",
|
||||
"args": {"file_path": "/home/gem/user-data/workspace/a.png"},
|
||||
"id": "call_image",
|
||||
},
|
||||
{"name": "ls", "args": {"path": "/home/gem/user-data/workspace"}, "id": "call_ls"},
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
content_blocks=[{"type": "image", "base64": "abc", "mime_type": "image/png"}],
|
||||
name="read_file",
|
||||
tool_call_id="call_image",
|
||||
),
|
||||
ToolMessage(content="['a.png']", name="ls", tool_call_id="call_ls"),
|
||||
]
|
||||
)
|
||||
|
||||
assert [message["role"] for message in payload["messages"]] == ["user", "assistant", "tool", "tool", "user"]
|
||||
assert payload["messages"][2]["tool_call_id"] == "call_image"
|
||||
assert payload["messages"][3]["tool_call_id"] == "call_ls"
|
||||
assert payload["messages"][4]["content"][1] == {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,abc"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_langchain_chat_adapter_preserves_call_response_contract():
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakeLangChainModel:
|
||||
async def ainvoke(self, messages):
|
||||
captured["messages"] = messages
|
||||
return AIMessage(content=[{"type": "text", "text": "he"}, {"type": "text", "text": "llo"}])
|
||||
|
||||
adapter = LangChainChatAdapter(FakeLangChainModel(), model_name="test-model")
|
||||
|
||||
response = await adapter.call([{"role": "user", "content": "Say hello"}], stream=False)
|
||||
|
||||
assert response.content == "hello"
|
||||
assert response.is_full is False
|
||||
assert type(captured["messages"][0]).__name__ == "HumanMessage"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_connection_checks_configured_dimension(monkeypatch):
|
||||
model = OtherEmbedding(
|
||||
model="namespace/embedding-model",
|
||||
base_url="https://example.com/v1/embeddings",
|
||||
api_key="test-key",
|
||||
dimension=3,
|
||||
)
|
||||
|
||||
async def fake_aencode(_messages):
|
||||
return [[0.1, 0.2, 0.3]]
|
||||
|
||||
monkeypatch.setattr(model, "aencode", fake_aencode)
|
||||
|
||||
assert await model.test_connection() == (True, "连接正常")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_connection_reports_dimension_mismatch(monkeypatch):
|
||||
model = OtherEmbedding(
|
||||
model="namespace/embedding-model",
|
||||
base_url="https://example.com/v1/embeddings",
|
||||
api_key="test-key",
|
||||
dimension=4,
|
||||
)
|
||||
|
||||
async def fake_aencode(_messages):
|
||||
return [[0.1, 0.2, 0.3]]
|
||||
|
||||
monkeypatch.setattr(model, "aencode", fake_aencode)
|
||||
|
||||
assert await model.test_connection() == (False, "Embedding 维度不一致:配置 4,实际 3")
|
||||
|
||||
|
||||
def test_embedding_sync_400_logs_warning(monkeypatch):
|
||||
warnings = _capture_embed_warnings(monkeypatch)
|
||||
model = OtherEmbedding(
|
||||
model="namespace/embedding-model",
|
||||
base_url="https://example.com/v1/embeddings",
|
||||
api_key="test-key",
|
||||
)
|
||||
response = _requests_embedding_response(400, b'{"error":"bad embedding input"}')
|
||||
calls = []
|
||||
|
||||
def fake_post(*_args, **_kwargs):
|
||||
calls.append(1)
|
||||
return response
|
||||
|
||||
monkeypatch.setattr("yuxi.models.embed.requests.post", fake_post)
|
||||
|
||||
with pytest.raises(ValueError, match="400 Client Error"):
|
||||
model.encode(["hello", "test"])
|
||||
|
||||
assert len(calls) == 1
|
||||
assert len(warnings) == 1
|
||||
warning = warnings[0]
|
||||
assert "400 Bad Request" in warning
|
||||
assert "model=namespace/embedding-model" in warning
|
||||
assert "input_count=2" in warning
|
||||
assert "input_lengths=[5, 4]" in warning
|
||||
assert "bad embedding input" in warning
|
||||
|
||||
|
||||
def test_embedding_sync_429_retries_ten_times_before_success(monkeypatch):
|
||||
warnings = _capture_embed_warnings(monkeypatch)
|
||||
sleeps = []
|
||||
monkeypatch.setattr("yuxi.models.embed.time.sleep", sleeps.append)
|
||||
|
||||
model = OtherEmbedding(
|
||||
model="namespace/embedding-model",
|
||||
base_url="https://example.com/v1/embeddings",
|
||||
api_key="test-key",
|
||||
)
|
||||
success = _requests_embedding_response(200, b'{"data":[{"embedding":[0.1,0.2]}]}')
|
||||
responses = [_requests_embedding_response(429) for _ in range(10)] + [success]
|
||||
|
||||
monkeypatch.setattr("yuxi.models.embed.requests.post", lambda *_args, **_kwargs: responses.pop(0))
|
||||
|
||||
assert model.encode(["hello"]) == [[0.1, 0.2]]
|
||||
assert len(sleeps) == 10
|
||||
assert sleeps == [1.0, 2.0, 4.0, 8.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]
|
||||
assert len(warnings) == 10
|
||||
assert "status=429" in warnings[-1]
|
||||
assert "retry=10/10" in warnings[-1]
|
||||
|
||||
|
||||
def test_embedding_sync_5xx_uses_short_retry_budget(monkeypatch):
|
||||
warnings = _capture_embed_warnings(monkeypatch)
|
||||
sleeps = []
|
||||
calls = []
|
||||
monkeypatch.setattr("yuxi.models.embed.time.sleep", sleeps.append)
|
||||
|
||||
model = OtherEmbedding(
|
||||
model="namespace/embedding-model",
|
||||
base_url="https://example.com/v1/embeddings",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
def fake_post(*_args, **_kwargs):
|
||||
calls.append(1)
|
||||
return _requests_embedding_response(503)
|
||||
|
||||
monkeypatch.setattr("yuxi.models.embed.requests.post", fake_post)
|
||||
|
||||
with pytest.raises(ValueError, match="503 Server Error"):
|
||||
model.encode(["hello"])
|
||||
|
||||
assert len(calls) == 3
|
||||
assert sleeps == [1.0, 2.0]
|
||||
assert len(warnings) == 2
|
||||
assert "retry=2/2" in warnings[-1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_async_400_logs_warning(monkeypatch):
|
||||
warnings = _capture_embed_warnings(monkeypatch)
|
||||
model = OtherEmbedding(
|
||||
model="namespace/embedding-model",
|
||||
base_url="https://example.com/v1/embeddings",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
class FakeAsyncClient:
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
return False
|
||||
|
||||
async def post(self, url, **_kwargs):
|
||||
request = httpx.Request("POST", url)
|
||||
return httpx.Response(400, request=request, text='{"error":"bad embedding input"}')
|
||||
|
||||
monkeypatch.setattr("yuxi.models.embed.httpx.AsyncClient", FakeAsyncClient)
|
||||
|
||||
with pytest.raises(httpx.HTTPStatusError, match="400 Bad Request"):
|
||||
await model.aencode(["hello", "test"])
|
||||
|
||||
assert len(warnings) == 1
|
||||
warning = warnings[0]
|
||||
assert "400 Bad Request" in warning
|
||||
assert "model=namespace/embedding-model" in warning
|
||||
assert "input_count=2" in warning
|
||||
assert "input_lengths=[5, 4]" in warning
|
||||
assert "bad embedding input" in warning
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_async_429_retries_ten_times_before_success(monkeypatch):
|
||||
warnings = _capture_embed_warnings(monkeypatch)
|
||||
sleeps = []
|
||||
|
||||
async def fake_sleep(delay):
|
||||
sleeps.append(delay)
|
||||
|
||||
monkeypatch.setattr("yuxi.models.embed.asyncio.sleep", fake_sleep)
|
||||
|
||||
model = OtherEmbedding(
|
||||
model="namespace/embedding-model",
|
||||
base_url="https://example.com/v1/embeddings",
|
||||
api_key="test-key",
|
||||
)
|
||||
success = _httpx_embedding_response(200, '{"data":[{"embedding":[0.1,0.2]}]}')
|
||||
responses = [_httpx_embedding_response(429) for _ in range(10)] + [success]
|
||||
|
||||
class FakeAsyncClient:
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
return False
|
||||
|
||||
async def post(self, *_args, **_kwargs):
|
||||
return responses.pop(0)
|
||||
|
||||
monkeypatch.setattr("yuxi.models.embed.httpx.AsyncClient", FakeAsyncClient)
|
||||
|
||||
assert await model.aencode(["hello"]) == [[0.1, 0.2]]
|
||||
assert sleeps == [1.0, 2.0, 4.0, 8.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]
|
||||
assert len(warnings) == 10
|
||||
assert "status=429" in warnings[-1]
|
||||
assert "retry=10/10" in warnings[-1]
|
||||
|
||||
|
||||
def test_get_reranker_loads_model_from_cache(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"yuxi.models.rerank.model_cache.get_model_info",
|
||||
lambda spec: _model_info("rerank") if spec == "test-provider:namespace/rerank-model" else None,
|
||||
)
|
||||
|
||||
reranker = get_reranker("test-provider:namespace/rerank-model")
|
||||
|
||||
assert isinstance(reranker, OpenAIReranker)
|
||||
assert reranker.model == "namespace/rerank-model"
|
||||
@@ -0,0 +1,103 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from urllib.parse import unquote
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
os.environ.setdefault("OPENAI_API_KEY", "dummy")
|
||||
|
||||
from yuxi.services import oidc_service
|
||||
from yuxi.storage.postgres.models_business import User
|
||||
|
||||
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.unit]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def oidc_session():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(User.__table__.create)
|
||||
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with session_factory() as session:
|
||||
yield session
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _create_user(session, uid: str = "alice") -> User:
|
||||
user = User(username="alice", uid=uid, password_hash="x", role="user", is_deleted=0)
|
||||
session.add(user)
|
||||
await session.commit()
|
||||
await session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
async def test_find_user_by_oidc_sub_resolves_placeholder_when_sub_contains_colon(oidc_session):
|
||||
user = await _create_user(oidc_session)
|
||||
|
||||
await oidc_service._create_oidc_binding_placeholder(oidc_session, "tenant:user", user)
|
||||
|
||||
resolved = await oidc_service.find_user_by_oidc_sub(oidc_session, "tenant:user")
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.id == user.id
|
||||
assert resolved.uid == user.uid
|
||||
assert resolved.is_deleted == 0
|
||||
|
||||
|
||||
async def test_find_deleted_oidc_user_by_sub_resolves_deleted_target_when_sub_contains_colon(oidc_session):
|
||||
user = await _create_user(oidc_session)
|
||||
user.is_deleted = 1
|
||||
await oidc_session.commit()
|
||||
|
||||
await oidc_service._create_oidc_binding_placeholder(oidc_session, "tenant:user", user)
|
||||
|
||||
resolved = await oidc_service.find_deleted_oidc_user_by_sub(oidc_session, "tenant:user")
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.id == user.id
|
||||
assert resolved.uid == user.uid
|
||||
assert resolved.is_deleted == 1
|
||||
|
||||
|
||||
async def test_oidc_callback_allows_existing_binding_when_sub_contains_colon(oidc_session, monkeypatch):
|
||||
user = await _create_user(oidc_session)
|
||||
await oidc_service._create_oidc_binding_placeholder(oidc_session, "tenant:user", user)
|
||||
|
||||
monkeypatch.setattr(oidc_service.oidc_config, "enabled", True)
|
||||
monkeypatch.setattr(oidc_service.oidc_config, "client_id", "cid")
|
||||
monkeypatch.setattr(oidc_service.oidc_config, "client_secret", "secret")
|
||||
monkeypatch.setattr(oidc_service.oidc_config, "token_endpoint", "https://example/token")
|
||||
monkeypatch.setattr(oidc_service.oidc_config, "authorization_endpoint", "https://example/auth")
|
||||
monkeypatch.setattr(oidc_service.oidc_config, "userinfo_endpoint", "https://example/userinfo")
|
||||
monkeypatch.setattr(oidc_service.oidc_config, "use_raw_username", True)
|
||||
monkeypatch.setattr(oidc_service.oidc_config, "auto_create_user", False)
|
||||
|
||||
monkeypatch.setattr(
|
||||
oidc_service.OIDCUtils,
|
||||
"verify_state",
|
||||
classmethod(lambda cls, state: {"redirect_path": "/"}),
|
||||
)
|
||||
|
||||
async def fake_exchange(cls, code):
|
||||
return {"access_token": "token"}
|
||||
|
||||
async def fake_userinfo(cls, access_token):
|
||||
return {"sub": "tenant:user", "preferred_username": "alice"}
|
||||
|
||||
async def fake_log_operation(db, user_id, operation, request=None):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(oidc_service.OIDCUtils, "exchange_code_for_token", classmethod(fake_exchange))
|
||||
monkeypatch.setattr(oidc_service.OIDCUtils, "get_userinfo", classmethod(fake_userinfo))
|
||||
monkeypatch.setattr(oidc_service, "log_operation", fake_log_operation)
|
||||
|
||||
response = await oidc_service.oidc_callback_handler("dummy-code", "dummy-state", oidc_session)
|
||||
|
||||
assert response.status_code == 302
|
||||
assert unquote(response.headers["location"]).startswith("/auth/oidc/callback?code=")
|
||||
@@ -0,0 +1,92 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import yuxi.services.run_queue_service as run_queue_service
|
||||
|
||||
|
||||
class _FakeStreamRedis:
|
||||
def __init__(self):
|
||||
self.streams: dict[str, list[tuple[str, dict[str, str]]]] = {}
|
||||
self.expire_calls: list[tuple[str, int]] = []
|
||||
|
||||
async def xadd(self, key: str, fields: dict[str, str], **kwargs):
|
||||
del kwargs
|
||||
stream = self.streams.setdefault(key, [])
|
||||
event_id = f"{1700000000000 + len(stream)}-0"
|
||||
stream.append((event_id, dict(fields)))
|
||||
return event_id
|
||||
|
||||
async def expire(self, key: str, ttl: int):
|
||||
self.expire_calls.append((key, ttl))
|
||||
|
||||
async def xrange(self, key: str, min: str, max: str, count: int):
|
||||
del max
|
||||
rows = list(self.streams.get(key, []))
|
||||
if min.startswith("("):
|
||||
cursor = min[1:]
|
||||
rows = [(event_id, fields) for event_id, fields in rows if event_id > cursor]
|
||||
elif min == "-":
|
||||
rows = list(rows)
|
||||
return rows[:count]
|
||||
|
||||
async def xrevrange(self, key: str, max: str, min: str, count: int):
|
||||
del max, min
|
||||
rows = list(reversed(self.streams.get(key, [])))
|
||||
return rows[:count]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_redis_client_uses_storage_client(monkeypatch: pytest.MonkeyPatch):
|
||||
fake_client = object()
|
||||
|
||||
async def fake_get_async_redis_client():
|
||||
return fake_client
|
||||
|
||||
monkeypatch.setattr(run_queue_service, "get_async_redis_client", fake_get_async_redis_client)
|
||||
|
||||
assert await run_queue_service.get_redis_client() is fake_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_stream_event_roundtrip(monkeypatch: pytest.MonkeyPatch):
|
||||
fake_redis = _FakeStreamRedis()
|
||||
|
||||
async def fake_get_async_redis_client():
|
||||
return fake_redis
|
||||
|
||||
monkeypatch.setattr(run_queue_service, "get_async_redis_client", fake_get_async_redis_client)
|
||||
|
||||
run_id = "run-1"
|
||||
seq1 = await run_queue_service.append_run_stream_event(run_id, "loading", {"items": [1]})
|
||||
seq2 = await run_queue_service.append_run_stream_event(
|
||||
run_id,
|
||||
"finished",
|
||||
{"chunk": {"status": "finished", "thread_id": "child-thread"}},
|
||||
)
|
||||
|
||||
assert seq1 < seq2
|
||||
|
||||
events = await run_queue_service.list_run_stream_events(run_id, after_seq="0-0", limit=100)
|
||||
assert [item["event_type"] for item in events] == ["loading", "finished"]
|
||||
assert events[0]["payload"]["schema_version"] == 1
|
||||
assert events[0]["payload"]["run_id"] == run_id
|
||||
assert events[0]["payload"]["payload"] == {"items": [1]}
|
||||
assert events[1]["payload"]["thread_id"] == "child-thread"
|
||||
|
||||
next_events = await run_queue_service.list_run_stream_events(run_id, after_seq=seq1, limit=100)
|
||||
assert len(next_events) == 1
|
||||
assert next_events[0]["seq"] == seq2
|
||||
|
||||
last_seq = await run_queue_service.get_last_run_stream_seq(run_id)
|
||||
assert last_seq == seq2
|
||||
|
||||
recent_events = await run_queue_service.list_recent_run_stream_events(run_id, limit=2)
|
||||
assert [item["seq"] for item in recent_events] == [seq2, seq1]
|
||||
assert [item["event_type"] for item in recent_events] == ["finished", "loading"]
|
||||
|
||||
|
||||
def test_normalize_after_seq_stream_id_only():
|
||||
assert run_queue_service.normalize_after_seq(None) == "0-0"
|
||||
assert run_queue_service.normalize_after_seq("1700000000000-3") == "1700000000000-3"
|
||||
assert run_queue_service.normalize_after_seq("12") == "0-0"
|
||||
assert run_queue_service.normalize_after_seq("bad-value") == "0-0"
|
||||
@@ -0,0 +1,466 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import yuxi.services.run_worker as run_worker
|
||||
|
||||
|
||||
class _RaisingAsyncIter:
|
||||
def __init__(self, exc: Exception):
|
||||
self._exc = exc
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
raise self._exc
|
||||
|
||||
|
||||
class _BytesAsyncIter:
|
||||
def __init__(self, values: list[bytes]):
|
||||
self._values = list(values)
|
||||
self._idx = 0
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if self._idx >= len(self._values):
|
||||
raise StopAsyncIteration
|
||||
value = self._values[self._idx]
|
||||
self._idx += 1
|
||||
return value
|
||||
|
||||
|
||||
def _build_run() -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
id="run-1",
|
||||
status="pending",
|
||||
request_id="req-1",
|
||||
input_payload={"model_spec": "provider:model"},
|
||||
input_message_id=10,
|
||||
run_type="chat",
|
||||
agent_slug="ChatbotAgent",
|
||||
uid="user-1",
|
||||
conversation_thread_id="thread-1",
|
||||
created_by_run_id=None,
|
||||
)
|
||||
|
||||
|
||||
def _patch_common(monkeypatch: pytest.MonkeyPatch, run_obj: SimpleNamespace):
|
||||
@asynccontextmanager
|
||||
async def fake_session_ctx():
|
||||
yield object()
|
||||
|
||||
async def fake_noop(*args, **kwargs):
|
||||
del args, kwargs
|
||||
return None
|
||||
|
||||
async def fake_get_run(run_id: str):
|
||||
del run_id
|
||||
return run_obj
|
||||
|
||||
async def fake_load_user(uid: str):
|
||||
del uid
|
||||
return SimpleNamespace(id=1, uid="user-1")
|
||||
|
||||
async def fake_load_input_message(message_id: int | None):
|
||||
assert message_id == 10
|
||||
return SimpleNamespace(content="hello", image_content=None, extra_metadata={})
|
||||
|
||||
async def fake_not_cancelled(self):
|
||||
del self
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(run_worker.pg_manager, "get_async_session_context", fake_session_ctx)
|
||||
monkeypatch.setattr(run_worker, "_get_run", fake_get_run)
|
||||
monkeypatch.setattr(run_worker, "_load_user", fake_load_user)
|
||||
monkeypatch.setattr(run_worker, "_load_input_message", fake_load_input_message)
|
||||
monkeypatch.setattr(run_worker, "mark_run_running", fake_noop)
|
||||
monkeypatch.setattr(run_worker, "clear_cancel_signal", fake_noop)
|
||||
monkeypatch.setattr(run_worker, "stream_agent_chat", lambda **kwargs: object())
|
||||
monkeypatch.setattr(run_worker.RunContext, "start", fake_noop)
|
||||
monkeypatch.setattr(run_worker.RunContext, "close", fake_noop)
|
||||
monkeypatch.setattr(run_worker.RunContext, "is_cancelled", fake_not_cancelled)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_agent_run_restores_invocation_meta(monkeypatch: pytest.MonkeyPatch):
|
||||
run_obj = _build_run()
|
||||
_patch_common(monkeypatch, run_obj)
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
events: list[dict] = []
|
||||
terminal_statuses: list[str] = []
|
||||
|
||||
async def fake_load_input_message(message_id: int | None):
|
||||
assert message_id == 10
|
||||
return SimpleNamespace(
|
||||
content="hello",
|
||||
image_content=None,
|
||||
extra_metadata={
|
||||
"source": "agent_call",
|
||||
"agent_invocation_meta": {"trace_id": "trace-1"},
|
||||
"evaluation": {"dataset_name": "legacy-top-level"},
|
||||
"custom_variables": {"system_prompt": "legacy"},
|
||||
},
|
||||
)
|
||||
|
||||
async def fake_append_event(run_id: str, event_type: str, payload: dict, **kwargs):
|
||||
del kwargs
|
||||
events.append({"run_id": run_id, "event_type": event_type, "payload": payload})
|
||||
|
||||
async def fake_mark_terminal(run_id: str, status: str, error_type=None, error_message=None):
|
||||
del run_id, error_type, error_message
|
||||
terminal_statuses.append(status)
|
||||
|
||||
def fake_stream_agent_chat(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return _BytesAsyncIter([b'{"status":"finished","request_id":"req-1","thread_id":"thread-1"}\n'])
|
||||
|
||||
monkeypatch.setattr(run_worker, "_load_input_message", fake_load_input_message)
|
||||
monkeypatch.setattr(run_worker, "append_run_event", fake_append_event)
|
||||
monkeypatch.setattr(run_worker, "mark_run_terminal", fake_mark_terminal)
|
||||
monkeypatch.setattr(run_worker, "stream_agent_chat", fake_stream_agent_chat)
|
||||
|
||||
await run_worker.process_agent_run({"job_try": 1}, "run-1")
|
||||
|
||||
meta = captured["meta"]
|
||||
assert meta["source"] == "agent_call"
|
||||
assert meta["agent_invocation_meta"] == {"trace_id": "trace-1"}
|
||||
assert "evaluation" not in meta
|
||||
assert "custom_variables" not in meta
|
||||
metadata_event = next(event for event in events if event["event_type"] == "metadata")
|
||||
assert metadata_event["payload"]["agent_invocation_meta"] == {"trace_id": "trace-1"}
|
||||
assert "evaluation" not in metadata_event["payload"]
|
||||
assert "custom_variables" not in metadata_event["payload"]
|
||||
assert terminal_statuses == ["completed"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_agent_run_non_retryable_error_marks_failed(monkeypatch: pytest.MonkeyPatch):
|
||||
run_obj = _build_run()
|
||||
_patch_common(monkeypatch, run_obj)
|
||||
|
||||
terminal_statuses: list[str] = []
|
||||
events: list[str] = []
|
||||
|
||||
async def fake_append_event(run_id: str, event_type: str, payload: dict, **kwargs):
|
||||
del run_id, payload, kwargs
|
||||
events.append(event_type)
|
||||
|
||||
async def fake_mark_terminal(run_id: str, status: str, error_type=None, error_message=None):
|
||||
del run_id, error_type, error_message
|
||||
terminal_statuses.append(status)
|
||||
|
||||
monkeypatch.setattr(run_worker, "append_run_event", fake_append_event)
|
||||
monkeypatch.setattr(run_worker, "mark_run_terminal", fake_mark_terminal)
|
||||
monkeypatch.setattr(
|
||||
run_worker,
|
||||
"_consume_stream_with_cancel",
|
||||
lambda stream, run_ctx: _RaisingAsyncIter(RuntimeError("boom")),
|
||||
)
|
||||
|
||||
await run_worker.process_agent_run({"job_try": 1}, "run-1")
|
||||
|
||||
assert "error" in events
|
||||
assert terminal_statuses == ["failed"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_agent_run_retryable_error_retries_then_completes(monkeypatch: pytest.MonkeyPatch):
|
||||
run_obj = _build_run()
|
||||
_patch_common(monkeypatch, run_obj)
|
||||
|
||||
terminal_statuses: list[str] = []
|
||||
events: list[dict] = []
|
||||
attempts = {"count": 0}
|
||||
|
||||
async def fake_append_event(run_id: str, event_type: str, payload: dict, **kwargs):
|
||||
del run_id, kwargs
|
||||
events.append({"event_type": event_type, "payload": payload})
|
||||
|
||||
async def fake_mark_terminal(run_id: str, status: str, error_type=None, error_message=None):
|
||||
del run_id, error_type, error_message
|
||||
terminal_statuses.append(status)
|
||||
|
||||
def fake_consume(stream, run_ctx):
|
||||
del stream, run_ctx
|
||||
attempts["count"] += 1
|
||||
if attempts["count"] == 1:
|
||||
return _RaisingAsyncIter(run_worker.RetryableRunError("temporary failure"))
|
||||
return _BytesAsyncIter([b'{"status":"finished","request_id":"req-1"}\n'])
|
||||
|
||||
monkeypatch.setattr(run_worker, "append_run_event", fake_append_event)
|
||||
monkeypatch.setattr(run_worker, "mark_run_terminal", fake_mark_terminal)
|
||||
monkeypatch.setattr(run_worker, "_consume_stream_with_cancel", fake_consume)
|
||||
|
||||
with pytest.raises(run_worker.RetryableRunError):
|
||||
await run_worker.process_agent_run({"job_try": 1}, "run-1")
|
||||
|
||||
assert terminal_statuses == []
|
||||
assert any(
|
||||
item["event_type"] == "error" and item["payload"]["chunk"].get("error_type") == "retryable_worker_error"
|
||||
for item in events
|
||||
)
|
||||
|
||||
await run_worker.process_agent_run({"job_try": 2}, "run-1")
|
||||
assert terminal_statuses == ["completed"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_subagent_run_restores_runtime_context(monkeypatch: pytest.MonkeyPatch):
|
||||
run_obj = _build_run()
|
||||
run_obj.run_type = "subagent"
|
||||
run_obj.agent_slug = "worker"
|
||||
run_obj.conversation_thread_id = "child-thread"
|
||||
run_obj.created_by_run_id = "parent-run"
|
||||
run_obj.input_payload = {
|
||||
"model_spec": "provider:model",
|
||||
"runtime": {
|
||||
"parent_thread_id": "parent-thread",
|
||||
"file_thread_id": "shared-file-thread",
|
||||
"skills_thread_id": "child-thread",
|
||||
},
|
||||
}
|
||||
_patch_common(monkeypatch, run_obj)
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
terminal_statuses: list[str] = []
|
||||
|
||||
async def fake_append_event(run_id: str, event_type: str, payload: dict, **kwargs):
|
||||
del run_id, event_type, payload, kwargs
|
||||
|
||||
async def fake_mark_terminal(run_id: str, status: str, error_type=None, error_message=None):
|
||||
del run_id, error_type, error_message
|
||||
terminal_statuses.append(status)
|
||||
|
||||
def fake_stream_agent_chat(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return _BytesAsyncIter([b'{"status":"finished","request_id":"req-1","thread_id":"child-thread"}\n'])
|
||||
|
||||
monkeypatch.setattr(run_worker, "append_run_event", fake_append_event)
|
||||
monkeypatch.setattr(run_worker, "mark_run_terminal", fake_mark_terminal)
|
||||
monkeypatch.setattr(run_worker, "stream_agent_chat", fake_stream_agent_chat)
|
||||
|
||||
await run_worker.process_agent_run({"job_try": 1}, "run-1")
|
||||
|
||||
meta = captured["meta"]
|
||||
assert meta["run_type"] == "subagent"
|
||||
assert meta["parent_thread_id"] == "parent-thread"
|
||||
assert meta["file_thread_id"] == "shared-file-thread"
|
||||
assert meta["skills_thread_id"] == "child-thread"
|
||||
assert captured["agent_slug"] == "worker"
|
||||
assert captured["thread_id"] == "child-thread"
|
||||
assert captured["input_message"].content == "hello"
|
||||
assert captured["input_message"].langchain_message.content == "hello"
|
||||
assert "image_content" not in captured
|
||||
assert terminal_statuses == ["completed"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_agent_run_rejects_unknown_run_type(monkeypatch: pytest.MonkeyPatch):
|
||||
run_obj = _build_run()
|
||||
run_obj.run_type = "unknown"
|
||||
_patch_common(monkeypatch, run_obj)
|
||||
|
||||
terminal_errors: list[dict] = []
|
||||
|
||||
async def fake_mark_terminal(run_id: str, status: str, error_type=None, error_message=None):
|
||||
terminal_errors.append(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"status": status,
|
||||
"error_type": error_type,
|
||||
"error_message": error_message,
|
||||
}
|
||||
)
|
||||
|
||||
def fail_stream_agent_chat(**kwargs):
|
||||
del kwargs
|
||||
raise AssertionError("unknown run_type must not enter chat stream")
|
||||
|
||||
monkeypatch.setattr(run_worker, "mark_run_terminal", fake_mark_terminal)
|
||||
monkeypatch.setattr(run_worker, "stream_agent_chat", fail_stream_agent_chat)
|
||||
|
||||
await run_worker.process_agent_run({"job_try": 1}, "run-1")
|
||||
|
||||
assert terminal_errors == [
|
||||
{
|
||||
"run_id": "run-1",
|
||||
"status": "failed",
|
||||
"error_type": "invalid_run_type",
|
||||
"error_message": "不支持的 run_type: unknown",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_agent_run_rejects_invalid_raw_input_message(monkeypatch: pytest.MonkeyPatch):
|
||||
run_obj = _build_run()
|
||||
_patch_common(monkeypatch, run_obj)
|
||||
|
||||
terminal_errors: list[dict] = []
|
||||
|
||||
async def fake_load_input_message(message_id: int | None):
|
||||
assert message_id == 10
|
||||
return SimpleNamespace(
|
||||
content="hello",
|
||||
image_content=None,
|
||||
extra_metadata={"raw_message": {"type": "human", "content": object()}},
|
||||
)
|
||||
|
||||
async def fake_mark_terminal(run_id: str, status: str, error_type=None, error_message=None):
|
||||
terminal_errors.append(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"status": status,
|
||||
"error_type": error_type,
|
||||
"error_message": error_message,
|
||||
}
|
||||
)
|
||||
|
||||
def fail_stream_agent_chat(**kwargs):
|
||||
del kwargs
|
||||
raise AssertionError("invalid input message must not enter chat stream")
|
||||
|
||||
monkeypatch.setattr(run_worker, "_load_input_message", fake_load_input_message)
|
||||
monkeypatch.setattr(run_worker, "mark_run_terminal", fake_mark_terminal)
|
||||
monkeypatch.setattr(run_worker, "stream_agent_chat", fail_stream_agent_chat)
|
||||
|
||||
await run_worker.process_agent_run({"job_try": 1}, "run-1")
|
||||
|
||||
assert terminal_errors == [
|
||||
{
|
||||
"run_id": "run-1",
|
||||
"status": "failed",
|
||||
"error_type": "invalid_input_message",
|
||||
"error_message": "invalid raw_message for chat input message",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunked_event_writer_flushes_loading_chunks_by_thread(monkeypatch: pytest.MonkeyPatch):
|
||||
events: list[dict] = []
|
||||
|
||||
async def fake_append_run_event(run_id: str, event_type: str, payload: dict, *, thread_id: str | None = None):
|
||||
events.append({"run_id": run_id, "event_type": event_type, "payload": payload, "thread_id": thread_id})
|
||||
|
||||
monkeypatch.setattr(run_worker, "append_run_event", fake_append_run_event)
|
||||
|
||||
writer = run_worker.ChunkedEventWriter("run-1", "parent-thread")
|
||||
await writer.append({"status": "loading", "response": "parent", "thread_id": "parent-thread"})
|
||||
await writer.append({"status": "loading", "response": "child", "thread_id": "child-thread"})
|
||||
await writer.flush()
|
||||
|
||||
assert events == [
|
||||
{
|
||||
"run_id": "run-1",
|
||||
"event_type": "messages",
|
||||
"payload": {"items": [{"status": "loading", "response": "parent", "thread_id": "parent-thread"}]},
|
||||
"thread_id": "parent-thread",
|
||||
},
|
||||
{
|
||||
"run_id": "run-1",
|
||||
"event_type": "messages",
|
||||
"payload": {"items": [{"status": "loading", "response": "child", "thread_id": "child-thread"}]},
|
||||
"thread_id": "child-thread",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunked_event_writer_flushes_semantic_tool_call_immediately(monkeypatch: pytest.MonkeyPatch):
|
||||
events: list[dict] = []
|
||||
|
||||
async def fake_append_run_event(run_id: str, event_type: str, payload: dict, *, thread_id: str | None = None):
|
||||
events.append({"run_id": run_id, "event_type": event_type, "payload": payload, "thread_id": thread_id})
|
||||
|
||||
monkeypatch.setattr(run_worker, "append_run_event", fake_append_run_event)
|
||||
|
||||
writer = run_worker.ChunkedEventWriter("run-1", "parent-thread")
|
||||
chunk = {
|
||||
"status": "loading",
|
||||
"response": "",
|
||||
"thread_id": "parent-thread",
|
||||
"stream_event": {
|
||||
"type": "tool_call",
|
||||
"message_id": "msg-1",
|
||||
"tool_call_id": "call-1",
|
||||
"name": "task",
|
||||
"args": {"description": "do work"},
|
||||
"index": 0,
|
||||
"thread_id": "parent-thread",
|
||||
"namespace": [],
|
||||
},
|
||||
}
|
||||
await writer.append(chunk)
|
||||
|
||||
assert events == [
|
||||
{
|
||||
"run_id": "run-1",
|
||||
"event_type": "messages",
|
||||
"payload": {"items": [chunk]},
|
||||
"thread_id": "parent-thread",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_chunk_thread_id_uses_fallback_for_unstable_nested_metadata():
|
||||
assert (
|
||||
run_worker._chunk_thread_id(
|
||||
{"metadata": {"configurable": {"thread_id": "child-thread"}}},
|
||||
"parent-thread",
|
||||
)
|
||||
== "parent-thread"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_startup_ensures_builtin_mcp_servers(monkeypatch: pytest.MonkeyPatch):
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_initialize():
|
||||
calls.append("initialize")
|
||||
|
||||
async def fake_create_business_tables():
|
||||
calls.append("create_business_tables")
|
||||
|
||||
async def fake_ensure_business_schema():
|
||||
calls.append("ensure_business_schema")
|
||||
|
||||
async def fake_ensure_builtin_mcp_servers_in_db():
|
||||
calls.append("ensure_builtin_mcp_servers_in_db")
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_session_ctx():
|
||||
yield object()
|
||||
|
||||
async def fake_init_builtin_skills(session):
|
||||
del session
|
||||
calls.append("init_builtin_skills")
|
||||
|
||||
def fake_start_runtime_sync():
|
||||
calls.append("start_runtime_sync")
|
||||
|
||||
monkeypatch.setattr(run_worker.pg_manager, "initialize", fake_initialize)
|
||||
monkeypatch.setattr(run_worker.pg_manager, "create_business_tables", fake_create_business_tables)
|
||||
monkeypatch.setattr(run_worker.pg_manager, "ensure_business_schema", fake_ensure_business_schema)
|
||||
monkeypatch.setattr(run_worker.pg_manager, "get_async_session_context", fake_session_ctx)
|
||||
monkeypatch.setattr(run_worker, "ensure_builtin_mcp_servers_in_db", fake_ensure_builtin_mcp_servers_in_db)
|
||||
monkeypatch.setattr(run_worker, "init_builtin_skills", fake_init_builtin_skills)
|
||||
monkeypatch.setattr(run_worker.sys_config, "start_runtime_sync", fake_start_runtime_sync)
|
||||
|
||||
await run_worker._worker_startup({})
|
||||
|
||||
assert calls == [
|
||||
"initialize",
|
||||
"create_business_tables",
|
||||
"ensure_business_schema",
|
||||
"ensure_builtin_mcp_servers_in_db",
|
||||
"init_builtin_skills",
|
||||
"start_runtime_sync",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,726 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
import yuxi.services.agent_run_service as agent_run_service
|
||||
import yuxi.services.subagent_run_service as service_module
|
||||
from yuxi.services.input_message_service import build_chat_input_message
|
||||
from yuxi.services.subagent_run_service import SubagentRunBusy, SubagentRunService
|
||||
from yuxi.utils.hash_utils import subagent_child_thread_id
|
||||
|
||||
|
||||
def make_child_thread_id(parent_thread_id: str, agent_slug: str, tool_call_id: str) -> str:
|
||||
return subagent_child_thread_id(parent_thread_id, agent_slug, tool_call_id)
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
def __init__(self):
|
||||
self.flushes = 0
|
||||
self.added = []
|
||||
self.deleted = []
|
||||
self.committed = False
|
||||
self.created_run = None
|
||||
self.created_run_kwargs = None
|
||||
self.request_id_lookups: list[str] = []
|
||||
self.active_run_lookup = None
|
||||
self.active_run = None
|
||||
self.existing_run = None
|
||||
self._message_id = 10
|
||||
|
||||
async def flush(self):
|
||||
self.flushes += 1
|
||||
for item in self.added:
|
||||
if getattr(item, "id", None) is None:
|
||||
item.id = self._message_id
|
||||
|
||||
async def execute(self, stmt):
|
||||
del stmt
|
||||
return SimpleNamespace(scalar_one_or_none=lambda: SimpleNamespace(uid="user-1", role="user"))
|
||||
|
||||
def add(self, item):
|
||||
self.added.append(item)
|
||||
|
||||
async def commit(self):
|
||||
self.committed = True
|
||||
|
||||
async def rollback(self):
|
||||
pass
|
||||
|
||||
async def delete(self, item):
|
||||
self.deleted.append(item)
|
||||
|
||||
def begin_nested(self):
|
||||
class NestedTransaction:
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
return NestedTransaction()
|
||||
|
||||
|
||||
def _agent(slug: str = "worker"):
|
||||
return SimpleNamespace(slug=slug, name="Worker")
|
||||
|
||||
|
||||
def _parent_run():
|
||||
return SimpleNamespace(id="parent-run", conversation_thread_id="parent-thread", conversation_id=10, run_type="chat")
|
||||
|
||||
|
||||
def _relation(
|
||||
*,
|
||||
child_thread_id: str = "child-thread",
|
||||
parent_conversation_id: int = 10,
|
||||
subagent_slug: str = "worker",
|
||||
):
|
||||
return SimpleNamespace(
|
||||
id=77,
|
||||
parent_conversation_id=parent_conversation_id,
|
||||
child_conversation_id=20,
|
||||
child_thread_id=child_thread_id,
|
||||
subagent_slug=subagent_slug,
|
||||
)
|
||||
|
||||
|
||||
def _child_run(*, relation_id: int = 77, created_by_run_id: str = "parent-run"):
|
||||
return SimpleNamespace(
|
||||
id="child-run",
|
||||
run_type="subagent",
|
||||
conversation_thread_id="child-thread",
|
||||
conversation_id=20,
|
||||
created_by_run_id=created_by_run_id,
|
||||
subagent_thread_relation_id=relation_id,
|
||||
)
|
||||
|
||||
|
||||
def _patch_repos(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
captured: dict[str, object] | None = None,
|
||||
parent_run=None,
|
||||
child_run=None,
|
||||
child_conversation=None,
|
||||
existing_child_conversation=None,
|
||||
existing_relation=None,
|
||||
created_relation=None,
|
||||
relation_by_id=None,
|
||||
):
|
||||
captured = captured if captured is not None else {}
|
||||
parent_run = parent_run or _parent_run()
|
||||
child_conversation = child_conversation or SimpleNamespace(id=20, uid="user-1", agent_id="worker", status="active")
|
||||
|
||||
class RunRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_run_for_user(self, run_id: str, uid: str):
|
||||
assert uid == "user-1"
|
||||
return {"parent-run": parent_run, "child-run": child_run}.get(run_id)
|
||||
|
||||
async def get_subagent_run_for_creator(self, *, uid: str, created_by_run_id: str, run_id: str):
|
||||
assert uid == "user-1"
|
||||
captured["get_subagent_run_for_creator"] = {
|
||||
"uid": uid,
|
||||
"created_by_run_id": created_by_run_id,
|
||||
"run_id": run_id,
|
||||
}
|
||||
creator_run = await self.get_run_for_user(created_by_run_id, uid)
|
||||
run = await self.get_run_for_user(run_id, uid)
|
||||
if not creator_run or not run or run.run_type != "subagent":
|
||||
return None
|
||||
if run.created_by_run_id != creator_run.id:
|
||||
return None
|
||||
relation_id = run.subagent_thread_relation_id
|
||||
if not relation_id or not relation_by_id or relation_by_id.id != relation_id:
|
||||
return None
|
||||
if relation_by_id.parent_conversation_id != creator_run.conversation_id:
|
||||
return None
|
||||
if relation_by_id.child_thread_id != run.conversation_thread_id:
|
||||
return None
|
||||
return run
|
||||
|
||||
class ConvRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
captured["lookup_thread_id"] = thread_id
|
||||
if existing_child_conversation and getattr(existing_child_conversation, "thread_id", None) == thread_id:
|
||||
return existing_child_conversation
|
||||
return None
|
||||
|
||||
async def add_conversation(
|
||||
self,
|
||||
*,
|
||||
uid: str,
|
||||
agent_id: str,
|
||||
title: str,
|
||||
thread_id: str,
|
||||
metadata: dict,
|
||||
):
|
||||
captured["conversation"] = {
|
||||
"uid": uid,
|
||||
"agent_id": agent_id,
|
||||
"title": title,
|
||||
"thread_id": thread_id,
|
||||
"metadata": metadata,
|
||||
}
|
||||
child_conversation.thread_id = thread_id
|
||||
return child_conversation
|
||||
|
||||
class ThreadRepo:
|
||||
def __init__(self, _db):
|
||||
pass
|
||||
|
||||
async def get_by_child_thread_for_user(self, child_thread_id: str, uid: str):
|
||||
assert uid == "user-1"
|
||||
captured["relation_lookup_thread_id"] = child_thread_id
|
||||
if existing_relation and existing_relation.child_thread_id != child_thread_id:
|
||||
return None
|
||||
return existing_relation
|
||||
|
||||
async def create(self, **kwargs):
|
||||
captured["relation"] = kwargs
|
||||
relation = created_relation or _relation(child_thread_id=kwargs["child_thread_id"])
|
||||
relation.child_thread_id = kwargs["child_thread_id"]
|
||||
return relation
|
||||
|
||||
async def get_for_user(self, relation_id: int, uid: str):
|
||||
assert relation_id == 77
|
||||
assert uid == "user-1"
|
||||
return relation_by_id
|
||||
|
||||
monkeypatch.setattr(service_module, "AgentRunRepository", RunRepo)
|
||||
monkeypatch.setattr(service_module, "ConversationRepository", ConvRepo)
|
||||
monkeypatch.setattr(service_module, "SubagentThreadRepository", ThreadRepo)
|
||||
|
||||
|
||||
def _patch_run_record_creation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
db: _FakeDB,
|
||||
*,
|
||||
missing_subagent: bool = False,
|
||||
active_run=None,
|
||||
):
|
||||
db.active_run = active_run
|
||||
|
||||
class _FakeContext:
|
||||
def __init__(self):
|
||||
self.model = "agent-default-model"
|
||||
|
||||
def update_from_dict(self, data: dict):
|
||||
for key, value in data.items():
|
||||
if hasattr(self, key):
|
||||
setattr(self, key, value)
|
||||
|
||||
class _FakeBackend:
|
||||
context_schema = _FakeContext
|
||||
|
||||
class ConvRepo:
|
||||
def __init__(self, db_session):
|
||||
del db_session
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
del thread_id
|
||||
return SimpleNamespace(id=20, uid="user-1", status="subagent", agent_id="worker")
|
||||
|
||||
class AgentRepo:
|
||||
def __init__(self, db_session):
|
||||
del db_session
|
||||
|
||||
async def get_visible_by_slug(self, *, slug: str, user, kind="main"):
|
||||
del user
|
||||
assert kind == "subagent"
|
||||
if missing_subagent:
|
||||
return None
|
||||
return SimpleNamespace(
|
||||
slug=slug,
|
||||
name="Worker",
|
||||
backend_id="SubAgentBackend",
|
||||
config_json={"context": {}},
|
||||
is_subagent=True,
|
||||
)
|
||||
|
||||
class RunRepo:
|
||||
def __init__(self, db_session):
|
||||
self.db = db_session
|
||||
|
||||
async def get_run_by_request_id(self, request_id: str):
|
||||
self.db.request_id_lookups.append(request_id)
|
||||
return self.db.existing_run
|
||||
|
||||
async def get_active_run_by_thread_for_user(self, *, agent_slug: str, conversation_thread_id: str, uid: str):
|
||||
self.db.active_run_lookup = {
|
||||
"agent_slug": agent_slug,
|
||||
"conversation_thread_id": conversation_thread_id,
|
||||
"uid": uid,
|
||||
}
|
||||
return self.db.active_run
|
||||
|
||||
async def create_run(self, **kwargs):
|
||||
self.db.created_run_kwargs = kwargs
|
||||
self.db.created_run = SimpleNamespace(
|
||||
id=kwargs["run_id"],
|
||||
conversation_thread_id=kwargs["conversation_thread_id"],
|
||||
agent_slug=kwargs["agent_slug"],
|
||||
status="pending",
|
||||
request_id=kwargs["request_id"],
|
||||
uid=kwargs["uid"],
|
||||
run_type=kwargs["run_type"],
|
||||
created_by_run_id=kwargs.get("created_by_run_id"),
|
||||
subagent_thread_relation_id=kwargs.get("subagent_thread_relation_id"),
|
||||
)
|
||||
return self.db.created_run
|
||||
|
||||
monkeypatch.setattr(agent_run_service.agent_manager, "get_agent", lambda backend_id: _FakeBackend())
|
||||
monkeypatch.setattr(agent_run_service, "ConversationRepository", ConvRepo)
|
||||
monkeypatch.setattr(agent_run_service, "AgentRepository", AgentRepo)
|
||||
monkeypatch.setattr(agent_run_service, "AgentRunRepository", RunRepo)
|
||||
|
||||
|
||||
def _fake_create_run_record(captured: dict[str, object], *, run_id: str = "child-run"):
|
||||
async def fake_create_run_record(_self, **kwargs):
|
||||
captured["create_run_record"] = kwargs
|
||||
return (
|
||||
SimpleNamespace(
|
||||
id=run_id,
|
||||
conversation_thread_id="child-thread",
|
||||
agent_slug="worker",
|
||||
status="pending",
|
||||
created_by_run_id=kwargs["creator_run"].id,
|
||||
subagent_thread_relation_id=kwargs["relation"].id,
|
||||
input_payload={
|
||||
"runtime": {
|
||||
"tool_call_id": kwargs["tool_call_id"],
|
||||
},
|
||||
},
|
||||
),
|
||||
True,
|
||||
)
|
||||
|
||||
return fake_create_run_record
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_creates_child_relation_run_and_enqueue(monkeypatch: pytest.MonkeyPatch):
|
||||
db = _FakeDB()
|
||||
captured: dict[str, object] = {}
|
||||
enqueued: list[str] = []
|
||||
child_conversation = SimpleNamespace(id=20, uid="user-1", agent_id="worker", status="active")
|
||||
relation = _relation(child_thread_id="")
|
||||
|
||||
_patch_repos(
|
||||
monkeypatch,
|
||||
captured=captured,
|
||||
child_conversation=child_conversation,
|
||||
created_relation=relation,
|
||||
)
|
||||
|
||||
async def fake_enqueue(run_id: str):
|
||||
enqueued.append(run_id)
|
||||
|
||||
monkeypatch.setattr(SubagentRunService, "_create_run_record", _fake_create_run_record(captured))
|
||||
monkeypatch.setattr(service_module.agent_run_service, "enqueue_agent_run", fake_enqueue)
|
||||
|
||||
result = await SubagentRunService(db).start(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
agent_item=_agent(),
|
||||
input_message=build_chat_input_message("run in background"),
|
||||
tool_call_id="tool-1",
|
||||
model_spec="provider:model",
|
||||
)
|
||||
|
||||
child_thread_id = make_child_thread_id("parent-thread", "worker", "tool-1")
|
||||
assert result.created is True
|
||||
assert result.continuing is False
|
||||
assert result.relation.child_thread_id == child_thread_id
|
||||
assert result.relation is relation
|
||||
assert child_conversation.status == "subagent"
|
||||
assert captured["conversation"]["metadata"]["parent_conversation_id"] == 10
|
||||
assert captured["relation"] == {
|
||||
"uid": "user-1",
|
||||
"parent_conversation_id": 10,
|
||||
"child_conversation_id": 20,
|
||||
"child_thread_id": child_thread_id,
|
||||
"subagent_slug": "worker",
|
||||
"created_by_run_id": "parent-run",
|
||||
}
|
||||
assert captured["create_run_record"]["relation"].id == 77
|
||||
assert captured["create_run_record"]["model_spec"] == "provider:model"
|
||||
assert captured["create_run_record"]["creator_run"].id == "parent-run"
|
||||
assert captured["create_run_record"]["input_message"].content == "run in background"
|
||||
assert captured["create_run_record"]["input_message"].raw_message()["type"] == "human"
|
||||
assert captured["create_run_record"]["input_message"].raw_message()["content"] == "run in background"
|
||||
assert db.committed is True
|
||||
assert enqueued == ["child-run"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_continues_existing_relation(monkeypatch: pytest.MonkeyPatch):
|
||||
db = _FakeDB()
|
||||
captured: dict[str, object] = {}
|
||||
relation = _relation()
|
||||
_patch_repos(monkeypatch, captured=captured, existing_relation=relation)
|
||||
|
||||
async def fake_enqueue(run_id: str):
|
||||
captured["enqueued"] = run_id
|
||||
|
||||
monkeypatch.setattr(
|
||||
SubagentRunService,
|
||||
"_create_run_record",
|
||||
_fake_create_run_record(captured, run_id="child-run-2"),
|
||||
)
|
||||
monkeypatch.setattr(service_module.agent_run_service, "enqueue_agent_run", fake_enqueue)
|
||||
|
||||
result = await SubagentRunService(db).start(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
agent_item=_agent(),
|
||||
input_message=build_chat_input_message("continue"),
|
||||
tool_call_id="tool-2",
|
||||
requested_thread_id="child-thread",
|
||||
)
|
||||
|
||||
assert result.continuing is True
|
||||
assert result.relation is relation
|
||||
assert captured["relation_lookup_thread_id"] == "child-thread"
|
||||
assert captured["create_run_record"]["creator_run"].id == "parent-run"
|
||||
assert captured["create_run_record"]["input_message"].content == "continue"
|
||||
assert captured["create_run_record"]["input_message"].raw_message()["type"] == "human"
|
||||
assert captured["create_run_record"]["input_message"].raw_message()["content"] == "continue"
|
||||
assert captured["enqueued"] == "child-run-2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_rejects_thread_from_another_parent(monkeypatch: pytest.MonkeyPatch):
|
||||
_patch_repos(monkeypatch, existing_relation=_relation(parent_conversation_id=99))
|
||||
|
||||
with pytest.raises(ValueError, match="线程不属于当前对话"):
|
||||
await SubagentRunService(_FakeDB()).start(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
agent_item=_agent(),
|
||||
input_message=build_chat_input_message("continue"),
|
||||
tool_call_id="tool-2",
|
||||
requested_thread_id="child-thread",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_rejects_thread_from_another_subagent(monkeypatch: pytest.MonkeyPatch):
|
||||
_patch_repos(monkeypatch, existing_relation=_relation(subagent_slug="other"))
|
||||
|
||||
with pytest.raises(ValueError, match="属于子智能体 other"):
|
||||
await SubagentRunService(_FakeDB()).start(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
agent_item=_agent(),
|
||||
input_message=build_chat_input_message("continue"),
|
||||
tool_call_id="tool-2",
|
||||
requested_thread_id="child-thread",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_rejects_parent_run_without_conversation_before_child_creation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
captured: dict[str, object] = {}
|
||||
_patch_repos(
|
||||
monkeypatch,
|
||||
captured=captured,
|
||||
parent_run=SimpleNamespace(id="parent-run", conversation_thread_id="parent-thread", conversation_id=None),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="缺少 conversation_id"):
|
||||
await SubagentRunService(_FakeDB()).start(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
agent_item=_agent(),
|
||||
input_message=build_chat_input_message("new child"),
|
||||
tool_call_id="tool-2",
|
||||
)
|
||||
|
||||
assert "conversation" not in captured
|
||||
assert "relation" not in captured
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_rejects_subagent_as_creator(monkeypatch: pytest.MonkeyPatch):
|
||||
captured: dict[str, object] = {}
|
||||
_patch_repos(
|
||||
monkeypatch,
|
||||
captured=captured,
|
||||
parent_run=SimpleNamespace(
|
||||
id="parent-run",
|
||||
conversation_thread_id="parent-thread",
|
||||
conversation_id=10,
|
||||
run_type="subagent",
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="子智能体不能创建子智能体"):
|
||||
await SubagentRunService(_FakeDB()).start(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
agent_item=_agent(),
|
||||
input_message=build_chat_input_message("nested child"),
|
||||
tool_call_id="tool-2",
|
||||
)
|
||||
|
||||
assert "relation_lookup_thread_id" not in captured
|
||||
assert "conversation" not in captured
|
||||
assert "relation" not in captured
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_rejects_child_thread_owned_by_normal_conversation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
captured: dict[str, object] = {}
|
||||
child_thread_id = make_child_thread_id("parent-thread", "worker", "tool-2")
|
||||
_patch_repos(
|
||||
monkeypatch,
|
||||
captured=captured,
|
||||
existing_child_conversation=SimpleNamespace(
|
||||
id=20,
|
||||
uid="user-1",
|
||||
agent_id="worker",
|
||||
thread_id=child_thread_id,
|
||||
status="active",
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="普通对话占用"):
|
||||
await SubagentRunService(_FakeDB()).start(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
agent_item=_agent(),
|
||||
input_message=build_chat_input_message("new child"),
|
||||
tool_call_id="tool-2",
|
||||
)
|
||||
|
||||
assert captured["lookup_thread_id"] == child_thread_id
|
||||
assert "relation" not in captured
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_translates_busy_run(monkeypatch: pytest.MonkeyPatch):
|
||||
_patch_repos(monkeypatch, existing_relation=_relation())
|
||||
|
||||
async def fake_create_run_record(_self, **kwargs):
|
||||
del kwargs
|
||||
raise HTTPException(status_code=409, detail={"code": "run_busy", "active_run_id": "active-run"})
|
||||
|
||||
async def fake_enqueue(run_id: str):
|
||||
del run_id
|
||||
raise AssertionError("busy run should not enqueue")
|
||||
|
||||
monkeypatch.setattr(SubagentRunService, "_create_run_record", fake_create_run_record)
|
||||
monkeypatch.setattr(service_module.agent_run_service, "enqueue_agent_run", fake_enqueue)
|
||||
|
||||
with pytest.raises(SubagentRunBusy) as exc:
|
||||
await SubagentRunService(_FakeDB()).start(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
agent_item=_agent(),
|
||||
input_message=build_chat_input_message("continue"),
|
||||
tool_call_id="tool-2",
|
||||
requested_thread_id="child-thread",
|
||||
)
|
||||
|
||||
assert exc.value.thread_id == "child-thread"
|
||||
assert exc.value.active_run_id == "active-run"
|
||||
assert exc.value.to_payload() == {
|
||||
"status": "busy",
|
||||
"thread_id": "child-thread",
|
||||
"active_run_id": "active-run",
|
||||
"active_run_status": None,
|
||||
"message": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_create_run_record_persists_subagent_context(monkeypatch: pytest.MonkeyPatch):
|
||||
db = _FakeDB()
|
||||
_patch_run_record_creation(monkeypatch, db)
|
||||
creator_run = SimpleNamespace(id="parent-run", conversation_id=10, conversation_thread_id="parent-thread")
|
||||
relation = _relation(child_thread_id="child-thread", parent_conversation_id=10, subagent_slug="worker")
|
||||
|
||||
run, created = await SubagentRunService(db)._create_run_record(
|
||||
input_message=build_chat_input_message("delegate this"),
|
||||
request_id="subagent-req",
|
||||
current_uid="user-1",
|
||||
model_spec=None,
|
||||
creator_run=creator_run,
|
||||
relation=relation,
|
||||
tool_call_id="tool-1",
|
||||
file_thread_id="parent-file-thread",
|
||||
)
|
||||
|
||||
assert created is True
|
||||
assert run is db.created_run
|
||||
assert db.added[0].content == "delegate this"
|
||||
assert db.added[0].extra_metadata["source"] == "subagent"
|
||||
assert db.added[0].extra_metadata["raw_message"]["type"] == "human"
|
||||
assert db.added[0].extra_metadata["raw_message"]["content"] == "delegate this"
|
||||
assert db.created_run_kwargs["run_type"] == "subagent"
|
||||
assert db.created_run_kwargs["created_by_run_id"] == "parent-run"
|
||||
assert db.created_run_kwargs["subagent_thread_relation_id"] == 77
|
||||
assert db.created_run_kwargs["conversation_thread_id"] == "child-thread"
|
||||
assert db.created_run_kwargs["input_message_id"] == 10
|
||||
assert db.created_run_kwargs["input_payload"] == {
|
||||
"model_spec": "agent-default-model",
|
||||
"runtime": {
|
||||
"tool_call_id": "tool-1",
|
||||
"subagent_name": "Worker",
|
||||
"parent_thread_id": "parent-thread",
|
||||
"file_thread_id": "parent-file-thread",
|
||||
"skills_thread_id": "child-thread",
|
||||
},
|
||||
}
|
||||
assert db.committed is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_create_run_record_uses_creator_thread_when_file_thread_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
db = _FakeDB()
|
||||
_patch_run_record_creation(monkeypatch, db)
|
||||
creator_run = SimpleNamespace(id="parent-run", conversation_id=10, conversation_thread_id="current-parent-thread")
|
||||
relation = _relation(child_thread_id="child-thread", parent_conversation_id=10, subagent_slug="worker")
|
||||
|
||||
await SubagentRunService(db)._create_run_record(
|
||||
input_message=build_chat_input_message("continue this"),
|
||||
request_id="subagent-req-2",
|
||||
current_uid="user-1",
|
||||
model_spec=None,
|
||||
creator_run=creator_run,
|
||||
relation=relation,
|
||||
tool_call_id="tool-2",
|
||||
file_thread_id=None,
|
||||
)
|
||||
|
||||
assert db.created_run_kwargs["created_by_run_id"] == "parent-run"
|
||||
assert db.created_run_kwargs["input_payload"]["runtime"]["parent_thread_id"] == "current-parent-thread"
|
||||
assert db.created_run_kwargs["input_payload"]["runtime"]["file_thread_id"] == "current-parent-thread"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_create_run_record_rejects_non_subagent_definition(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
db = _FakeDB()
|
||||
_patch_run_record_creation(monkeypatch, db, missing_subagent=True)
|
||||
creator_run = SimpleNamespace(id="parent-run", conversation_id=10, conversation_thread_id="parent-thread")
|
||||
relation = _relation(child_thread_id="child-thread", parent_conversation_id=10, subagent_slug="worker")
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await SubagentRunService(db)._create_run_record(
|
||||
input_message=build_chat_input_message("delegate this"),
|
||||
request_id="subagent-req",
|
||||
current_uid="user-1",
|
||||
model_spec=None,
|
||||
creator_run=creator_run,
|
||||
relation=relation,
|
||||
tool_call_id="tool-1",
|
||||
file_thread_id="parent-thread",
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 404
|
||||
assert "智能体不存在" in exc.value.detail
|
||||
assert db.created_run_kwargs is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_create_run_record_rejects_relation_parent_mismatch(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
db = _FakeDB()
|
||||
_patch_run_record_creation(monkeypatch, db)
|
||||
creator_run = SimpleNamespace(id="parent-run", conversation_id=99, conversation_thread_id="parent-thread")
|
||||
relation = _relation(child_thread_id="child-thread", parent_conversation_id=10, subagent_slug="worker")
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await SubagentRunService(db)._create_run_record(
|
||||
input_message=build_chat_input_message("delegate this"),
|
||||
request_id="subagent-req",
|
||||
current_uid="user-1",
|
||||
model_spec=None,
|
||||
creator_run=creator_run,
|
||||
relation=relation,
|
||||
tool_call_id="tool-1",
|
||||
file_thread_id="parent-thread",
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 409
|
||||
assert "subagent thread relation" in exc.value.detail
|
||||
assert db.created_run_kwargs is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_loads_run_only_for_current_parent_conversation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
parent_run = SimpleNamespace(id="parent-run", conversation_id=10)
|
||||
child_run = _child_run()
|
||||
_patch_repos(
|
||||
monkeypatch,
|
||||
parent_run=parent_run,
|
||||
child_run=child_run,
|
||||
relation_by_id=_relation(),
|
||||
)
|
||||
|
||||
run = await SubagentRunService(_FakeDB()).get_run_for_creator(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
run_id="child-run",
|
||||
)
|
||||
|
||||
assert run is child_run
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_rejects_run_from_another_parent_conversation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
parent_run = SimpleNamespace(id="parent-run", conversation_id=10)
|
||||
_patch_repos(
|
||||
monkeypatch,
|
||||
parent_run=parent_run,
|
||||
child_run=_child_run(),
|
||||
relation_by_id=_relation(parent_conversation_id=99),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="不存在或不属于当前父运行"):
|
||||
await SubagentRunService(_FakeDB()).get_run_for_creator(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
run_id="child-run",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subagent_run_service_rejects_run_from_another_parent_run(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
parent_run = SimpleNamespace(id="parent-run", conversation_id=10)
|
||||
_patch_repos(
|
||||
monkeypatch,
|
||||
parent_run=parent_run,
|
||||
child_run=_child_run(created_by_run_id="previous-parent-run"),
|
||||
relation_by_id=_relation(parent_conversation_id=10),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="不存在或不属于当前父运行"):
|
||||
await SubagentRunService(_FakeDB()).get_run_for_creator(
|
||||
uid="user-1",
|
||||
created_by_run_id="parent-run",
|
||||
run_id="child-run",
|
||||
)
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Tasker 行为单元测试:payload 暴露、进度节流、终态保留与重启恢复。
|
||||
|
||||
使用内存 fake repo,不依赖真实数据库与 Docker。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.services import task_service
|
||||
from yuxi.services.task_service import Tasker
|
||||
|
||||
|
||||
class FakeRecord:
|
||||
def __init__(self, data: dict):
|
||||
self._data = data
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return self._data
|
||||
|
||||
|
||||
class FakeRepo:
|
||||
"""记录 upsert/delete 调用的内存仓库替身。"""
|
||||
|
||||
def __init__(self, preset: list[FakeRecord] | None = None):
|
||||
self.preset = preset or []
|
||||
self.upsert_calls = 0
|
||||
self.progress_writes: list[float] = []
|
||||
self.deleted: list[str] = []
|
||||
|
||||
async def upsert(self, task_id: str, data: dict) -> None:
|
||||
self.upsert_calls += 1
|
||||
self.progress_writes.append(data.get("progress"))
|
||||
|
||||
async def delete(self, task_id: str) -> bool:
|
||||
self.deleted.append(task_id)
|
||||
return True
|
||||
|
||||
async def list_all(self) -> list[FakeRecord]:
|
||||
return self.preset
|
||||
|
||||
|
||||
async def _make_tasker(repo: FakeRepo, worker_count: int = 1) -> Tasker:
|
||||
tasker = Tasker(worker_count=worker_count)
|
||||
tasker._repo = repo
|
||||
await tasker.start()
|
||||
return tasker
|
||||
|
||||
|
||||
async def _wait_status(tasker: Tasker, task_id: str, statuses: set[str], timeout: float = 2.0) -> dict:
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout
|
||||
while True:
|
||||
task = await tasker.get_task(task_id)
|
||||
if task and task["status"] in statuses:
|
||||
return task
|
||||
if loop.time() > deadline:
|
||||
raise AssertionError(f"任务 {task_id} 未在超时内进入 {statuses}")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
async def test_task_context_exposes_payload():
|
||||
repo = FakeRepo()
|
||||
tasker = await _make_tasker(repo)
|
||||
seen: dict = {}
|
||||
|
||||
async def coro(ctx):
|
||||
seen["payload"] = ctx.payload
|
||||
return "ok"
|
||||
|
||||
task = await tasker.enqueue(name="x", task_type="demo", payload={"a": 1}, coroutine=coro)
|
||||
await _wait_status(tasker, task.id, {"success"})
|
||||
|
||||
assert seen["payload"] == {"a": 1}
|
||||
await tasker.shutdown()
|
||||
|
||||
|
||||
async def test_progress_updates_are_throttled():
|
||||
repo = FakeRepo()
|
||||
tasker = await _make_tasker(repo)
|
||||
|
||||
async def coro(ctx):
|
||||
for percent in range(101):
|
||||
await ctx.set_progress(percent)
|
||||
return "done"
|
||||
|
||||
task = await tasker.enqueue(name="x", task_type="demo", coroutine=coro)
|
||||
final = await _wait_status(tasker, task.id, {"success"})
|
||||
|
||||
# 101 次进度推进经节流后落库次数应远小于 101(含 enqueue/running/success 也仅个位数额外写入)
|
||||
assert repo.upsert_calls < 60
|
||||
# 内存中进度仍为完整的 100
|
||||
assert final["progress"] == 100
|
||||
await tasker.shutdown()
|
||||
|
||||
|
||||
async def test_explicit_none_result_is_persisted():
|
||||
repo = FakeRepo()
|
||||
tasker = await _make_tasker(repo)
|
||||
|
||||
async def coro(ctx):
|
||||
await ctx.set_result("partial")
|
||||
return None
|
||||
|
||||
task = await tasker.enqueue(name="x", task_type="demo", coroutine=coro)
|
||||
final = await _wait_status(tasker, task.id, {"success"})
|
||||
|
||||
# 协程最终返回 None 应覆盖中途结果(sentinel 区分「未传」与「显式 None」)
|
||||
assert final["result"] is None
|
||||
await tasker.shutdown()
|
||||
|
||||
|
||||
async def test_completed_tasks_are_pruned_to_limit(monkeypatch):
|
||||
monkeypatch.setattr(task_service, "MAX_TERMINAL_TASKS", 3)
|
||||
repo = FakeRepo()
|
||||
tasker = await _make_tasker(repo)
|
||||
|
||||
async def coro(ctx):
|
||||
return "ok"
|
||||
|
||||
for index in range(6):
|
||||
task = await tasker.enqueue(name=f"t{index}", task_type="demo", coroutine=coro)
|
||||
await _wait_status(tasker, task.id, {"success"})
|
||||
|
||||
listing = await tasker.list_tasks(limit=100)
|
||||
assert listing["summary"]["total"] <= 3
|
||||
assert len(repo.deleted) >= 3
|
||||
await tasker.shutdown()
|
||||
|
||||
|
||||
async def test_load_state_marks_interrupted_and_prunes(monkeypatch):
|
||||
monkeypatch.setattr(task_service, "MAX_TERMINAL_TASKS", 2)
|
||||
repo = FakeRepo(
|
||||
preset=[
|
||||
FakeRecord({"id": "a", "name": "a", "type": "demo", "status": "running",
|
||||
"created_at": "2026-01-01T00:00:05"}),
|
||||
FakeRecord({"id": "b", "name": "b", "type": "demo", "status": "success",
|
||||
"created_at": "2026-01-01T00:00:04"}),
|
||||
FakeRecord({"id": "c", "name": "c", "type": "demo", "status": "success",
|
||||
"created_at": "2026-01-01T00:00:03"}),
|
||||
FakeRecord({"id": "d", "name": "d", "type": "demo", "status": "success",
|
||||
"created_at": "2026-01-01T00:00:02"}),
|
||||
]
|
||||
)
|
||||
tasker = await _make_tasker(repo)
|
||||
|
||||
# 中断的 running 任务被标记为 failed
|
||||
interrupted = await tasker.get_task("a")
|
||||
assert interrupted["status"] == "failed"
|
||||
# 仅保留最近 MAX_TERMINAL_TASKS 条终态任务,最旧的被清理
|
||||
listing = await tasker.list_tasks(limit=100)
|
||||
assert listing["summary"]["total"] == 2
|
||||
assert "c" in repo.deleted and "d" in repo.deleted
|
||||
await tasker.shutdown()
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from yuxi.services import thread_files_service as svc
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_thread_artifact_view_blocks_symlink_escape(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
thread_root = tmp_path / "threads" / "thread-1" / "user-data"
|
||||
uploads_dir = thread_root / "uploads"
|
||||
uploads_dir.mkdir(parents=True, exist_ok=True)
|
||||
outside_file = tmp_path / "outside.txt"
|
||||
outside_file.write_text("secret", encoding="utf-8")
|
||||
(uploads_dir / "escape.txt").symlink_to(outside_file)
|
||||
|
||||
class _Conversation:
|
||||
uid = "user-1"
|
||||
|
||||
async def _fake_require_user_conversation(_repo, _thread_id: str, _current_uid: str):
|
||||
return _Conversation()
|
||||
|
||||
monkeypatch.setattr(svc, "require_user_conversation", _fake_require_user_conversation)
|
||||
monkeypatch.setattr(svc, "ensure_thread_dirs", lambda _thread_id, _uid: None)
|
||||
monkeypatch.setattr(
|
||||
svc,
|
||||
"sandbox_workspace_dir",
|
||||
lambda _thread_id, _uid: tmp_path / "shared" / _uid / "workspace",
|
||||
)
|
||||
monkeypatch.setattr(svc, "sandbox_uploads_dir", lambda _thread_id: uploads_dir)
|
||||
monkeypatch.setattr(svc, "sandbox_outputs_dir", lambda _thread_id: thread_root / "outputs")
|
||||
monkeypatch.setattr(svc, "resolve_virtual_path", lambda _thread_id, _path, *, uid: uploads_dir / "escape.txt")
|
||||
monkeypatch.setattr(svc, "ConversationRepository", lambda _db: object())
|
||||
|
||||
with pytest.raises(HTTPException, match="access denied"):
|
||||
await svc.resolve_thread_artifact_view(
|
||||
thread_id="thread-1",
|
||||
current_uid="user-1",
|
||||
db=None,
|
||||
path="/home/gem/user-data/uploads/escape.txt",
|
||||
)
|
||||
@@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from yuxi.agents.toolkits import service as tool_service
|
||||
|
||||
|
||||
def test_get_tool_metadata_includes_config_guide(monkeypatch):
|
||||
tool_service._metadata_cache.clear()
|
||||
|
||||
fake_tool = SimpleNamespace(
|
||||
name="demo_tool",
|
||||
description="demo description",
|
||||
metadata={},
|
||||
args_schema=None,
|
||||
)
|
||||
fake_extra = SimpleNamespace(
|
||||
category="buildin",
|
||||
tags=["demo"],
|
||||
display_name="演示工具",
|
||||
config_guide="请先配置 DEMO_API_KEY",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.toolkits.registry.get_all_tool_instances",
|
||||
lambda: [fake_tool],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.toolkits.registry.get_all_extra_metadata",
|
||||
lambda: {"demo_tool": fake_extra},
|
||||
)
|
||||
|
||||
result = tool_service.get_tool_metadata()
|
||||
|
||||
assert result == [
|
||||
{
|
||||
"slug": "demo_tool",
|
||||
"name": "演示工具",
|
||||
"description": "demo description",
|
||||
"metadata": {},
|
||||
"args": [],
|
||||
"category": "buildin",
|
||||
"tags": ["demo"],
|
||||
"config_guide": "请先配置 DEMO_API_KEY",
|
||||
}
|
||||
]
|
||||
|
||||
tool_service._metadata_cache.clear()
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from yuxi.agents.backends.sandbox import paths as sandbox_paths
|
||||
from yuxi.services import viewer_filesystem_service as svc
|
||||
from yuxi.services import workspace_service
|
||||
|
||||
|
||||
def test_resolve_local_user_data_path_blocks_upload_symlink_escape(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(sandbox_paths.conf, "save_dir", str(tmp_path))
|
||||
thread_id = "thread-1"
|
||||
uid = "user-1"
|
||||
sandbox_paths.ensure_thread_dirs(thread_id, uid)
|
||||
|
||||
outside_file = tmp_path / "outside.txt"
|
||||
outside_file.write_text("outside", encoding="utf-8")
|
||||
escape_link = sandbox_paths.sandbox_uploads_dir(thread_id) / "escape.txt"
|
||||
escape_link.symlink_to(outside_file)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
svc._resolve_local_user_data_path(thread_id, uid, "/home/gem/user-data/uploads/escape.txt")
|
||||
|
||||
assert exc_info.value.status_code == 403
|
||||
|
||||
|
||||
def test_list_local_entries_skips_symlink_escape(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(sandbox_paths.conf, "save_dir", str(tmp_path))
|
||||
thread_id = "thread-1"
|
||||
uid = "user-1"
|
||||
sandbox_paths.ensure_thread_dirs(thread_id, uid)
|
||||
|
||||
uploads_dir = sandbox_paths.sandbox_uploads_dir(thread_id)
|
||||
(uploads_dir / "safe.txt").write_text("safe", encoding="utf-8")
|
||||
outside_file = tmp_path / "outside.txt"
|
||||
outside_file.write_text("outside", encoding="utf-8")
|
||||
(uploads_dir / "escape.txt").symlink_to(outside_file)
|
||||
|
||||
entries = svc._list_local_entries(thread_id, uid, uploads_dir)
|
||||
|
||||
assert {entry["path"] for entry in entries} == {"/home/gem/user-data/uploads/safe.txt"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_viewer_workspace_office_file_returns_pdf_preview(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(sandbox_paths.conf, "save_dir", str(tmp_path))
|
||||
thread_id = "thread-1"
|
||||
uid = "user-1"
|
||||
user = SimpleNamespace(uid=uid)
|
||||
sandbox_paths.ensure_thread_dirs(thread_id, uid)
|
||||
target = sandbox_paths.sandbox_workspace_dir(thread_id, uid) / "slides.pptx"
|
||||
target.write_bytes(b"presentation")
|
||||
|
||||
async def fake_resolve_viewer_state(**kwargs):
|
||||
return None, None, []
|
||||
|
||||
async def fake_convert(filename: str, content: bytes) -> bytes:
|
||||
assert filename == "slides.pptx"
|
||||
assert content == b"presentation"
|
||||
return b"%PDF-1.4\npreview"
|
||||
|
||||
monkeypatch.setattr(svc, "_resolve_viewer_state", fake_resolve_viewer_state)
|
||||
monkeypatch.setattr(workspace_service, "convert_office_to_pdf", fake_convert)
|
||||
|
||||
response = await svc.read_viewer_file_content(
|
||||
thread_id=thread_id,
|
||||
path="/home/gem/user-data/workspace/slides.pptx",
|
||||
current_user=user,
|
||||
db=None,
|
||||
)
|
||||
body = b""
|
||||
async for chunk in response.body_iterator:
|
||||
body += chunk
|
||||
|
||||
assert response.media_type == "application/pdf"
|
||||
assert response.headers["x-yuxi-preview-type"] == "pdf"
|
||||
assert body == b"%PDF-1.4\npreview"
|
||||
@@ -0,0 +1,351 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException, UploadFile
|
||||
|
||||
from yuxi.agents.backends.sandbox import paths as workspace_paths
|
||||
from yuxi.services import workspace_service as svc
|
||||
|
||||
|
||||
def _user() -> SimpleNamespace:
|
||||
return SimpleNamespace(id="db-id-1", uid="user-1")
|
||||
|
||||
|
||||
def test_workspace_root_creates_default_agents_prompt_file(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
|
||||
root = svc._workspace_root(_user())
|
||||
|
||||
agents_file = root / "agents" / "AGENTS.md"
|
||||
assert root == tmp_path / "threads" / "shared" / "user-1" / "workspace"
|
||||
assert agents_file.is_file()
|
||||
assert agents_file.read_text(encoding="utf-8") == ""
|
||||
|
||||
|
||||
def test_ensure_thread_dirs_creates_default_agents_prompt_file(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
|
||||
workspace_paths.ensure_thread_dirs("thread-1", "user-1")
|
||||
|
||||
agents_file = tmp_path / "threads" / "shared" / "user-1" / "workspace" / "agents" / "AGENTS.md"
|
||||
assert agents_file.is_file()
|
||||
assert agents_file.read_text(encoding="utf-8") == ""
|
||||
|
||||
|
||||
def test_workspace_root_keeps_existing_agents_prompt_file(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
agents_dir = tmp_path / "threads" / "shared" / "user-1" / "workspace" / "agents"
|
||||
agents_dir.mkdir(parents=True)
|
||||
agents_file = agents_dir / "AGENTS.md"
|
||||
agents_file.write_text("保留已有内容", encoding="utf-8")
|
||||
|
||||
root = svc._workspace_root(_user())
|
||||
|
||||
assert root == tmp_path / "threads" / "shared" / "user-1" / "workspace"
|
||||
assert agents_file.read_text(encoding="utf-8") == "保留已有内容"
|
||||
|
||||
|
||||
def test_workspace_root_rejects_symlink_root(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user_root = tmp_path / "threads" / "shared" / "user-1"
|
||||
outside_root = tmp_path / "outside"
|
||||
user_root.mkdir(parents=True)
|
||||
outside_root.mkdir()
|
||||
(user_root / "workspace").symlink_to(outside_root, target_is_directory=True)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
svc._workspace_root(_user())
|
||||
|
||||
assert exc_info.value.status_code == 403
|
||||
|
||||
|
||||
def test_ensure_workspace_default_files_rejects_path_outside_threads_root(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path / "saves"))
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
workspace_paths.ensure_workspace_default_files(tmp_path / "outside-workspace")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_workspace_file_content_returns_unsupported_for_non_utf8_text(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
target = root / "bad.txt"
|
||||
target.write_bytes(b"\xff\xfe\x00")
|
||||
|
||||
result = await svc.read_workspace_file_content(path="/bad.txt", current_user=user)
|
||||
|
||||
assert result["content"] is None
|
||||
assert result["preview_type"] == "unsupported"
|
||||
assert result["supported"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_workspace_file_content_returns_pdf_preview_for_office_file(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
target = root / "demo.docx"
|
||||
target.write_bytes(b"office")
|
||||
|
||||
async def fake_convert(filename: str, content: bytes) -> bytes:
|
||||
assert filename == "demo.docx"
|
||||
assert content == b"office"
|
||||
return b"%PDF-1.4\npreview"
|
||||
|
||||
monkeypatch.setattr(svc, "convert_office_to_pdf", fake_convert)
|
||||
|
||||
result = await svc.read_workspace_file_content(path="/demo.docx", current_user=user)
|
||||
body = b""
|
||||
async for chunk in result.body_iterator:
|
||||
body += chunk
|
||||
|
||||
assert result.media_type == "application/pdf"
|
||||
assert result.headers["x-yuxi-preview-type"] == "pdf"
|
||||
assert body == b"%PDF-1.4\npreview"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_workspace_file_content_rejects_xlsx_preview(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
target = root / "sheet.xlsx"
|
||||
target.write_bytes(b"PK\x03\x04excel")
|
||||
|
||||
result = await svc.read_workspace_file_content(path="/sheet.xlsx", current_user=user)
|
||||
|
||||
assert result["content"] is None
|
||||
assert result["preview_type"] == "unsupported"
|
||||
assert result["supported"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preview_workspace_file_converts_office_file_to_pdf(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
target = root / "slides.pptx"
|
||||
target.write_bytes(b"presentation")
|
||||
|
||||
async def fake_convert(filename: str, content: bytes) -> bytes:
|
||||
assert filename == "slides.pptx"
|
||||
assert content == b"presentation"
|
||||
return b"%PDF-1.4\npreview"
|
||||
|
||||
monkeypatch.setattr(svc, "convert_office_to_pdf", fake_convert)
|
||||
|
||||
response = await svc.read_workspace_file_content(path="/slides.pptx", current_user=user)
|
||||
body = b""
|
||||
async for chunk in response.body_iterator:
|
||||
body += chunk
|
||||
|
||||
assert response.media_type == "application/pdf"
|
||||
assert body == b"%PDF-1.4\npreview"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preview_workspace_file_caches_office_pdf_conversion(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
target = root / "slides.pptx"
|
||||
target.write_bytes(b"presentation")
|
||||
|
||||
convert_calls = 0
|
||||
|
||||
async def fake_convert(filename: str, content: bytes) -> bytes:
|
||||
nonlocal convert_calls
|
||||
convert_calls += 1
|
||||
return b"%PDF-1.4\npreview"
|
||||
|
||||
monkeypatch.setattr(svc, "convert_office_to_pdf", fake_convert)
|
||||
|
||||
async def read_pdf() -> bytes:
|
||||
response = await svc.read_workspace_file_content(path="/slides.pptx", current_user=user)
|
||||
body = b""
|
||||
async for chunk in response.body_iterator:
|
||||
body += chunk
|
||||
return body
|
||||
|
||||
assert await read_pdf() == b"%PDF-1.4\npreview"
|
||||
assert await read_pdf() == b"%PDF-1.4\npreview"
|
||||
assert convert_calls == 1
|
||||
|
||||
target.write_bytes(b"presentation-v2")
|
||||
assert await read_pdf() == b"%PDF-1.4\npreview"
|
||||
assert convert_calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_workspace_file_keeps_office_original_file(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
target = root / "slides.pptx"
|
||||
target.write_bytes(b"presentation")
|
||||
|
||||
response = await svc.download_workspace_file(path="/slides.pptx", current_user=user)
|
||||
body = b""
|
||||
async for chunk in response.body_iterator:
|
||||
body += chunk
|
||||
|
||||
assert response.media_type == "application/vnd.openxmlformats-officedocument.presentationml.presentation"
|
||||
assert body == b"presentation"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_workspace_file_content_updates_markdown_file(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
target = root / "note.md"
|
||||
target.write_text("旧内容", encoding="utf-8")
|
||||
|
||||
result = await svc.write_workspace_file_content(path="/note.md", content="# 新内容", current_user=user)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["path"] == "/note.md"
|
||||
assert result["entry"]["path"] == "/note.md"
|
||||
assert target.read_text(encoding="utf-8") == "# 新内容"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_workspace_file_content_updates_txt_file(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
target = root / "note.txt"
|
||||
target.write_text("old", encoding="utf-8")
|
||||
|
||||
await svc.write_workspace_file_content(path="/note.txt", content="new", current_user=user)
|
||||
|
||||
assert target.read_text(encoding="utf-8") == "new"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_workspace_file_content_rejects_unsupported_suffix(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
target = root / "script.py"
|
||||
target.write_text("print('hello')", encoding="utf-8")
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await svc.write_workspace_file_content(path="/script.py", content="print('bye')", current_user=user)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert target.read_text(encoding="utf-8") == "print('hello')"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_workspace_file_content_rejects_directory_and_missing_file(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
svc._workspace_root(user)
|
||||
|
||||
with pytest.raises(HTTPException) as directory_error:
|
||||
await svc.write_workspace_file_content(path="/agents/", content="x", current_user=user)
|
||||
with pytest.raises(HTTPException) as missing_error:
|
||||
await svc.write_workspace_file_content(path="/missing.md", content="x", current_user=user)
|
||||
|
||||
assert directory_error.value.status_code == 400
|
||||
assert missing_error.value.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_workspace_file_content_blocks_path_traversal(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await svc.write_workspace_file_content(
|
||||
path="/../outside.md",
|
||||
content="x",
|
||||
current_user=_user(),
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_workspace_files_writes_files(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
uploads = [
|
||||
UploadFile(filename="demo.txt", file=BytesIO(b"hello")),
|
||||
UploadFile(filename="notes.md", file=BytesIO(b"# notes")),
|
||||
]
|
||||
|
||||
result = await svc.upload_workspace_files(parent_path="/", files=uploads, current_user=user)
|
||||
|
||||
assert result["success"] is True
|
||||
assert [entry["path"] for entry in result["entries"]] == ["/demo.txt", "/notes.md"]
|
||||
assert result["entries"][0]["size"] == 5
|
||||
assert (root / "demo.txt").read_bytes() == b"hello"
|
||||
assert (root / "notes.md").read_bytes() == b"# notes"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_workspace_files_rejects_oversized_file_and_cleans_partial_files(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
monkeypatch.setattr(svc, "MAX_WORKSPACE_UPLOAD_SIZE_BYTES", 5)
|
||||
user = _user()
|
||||
root = svc._workspace_root(user)
|
||||
uploads = [
|
||||
UploadFile(filename="small.txt", file=BytesIO(b"12345")),
|
||||
UploadFile(filename="large.txt", file=BytesIO(b"123456")),
|
||||
]
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await svc.upload_workspace_files(parent_path="/", files=uploads, current_user=user)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "100 MB" in exc_info.value.detail
|
||||
assert not (root / "small.txt").exists()
|
||||
assert not (root / "large.txt").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_workspace_files_rejects_more_than_limit(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace_paths.conf, "save_dir", str(tmp_path))
|
||||
user = _user()
|
||||
uploads = [
|
||||
UploadFile(filename=f"demo-{index}.txt", file=BytesIO(b"hello"))
|
||||
for index in range(svc.MAX_WORKSPACE_UPLOAD_FILES + 1)
|
||||
]
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await svc.upload_workspace_files(parent_path="/", files=uploads, current_user=user)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert f"一次最多上传 {svc.MAX_WORKSPACE_UPLOAD_FILES} 个文件" in exc_info.value.detail
|
||||
@@ -0,0 +1,336 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from yuxi.repositories.conversation_repository import (
|
||||
ConversationRepository,
|
||||
INVOCATION_CONVERSATION_SOURCES,
|
||||
MAX_CONVERSATION_TITLE_LENGTH,
|
||||
)
|
||||
from yuxi.storage.postgres.models_business import Base, Conversation, Message
|
||||
from yuxi.utils.datetime_utils import utc_now_naive
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
@pytest_asyncio.fixture()
|
||||
async def conversation_session():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
yield db
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def test_normalize_title_truncates_when_too_long():
|
||||
repo = ConversationRepository(None) # type: ignore[arg-type]
|
||||
raw = "a" * (MAX_CONVERSATION_TITLE_LENGTH + 50)
|
||||
|
||||
normalized = repo._normalize_title(raw)
|
||||
|
||||
assert normalized is not None
|
||||
assert len(normalized) == MAX_CONVERSATION_TITLE_LENGTH
|
||||
assert normalized == "a" * MAX_CONVERSATION_TITLE_LENGTH
|
||||
|
||||
|
||||
def test_normalize_title_trims_spaces():
|
||||
repo = ConversationRepository(None) # type: ignore[arg-type]
|
||||
|
||||
normalized = repo._normalize_title(" hello world ")
|
||||
|
||||
assert normalized == "hello world"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_conversations_excludes_invocation_sources(conversation_session):
|
||||
now = utc_now_naive()
|
||||
normal = Conversation(
|
||||
thread_id="thread-normal",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Normal",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
extra_metadata={},
|
||||
)
|
||||
agent_call = Conversation(
|
||||
thread_id="thread-call",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Agent Call Run",
|
||||
status="active",
|
||||
is_pinned=True,
|
||||
created_at=now,
|
||||
updated_at=now + timedelta(minutes=2),
|
||||
extra_metadata={"source": "agent_call"},
|
||||
)
|
||||
agent_eval = Conversation(
|
||||
thread_id="thread-eval",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Agent Evaluation Run",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now + timedelta(minutes=1),
|
||||
extra_metadata={"source": "agent_evaluation"},
|
||||
)
|
||||
conversation_session.add_all([normal, agent_call, agent_eval])
|
||||
await conversation_session.commit()
|
||||
|
||||
repo = ConversationRepository(conversation_session)
|
||||
items = await repo.list_conversations(
|
||||
uid="user-a",
|
||||
limit=20,
|
||||
offset=0,
|
||||
exclude_sources=INVOCATION_CONVERSATION_SOURCES,
|
||||
)
|
||||
|
||||
assert [item.thread_id for item in items] == ["thread-normal"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_conversations_by_message_content_filters_user_status_and_tool_messages(conversation_session):
|
||||
now = utc_now_naive()
|
||||
active = Conversation(
|
||||
thread_id="thread-active",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Active Thread",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
deleted = Conversation(
|
||||
thread_id="thread-deleted",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Deleted Thread",
|
||||
status="deleted",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
other_user = Conversation(
|
||||
thread_id="thread-other-user",
|
||||
uid="user-b",
|
||||
agent_id="agent-a",
|
||||
title="Other User Thread",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
tool_only = Conversation(
|
||||
thread_id="thread-tool-only",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Tool Only Thread",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
conversation_session.add_all([active, deleted, other_user, tool_only])
|
||||
await conversation_session.flush()
|
||||
conversation_session.add_all(
|
||||
[
|
||||
Message(
|
||||
conversation=active,
|
||||
role="assistant",
|
||||
content="大陆部署方案需要保留",
|
||||
message_type="text",
|
||||
created_at=now,
|
||||
),
|
||||
Message(
|
||||
conversation=deleted,
|
||||
role="assistant",
|
||||
content="大陆 deleted should not show",
|
||||
message_type="text",
|
||||
created_at=now,
|
||||
),
|
||||
Message(
|
||||
conversation=other_user,
|
||||
role="assistant",
|
||||
content="大陆 other user should not show",
|
||||
message_type="text",
|
||||
created_at=now,
|
||||
),
|
||||
Message(
|
||||
conversation=tool_only,
|
||||
role="tool",
|
||||
content="大陆 tool output should not show",
|
||||
message_type="tool_result",
|
||||
created_at=now,
|
||||
),
|
||||
]
|
||||
)
|
||||
await conversation_session.commit()
|
||||
|
||||
repo = ConversationRepository(conversation_session)
|
||||
items, has_more = await repo.search_conversations_by_message_content(
|
||||
uid="user-a",
|
||||
query="大陆",
|
||||
limit=20,
|
||||
offset=0,
|
||||
)
|
||||
|
||||
assert has_more is False
|
||||
assert [item["conversation"].thread_id for item in items] == ["thread-active"]
|
||||
assert items[0]["matched_count"] == 1
|
||||
assert items[0]["message_id"] is not None
|
||||
assert "大陆部署方案" in items[0]["snippets"][0]["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_conversations_by_message_content_excludes_invocation_sources(conversation_session):
|
||||
now = utc_now_naive()
|
||||
normal = Conversation(
|
||||
thread_id="thread-normal",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Normal",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
extra_metadata={},
|
||||
)
|
||||
agent_call = Conversation(
|
||||
thread_id="thread-call",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Agent Call Run",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now + timedelta(minutes=2),
|
||||
extra_metadata={"source": "agent_call"},
|
||||
)
|
||||
agent_eval = Conversation(
|
||||
thread_id="thread-eval",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Agent Evaluation Run",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now + timedelta(minutes=1),
|
||||
extra_metadata={"source": "agent_evaluation"},
|
||||
)
|
||||
conversation_session.add_all([normal, agent_call, agent_eval])
|
||||
await conversation_session.flush()
|
||||
conversation_session.add_all(
|
||||
[
|
||||
Message(conversation=normal, role="user", content="导航隐藏检查", message_type="text", created_at=now),
|
||||
Message(
|
||||
conversation=agent_call,
|
||||
role="user",
|
||||
content="导航隐藏检查 call",
|
||||
message_type="text",
|
||||
created_at=now,
|
||||
),
|
||||
Message(
|
||||
conversation=agent_eval,
|
||||
role="user",
|
||||
content="导航隐藏检查 eval",
|
||||
message_type="text",
|
||||
created_at=now,
|
||||
),
|
||||
]
|
||||
)
|
||||
await conversation_session.commit()
|
||||
|
||||
repo = ConversationRepository(conversation_session)
|
||||
items, has_more = await repo.search_conversations_by_message_content(
|
||||
uid="user-a",
|
||||
query="导航隐藏检查",
|
||||
limit=20,
|
||||
offset=0,
|
||||
exclude_sources=INVOCATION_CONVERSATION_SOURCES,
|
||||
)
|
||||
|
||||
assert has_more is False
|
||||
assert [item["conversation"].thread_id for item in items] == ["thread-normal"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_conversations_by_message_content_filters_agent_and_paginates(conversation_session):
|
||||
now = utc_now_naive()
|
||||
old = now - timedelta(days=1)
|
||||
first = Conversation(
|
||||
thread_id="thread-first",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="First",
|
||||
status="active",
|
||||
created_at=old,
|
||||
updated_at=old,
|
||||
)
|
||||
second = Conversation(
|
||||
thread_id="thread-second",
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
title="Second",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
other_agent = Conversation(
|
||||
thread_id="thread-other-agent",
|
||||
uid="user-a",
|
||||
agent_id="agent-b",
|
||||
title="Other Agent",
|
||||
status="active",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
conversation_session.add_all([first, second, other_agent])
|
||||
await conversation_session.flush()
|
||||
conversation_session.add_all(
|
||||
[
|
||||
Message(
|
||||
conversation=first,
|
||||
role="user",
|
||||
content="大陆关键词 old",
|
||||
message_type="text",
|
||||
created_at=old,
|
||||
),
|
||||
Message(
|
||||
conversation=second,
|
||||
role="assistant",
|
||||
content="大陆关键词 latest",
|
||||
message_type="text",
|
||||
created_at=now,
|
||||
),
|
||||
Message(
|
||||
conversation=other_agent,
|
||||
role="assistant",
|
||||
content="大陆关键词 other agent",
|
||||
message_type="text",
|
||||
created_at=now,
|
||||
),
|
||||
]
|
||||
)
|
||||
await conversation_session.commit()
|
||||
|
||||
repo = ConversationRepository(conversation_session)
|
||||
first_page, has_more = await repo.search_conversations_by_message_content(
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
query="大陆",
|
||||
limit=1,
|
||||
offset=0,
|
||||
)
|
||||
second_page, second_has_more = await repo.search_conversations_by_message_content(
|
||||
uid="user-a",
|
||||
agent_id="agent-a",
|
||||
query="大陆",
|
||||
limit=1,
|
||||
offset=1,
|
||||
)
|
||||
|
||||
assert has_more is True
|
||||
assert [item["conversation"].thread_id for item in first_page] == ["thread-second"]
|
||||
assert second_has_more is False
|
||||
assert [item["conversation"].thread_id for item in second_page] == ["thread-first"]
|
||||
@@ -0,0 +1,30 @@
|
||||
import pytest
|
||||
|
||||
from yuxi.storage.neo4j import Neo4jConnectionManager, safe_neo4j_label
|
||||
|
||||
|
||||
def test_storage_neo4j_exports_manager():
|
||||
import yuxi.storage.neo4j as neo4j_storage
|
||||
|
||||
assert neo4j_storage.Neo4jConnectionManager is Neo4jConnectionManager
|
||||
assert neo4j_storage.safe_neo4j_label is safe_neo4j_label
|
||||
|
||||
|
||||
@pytest.mark.parametrize("label", ["kb_test", "MilvusKB", "_internal_1"])
|
||||
def test_safe_neo4j_label_accepts_valid_labels(label):
|
||||
assert safe_neo4j_label(label) == label
|
||||
|
||||
|
||||
@pytest.mark.parametrize("label", ["", "1invalid", "has-dash", "has space", "中文"])
|
||||
def test_safe_neo4j_label_rejects_invalid_labels(label):
|
||||
with pytest.raises(ValueError, match="非法 Neo4j 标签"):
|
||||
safe_neo4j_label(label)
|
||||
|
||||
|
||||
def test_neo4j_connection_manager_skips_connection_in_lite_mode(monkeypatch):
|
||||
monkeypatch.setenv("LITE_MODE", "true")
|
||||
|
||||
manager = Neo4jConnectionManager()
|
||||
|
||||
assert manager.driver is None
|
||||
assert manager.status == "closed"
|
||||
@@ -0,0 +1,140 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.storage.postgres.manager import PostgresManager
|
||||
|
||||
|
||||
class _RecordingConnection:
|
||||
def __init__(self):
|
||||
self.statements: list[str] = []
|
||||
|
||||
async def execute(self, statement):
|
||||
self.statements.append(str(statement))
|
||||
|
||||
|
||||
class _RecordingBegin:
|
||||
def __init__(self, connection: _RecordingConnection):
|
||||
self.connection = connection
|
||||
|
||||
async def __aenter__(self):
|
||||
return self.connection
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class _RecordingEngine:
|
||||
def __init__(self, connection: _RecordingConnection):
|
||||
self.connection = connection
|
||||
|
||||
def begin(self):
|
||||
return _RecordingBegin(self.connection)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_business_schema_backfills_subagent_thread_columns_before_dropping_legacy_columns():
|
||||
manager = PostgresManager()
|
||||
original_initialized = manager._initialized
|
||||
original_engine = manager.async_engine
|
||||
connection = _RecordingConnection()
|
||||
|
||||
manager._initialized = True
|
||||
manager.async_engine = _RecordingEngine(connection)
|
||||
try:
|
||||
await manager.ensure_business_schema()
|
||||
finally:
|
||||
manager._initialized = original_initialized
|
||||
manager.async_engine = original_engine
|
||||
|
||||
statements = "\n".join(connection.statements)
|
||||
|
||||
assert "SET agent_slug = agent_id" in statements
|
||||
assert "SET conversation_thread_id = thread_id" in statements
|
||||
assert "SET created_by_run_id = COALESCE(parent_agent_run_id, parent_run_id)" in statements
|
||||
assert "SET subagent_slug = c.agent_id" in statements
|
||||
assert "SET created_by_run_id = created_by_parent_run_id::VARCHAR" in statements
|
||||
assert "ALTER COLUMN subagent_slug SET NOT NULL" in statements
|
||||
assert "ALTER COLUMN created_by_run_id SET NOT NULL" in statements
|
||||
assert statements.index("SET agent_slug = agent_id") < statements.index("DROP COLUMN IF EXISTS agent_id")
|
||||
assert statements.index("SET conversation_thread_id = thread_id") < statements.index(
|
||||
"DROP COLUMN IF EXISTS thread_id"
|
||||
)
|
||||
assert statements.index("COALESCE(parent_agent_run_id, parent_run_id)") < statements.index(
|
||||
"DROP COLUMN IF EXISTS parent_agent_run_id"
|
||||
)
|
||||
assert statements.index("created_by_parent_run_id") < statements.index(
|
||||
"DROP COLUMN IF EXISTS created_by_parent_run_id"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_business_schema_cleans_duplicate_active_agent_runs_before_unique_index():
|
||||
manager = PostgresManager()
|
||||
original_initialized = manager._initialized
|
||||
original_engine = manager.async_engine
|
||||
connection = _RecordingConnection()
|
||||
|
||||
manager._initialized = True
|
||||
manager.async_engine = _RecordingEngine(connection)
|
||||
try:
|
||||
await manager.ensure_business_schema()
|
||||
finally:
|
||||
manager._initialized = original_initialized
|
||||
manager.async_engine = original_engine
|
||||
|
||||
statements = "\n".join(connection.statements)
|
||||
|
||||
assert "WITH duplicated_active_runs AS" in statements
|
||||
assert "active_run_migration_conflict" in statements
|
||||
assert "CREATE UNIQUE INDEX IF NOT EXISTS uq_agent_runs_one_active_per_thread" in statements
|
||||
assert statements.index("WITH duplicated_active_runs AS") < statements.index(
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS uq_agent_runs_one_active_per_thread"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_business_schema_creates_user_config_table():
|
||||
manager = PostgresManager()
|
||||
original_initialized = manager._initialized
|
||||
original_engine = manager.async_engine
|
||||
connection = _RecordingConnection()
|
||||
|
||||
manager._initialized = True
|
||||
manager.async_engine = _RecordingEngine(connection)
|
||||
try:
|
||||
await manager.ensure_business_schema()
|
||||
finally:
|
||||
manager._initialized = original_initialized
|
||||
manager.async_engine = original_engine
|
||||
|
||||
statements = "\n".join(connection.statements)
|
||||
|
||||
assert "CREATE TABLE IF NOT EXISTS user_config" in statements
|
||||
assert "enable_memory BOOLEAN NOT NULL DEFAULT FALSE" in statements
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_business_schema_removes_unbound_api_keys_before_requiring_user_id():
|
||||
manager = PostgresManager()
|
||||
original_initialized = manager._initialized
|
||||
original_engine = manager.async_engine
|
||||
connection = _RecordingConnection()
|
||||
|
||||
manager._initialized = True
|
||||
manager.async_engine = _RecordingEngine(connection)
|
||||
try:
|
||||
await manager.ensure_business_schema()
|
||||
finally:
|
||||
manager._initialized = original_initialized
|
||||
manager.async_engine = original_engine
|
||||
|
||||
statements = "\n".join(connection.statements)
|
||||
|
||||
assert "UPDATE cli_auth_sessions" in statements
|
||||
assert "DELETE FROM api_keys WHERE user_id IS NULL" in statements
|
||||
assert "ALTER TABLE IF EXISTS api_keys ALTER COLUMN user_id SET NOT NULL" in statements
|
||||
assert statements.index("UPDATE cli_auth_sessions") < statements.index("DELETE FROM api_keys WHERE user_id IS NULL")
|
||||
assert statements.index("DELETE FROM api_keys WHERE user_id IS NULL") < statements.index(
|
||||
"ALTER TABLE IF EXISTS api_keys ALTER COLUMN user_id SET NOT NULL"
|
||||
)
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import yuxi.storage.redis.manager as redis_manager
|
||||
from yuxi.storage.redis import RedisConfig
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeSyncRedis:
|
||||
def __init__(self, *, ping_error: Exception | None = None):
|
||||
self.ping_error = ping_error
|
||||
self.ping_calls = 0
|
||||
self.closed = False
|
||||
|
||||
def ping(self):
|
||||
self.ping_calls += 1
|
||||
if self.ping_error is not None:
|
||||
raise self.ping_error
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _FakeAsyncRedis:
|
||||
def __init__(self, *, ping_error: Exception | None = None):
|
||||
self.ping_error = ping_error
|
||||
self.ping_calls = 0
|
||||
self.closed = False
|
||||
|
||||
async def ping(self):
|
||||
self.ping_calls += 1
|
||||
if self.ping_error is not None:
|
||||
raise self.ping_error
|
||||
|
||||
async def aclose(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
def test_create_sync_redis_client_uses_config_and_pings(monkeypatch: pytest.MonkeyPatch):
|
||||
redis = pytest.importorskip("redis")
|
||||
fake_client = _FakeSyncRedis()
|
||||
captured: dict = {}
|
||||
|
||||
def fake_from_url(url: str, **kwargs):
|
||||
captured["url"] = url
|
||||
captured["kwargs"] = kwargs
|
||||
return fake_client
|
||||
|
||||
monkeypatch.setattr(redis, "from_url", fake_from_url)
|
||||
|
||||
client = redis_manager.create_sync_redis_client(
|
||||
RedisConfig(
|
||||
url="redis://redis:6379/1",
|
||||
max_connections=7,
|
||||
decode_responses=True,
|
||||
socket_timeout=0.2,
|
||||
socket_connect_timeout=0.3,
|
||||
)
|
||||
)
|
||||
|
||||
assert client is fake_client
|
||||
assert fake_client.ping_calls == 1
|
||||
assert captured == {
|
||||
"url": "redis://redis:6379/1",
|
||||
"kwargs": {
|
||||
"decode_responses": True,
|
||||
"max_connections": 7,
|
||||
"socket_timeout": 0.2,
|
||||
"socket_connect_timeout": 0.3,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_create_sync_redis_client_closes_on_ping_failure(monkeypatch: pytest.MonkeyPatch):
|
||||
redis = pytest.importorskip("redis")
|
||||
fake_client = _FakeSyncRedis(ping_error=RuntimeError("redis unavailable"))
|
||||
monkeypatch.setattr(redis, "from_url", lambda *args, **kwargs: fake_client)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
redis_manager.create_sync_redis_client(RedisConfig(url="redis://:secret@redis:6379/0"))
|
||||
|
||||
assert "secret" not in str(exc_info.value)
|
||||
assert "Redis connection failed" in str(exc_info.value)
|
||||
assert fake_client.closed is True
|
||||
|
||||
|
||||
def test_sync_redis_client_closes_after_context(monkeypatch: pytest.MonkeyPatch):
|
||||
fake_client = _FakeSyncRedis()
|
||||
monkeypatch.setattr(redis_manager, "create_sync_redis_client", lambda *args, **kwargs: fake_client)
|
||||
|
||||
with redis_manager.sync_redis_client(RedisConfig(url="redis://redis:6379/1")) as client:
|
||||
assert client is fake_client
|
||||
|
||||
assert fake_client.closed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_async_redis_client_closes_on_ping_failure(monkeypatch: pytest.MonkeyPatch):
|
||||
redis_asyncio = pytest.importorskip("redis.asyncio")
|
||||
fake_client = _FakeAsyncRedis(ping_error=RuntimeError("redis unavailable"))
|
||||
monkeypatch.setattr(redis_asyncio.Redis, "from_url", staticmethod(lambda *args, **kwargs: fake_client))
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
await redis_manager.create_async_redis_client(RedisConfig(url="redis://:secret@redis:6379/0"))
|
||||
|
||||
assert "secret" not in str(exc_info.value)
|
||||
assert "Redis connection failed" in str(exc_info.value)
|
||||
assert fake_client.closed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_async_redis_client_caches_and_closes_client(monkeypatch: pytest.MonkeyPatch):
|
||||
fake_client = _FakeAsyncRedis()
|
||||
create_calls = 0
|
||||
|
||||
async def fake_create_async_client(config: RedisConfig | None = None):
|
||||
nonlocal create_calls
|
||||
assert config and config.url == "redis://redis:6379/2"
|
||||
create_calls += 1
|
||||
return fake_client
|
||||
|
||||
monkeypatch.setattr(redis_manager, "_async_redis_client", None)
|
||||
monkeypatch.setattr(redis_manager, "_async_redis_lock", None)
|
||||
monkeypatch.setattr(redis_manager, "create_async_redis_client", fake_create_async_client)
|
||||
|
||||
client_1 = await redis_manager.get_async_redis_client(RedisConfig(url="redis://redis:6379/2"))
|
||||
client_2 = await redis_manager.get_async_redis_client(RedisConfig(url="redis://redis:6379/2"))
|
||||
await redis_manager.close_async_redis_client()
|
||||
|
||||
assert client_1 is fake_client
|
||||
assert client_2 is fake_client
|
||||
assert create_calls == 1
|
||||
assert fake_client.closed is True
|
||||
@@ -0,0 +1,127 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from datetime import timedelta
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from yuxi.utils.datetime_utils import utc_now
|
||||
|
||||
from yuxi.utils.auth_utils import JWT_ALGORITHM, JWT_AUDIENCE, AuthUtils
|
||||
|
||||
|
||||
def test_generate_api_key_returns_secret_hash_and_prefix():
|
||||
full_key, key_hash, key_prefix = AuthUtils.generate_api_key()
|
||||
|
||||
assert full_key.startswith("yxkey_")
|
||||
assert len(full_key) == len("yxkey_") + 48
|
||||
assert key_prefix == full_key[:12]
|
||||
assert key_hash == hashlib.sha256(full_key.encode()).hexdigest()
|
||||
|
||||
|
||||
def test_hash_password_uses_argon2():
|
||||
hashed = AuthUtils.hash_password("secret-password")
|
||||
|
||||
assert hashed.startswith("$argon2")
|
||||
assert AuthUtils.verify_password(hashed, "secret-password") is True
|
||||
assert AuthUtils.verify_password(hashed, "wrong-password") is False
|
||||
|
||||
|
||||
def test_access_token_contains_instance_claims(monkeypatch):
|
||||
monkeypatch.setenv("JWT_SECRET_KEY", "test-secret-key-with-enough-randomness")
|
||||
monkeypatch.setenv("YUXI_INSTANCE_ID", "pytest-instance")
|
||||
|
||||
token = AuthUtils.create_access_token({"sub": "1"})
|
||||
payload = AuthUtils.verify_access_token(token)
|
||||
|
||||
assert payload["sub"] == "1"
|
||||
assert payload["iss"] == "yuxi-know:pytest-instance"
|
||||
assert payload["aud"] == JWT_AUDIENCE
|
||||
|
||||
|
||||
def test_access_token_auto_generates_dev_secret(monkeypatch):
|
||||
monkeypatch.setenv("YUXI_ENV", "development")
|
||||
monkeypatch.delenv("JWT_SECRET_KEY", raising=False)
|
||||
monkeypatch.setenv("YUXI_INSTANCE_ID", "pytest-instance")
|
||||
|
||||
token = AuthUtils.create_access_token({"sub": "1"})
|
||||
|
||||
assert AuthUtils.verify_access_token(token)["sub"] == "1"
|
||||
assert len(os.environ["JWT_SECRET_KEY"]) == 64
|
||||
|
||||
|
||||
def test_access_token_requires_configured_secret_in_production(monkeypatch):
|
||||
monkeypatch.setenv("YUXI_ENV", "production")
|
||||
monkeypatch.delenv("JWT_SECRET_KEY", raising=False)
|
||||
monkeypatch.setenv("YUXI_INSTANCE_ID", "pytest-instance")
|
||||
|
||||
with pytest.raises(ValueError, match="JWT_SECRET_KEY"):
|
||||
AuthUtils.create_access_token({"sub": "1"})
|
||||
|
||||
|
||||
def test_access_token_rejects_public_default_secret_in_production(monkeypatch):
|
||||
monkeypatch.setenv("YUXI_ENV", "production")
|
||||
monkeypatch.setenv("JWT_SECRET_KEY", "yuxi_know_secure_key")
|
||||
monkeypatch.setenv("YUXI_INSTANCE_ID", "pytest-instance")
|
||||
|
||||
with pytest.raises(ValueError, match="公开默认密钥"):
|
||||
AuthUtils.create_access_token({"sub": "1"})
|
||||
|
||||
|
||||
def test_access_token_auto_generates_dev_instance_id(monkeypatch):
|
||||
monkeypatch.setenv("YUXI_ENV", "development")
|
||||
monkeypatch.setenv("JWT_SECRET_KEY", "test-secret-key-with-enough-randomness")
|
||||
monkeypatch.delenv("YUXI_INSTANCE_ID", raising=False)
|
||||
|
||||
token = AuthUtils.create_access_token({"sub": "1"})
|
||||
|
||||
assert AuthUtils.verify_access_token(token)["iss"].startswith("yuxi-know:instance-")
|
||||
assert os.environ["YUXI_INSTANCE_ID"].startswith("instance-")
|
||||
|
||||
|
||||
def test_access_token_requires_instance_id_in_production(monkeypatch):
|
||||
monkeypatch.setenv("YUXI_ENV", "production")
|
||||
monkeypatch.setenv("JWT_SECRET_KEY", "test-secret-key-with-enough-randomness")
|
||||
monkeypatch.delenv("YUXI_INSTANCE_ID", raising=False)
|
||||
|
||||
with pytest.raises(ValueError, match="YUXI_INSTANCE_ID"):
|
||||
AuthUtils.create_access_token({"sub": "1"})
|
||||
|
||||
|
||||
def test_verify_access_token_rejects_wrong_issuer(monkeypatch):
|
||||
monkeypatch.setenv("JWT_SECRET_KEY", "test-secret-key-with-enough-randomness")
|
||||
monkeypatch.setenv("YUXI_INSTANCE_ID", "pytest-instance")
|
||||
token = jwt.encode(
|
||||
{"sub": "1", "exp": utc_now() + timedelta(minutes=5), "iss": "yuxi-know:other", "aud": JWT_AUDIENCE},
|
||||
"test-secret-key-with-enough-randomness",
|
||||
algorithm=JWT_ALGORITHM,
|
||||
)
|
||||
|
||||
assert AuthUtils.decode_token(token) is None
|
||||
|
||||
|
||||
def test_verify_access_token_rejects_wrong_audience(monkeypatch):
|
||||
monkeypatch.setenv("JWT_SECRET_KEY", "test-secret-key-with-enough-randomness")
|
||||
monkeypatch.setenv("YUXI_INSTANCE_ID", "pytest-instance")
|
||||
token = jwt.encode(
|
||||
{"sub": "1", "exp": utc_now() + timedelta(minutes=5), "iss": "yuxi-know:pytest-instance", "aud": "other-api"},
|
||||
"test-secret-key-with-enough-randomness",
|
||||
algorithm=JWT_ALGORITHM,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="无效的令牌"):
|
||||
AuthUtils.verify_access_token(token)
|
||||
|
||||
|
||||
def test_verify_access_token_requires_claims(monkeypatch):
|
||||
monkeypatch.setenv("JWT_SECRET_KEY", "test-secret-key-with-enough-randomness")
|
||||
monkeypatch.setenv("YUXI_INSTANCE_ID", "pytest-instance")
|
||||
token = jwt.encode(
|
||||
{"sub": "1", "exp": utc_now() + timedelta(minutes=5)},
|
||||
"test-secret-key-with-enough-randomness",
|
||||
algorithm=JWT_ALGORITHM,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="无效的令牌"):
|
||||
AuthUtils.verify_access_token(token)
|
||||
@@ -0,0 +1,201 @@
|
||||
"""测试分块 token 上限保护:general parser 对超长 chunk 的硬切分。
|
||||
|
||||
nlp.py / general.py 只依赖 re 和标准库,用 sys.modules 占位绕过 yuxi 包的
|
||||
重依赖链(langchain / pydantic / .env 配置等),实现纯单元测试。
|
||||
跑完后清理 sys.modules,避免污染其他测试。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_PKG = Path(__file__).resolve().parents[2] / "package"
|
||||
|
||||
_STUB_NAMES = [
|
||||
"yuxi",
|
||||
"yuxi.knowledge",
|
||||
"yuxi.knowledge.chunking",
|
||||
"yuxi.knowledge.chunking.ragflow_like",
|
||||
"yuxi.knowledge.chunking.ragflow_like.parsers",
|
||||
"yuxi.knowledge.chunking.ragflow_like.nlp",
|
||||
"yuxi.knowledge.chunking.ragflow_like.parsers.general",
|
||||
]
|
||||
|
||||
# 由 _isolated_modules fixture 在运行时注入
|
||||
nlp = None # type: ignore[assignment]
|
||||
general = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _isolated_modules():
|
||||
"""在模块级加载 nlp/general,跑完后清理 sys.modules 避免污染其他测试。"""
|
||||
saved = {name: sys.modules.get(name) for name in _STUB_NAMES}
|
||||
|
||||
for name in _STUB_NAMES[:5]:
|
||||
sys.modules.setdefault(name, types.ModuleType(name))
|
||||
|
||||
def _load(name: str, rel: str):
|
||||
spec = importlib.util.spec_from_file_location(name, _PKG / rel)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = mod
|
||||
spec.loader.exec_module(mod) # type: ignore[union-attr]
|
||||
return mod
|
||||
|
||||
_nlp = _load(
|
||||
"yuxi.knowledge.chunking.ragflow_like.nlp",
|
||||
"yuxi/knowledge/chunking/ragflow_like/nlp.py",
|
||||
)
|
||||
sys.modules["yuxi.knowledge.chunking.ragflow_like"].nlp = _nlp # type: ignore[attr-defined]
|
||||
|
||||
_general = _load(
|
||||
"yuxi.knowledge.chunking.ragflow_like.parsers.general",
|
||||
"yuxi/knowledge/chunking/ragflow_like/parsers/general.py",
|
||||
)
|
||||
|
||||
# 注入模块级变量供测试用例访问
|
||||
global nlp, general # noqa: PLW0603
|
||||
nlp = _nlp
|
||||
general = _general
|
||||
|
||||
yield
|
||||
|
||||
# 清理:恢复原始状态
|
||||
for name in _STUB_NAMES:
|
||||
if saved[name] is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = saved[name]
|
||||
|
||||
|
||||
# ── nlp.hard_split_by_token_limit ──────────────────────────────────
|
||||
|
||||
|
||||
class TestHardSplitByTokenLimit:
|
||||
def test_short_text_unchanged(self):
|
||||
text = "这是一段短文本"
|
||||
result = nlp.hard_split_by_token_limit(text, 512)
|
||||
assert result == [text]
|
||||
|
||||
def test_splits_long_chinese_text(self):
|
||||
text = "测试内容" * 300 # ~600 CJK tokens
|
||||
result = nlp.hard_split_by_token_limit(text, 512)
|
||||
assert len(result) > 1
|
||||
for chunk in result:
|
||||
assert nlp.count_tokens(chunk) <= 512
|
||||
|
||||
def test_optional_hard_limit_keeps_slightly_oversized_text(self):
|
||||
text = "内容" * 300 # 600 CJK tokens
|
||||
result = nlp.hard_split_by_token_limit(text, 512, hard_limit_token_num=768)
|
||||
assert result == [text]
|
||||
|
||||
def test_optional_hard_limit_merges_short_tail(self):
|
||||
text = "内容" * 635 # 1270 CJK tokens -> 512 + 512 + 246
|
||||
result = nlp.hard_split_by_token_limit(text, 512, hard_limit_token_num=768)
|
||||
assert len(result) == 2
|
||||
assert [nlp.count_tokens(chunk) for chunk in result] == [512, 758]
|
||||
|
||||
def test_splits_long_english_text(self):
|
||||
text = "hello world " * 1000 # ~2000 word tokens
|
||||
result = nlp.hard_split_by_token_limit(text, 512)
|
||||
assert len(result) > 1
|
||||
for chunk in result:
|
||||
assert nlp.count_tokens(chunk) <= 512
|
||||
|
||||
def test_empty_text_returns_empty(self):
|
||||
assert nlp.hard_split_by_token_limit("", 512) == []
|
||||
|
||||
def test_whitespace_only_returns_empty(self):
|
||||
assert nlp.hard_split_by_token_limit(" \n\t ", 512) == []
|
||||
|
||||
def test_zero_limit_floors_to_one(self):
|
||||
text = "a b c" # 3 个独立 token(单词)
|
||||
result = nlp.hard_split_by_token_limit(text, 0)
|
||||
# max_tokens = max(0, 1) = 1, 每个 token 单独一个 chunk
|
||||
assert len(result) == 3
|
||||
|
||||
def test_punctuation_only_text(self):
|
||||
text = ",。!?"
|
||||
result = nlp.hard_split_by_token_limit(text, 512)
|
||||
assert result == [",。!?"]
|
||||
|
||||
|
||||
# ── general._ensure_chunk_token_limit ──────────────────────────────
|
||||
|
||||
|
||||
class TestEnsureChunkTokenLimit:
|
||||
def test_all_chunks_within_limit_pass_through(self):
|
||||
chunks = ["短文本一", "短文本二", "短文本三"]
|
||||
result = general._ensure_chunk_token_limit(chunks, 512)
|
||||
assert result == ["短文本一", "短文本二", "短文本三"]
|
||||
|
||||
def test_slightly_oversized_chunk_passes_through(self):
|
||||
long_text = "内容" * 300 # ~600 CJK tokens
|
||||
chunks = ["短文本", long_text, "短文本二"]
|
||||
result = general._ensure_chunk_token_limit(chunks, 512)
|
||||
assert result == ["短文本", long_text, "短文本二"]
|
||||
|
||||
def test_oversized_chunk_gets_split_with_merged_tail(self):
|
||||
long_text = "内容" * 635 # 1270 CJK tokens -> 512 + 758
|
||||
chunks = ["短文本", long_text, "短文本二"]
|
||||
result = general._ensure_chunk_token_limit(chunks, 512)
|
||||
assert result[0] == "短文本"
|
||||
assert result[-1] == "短文本二"
|
||||
middle_chunks = result[1:-1]
|
||||
assert len(middle_chunks) == 2
|
||||
assert [nlp.count_tokens(chunk) for chunk in middle_chunks] == [512, 758]
|
||||
|
||||
def test_empty_chunks_filtered(self):
|
||||
chunks = ["有效文本", "", " ", "另一段"]
|
||||
result = general._ensure_chunk_token_limit(chunks, 512)
|
||||
assert result == ["有效文本", "另一段"]
|
||||
|
||||
def test_zero_limit_returns_stripped(self):
|
||||
chunks = [" 文本一 ", "文本二"]
|
||||
result = general._ensure_chunk_token_limit(chunks, 0)
|
||||
assert result == ["文本一", "文本二"]
|
||||
|
||||
|
||||
# ── general.chunk_markdown 集成 ────────────────────────────────────
|
||||
|
||||
|
||||
class TestGeneralChunkMarkdown:
|
||||
def test_normal_document_chunks_within_limit(self):
|
||||
doc = "# 标题\n\n第一段内容\n\n第二段内容\n\n第三段内容"
|
||||
chunks = general.chunk_markdown(doc, {"chunk_token_num": 512})
|
||||
assert len(chunks) > 0
|
||||
for chunk in chunks:
|
||||
assert nlp.count_tokens(chunk) <= 512
|
||||
|
||||
def test_oversized_single_line_gets_split(self):
|
||||
long_line = "运维知识" * 800 # ~3200 CJK tokens
|
||||
doc = f"# 运维知识库\n\n{long_line}"
|
||||
chunks = general.chunk_markdown(doc, {"chunk_token_num": 512})
|
||||
assert len(chunks) > 1
|
||||
for chunk in chunks:
|
||||
assert nlp.count_tokens(chunk) <= 768
|
||||
|
||||
def test_empty_document_returns_empty(self):
|
||||
assert general.chunk_markdown("", {"chunk_token_num": 512}) == []
|
||||
|
||||
def test_default_config_uses_512(self):
|
||||
doc = "测试\n" * 200
|
||||
chunks = general.chunk_markdown(doc)
|
||||
for chunk in chunks:
|
||||
assert nlp.count_tokens(chunk) <= 512
|
||||
|
||||
|
||||
# ── laws parser 回归 ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestLawsParserRegression:
|
||||
"""验证 nlp.hard_split_by_token_limit 可被 laws parser 正常调用。"""
|
||||
|
||||
def test_hard_split_produces_same_result(self):
|
||||
text = "法规内容" * 300
|
||||
result = nlp.hard_split_by_token_limit(text, 512)
|
||||
assert len(result) > 1
|
||||
for chunk in result:
|
||||
assert nlp.count_tokens(chunk) <= 512
|
||||
@@ -0,0 +1,12 @@
|
||||
import sys
|
||||
|
||||
|
||||
def test_import_yuxi_does_not_eagerly_import_knowledge(monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
|
||||
sys.modules.pop("yuxi", None)
|
||||
sys.modules.pop("yuxi.knowledge", None)
|
||||
|
||||
import yuxi
|
||||
|
||||
assert yuxi.get_version() == yuxi.__version__
|
||||
assert "yuxi.knowledge" not in sys.modules
|
||||
@@ -0,0 +1,351 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("OPENAI_API_KEY", "test-key")
|
||||
os.environ.setdefault("SAVE_DIR", os.path.join(os.environ.get("CLAUDE_JOB_DIR", tempfile.gettempdir()), "yuxi-test-saves"))
|
||||
|
||||
from yuxi.services import conversation_service as service
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class FakeUpload:
|
||||
def __init__(self, filename: str, content: bytes, content_type: str | None = None):
|
||||
self.filename = filename
|
||||
self.content_type = content_type
|
||||
self._content = content
|
||||
self._offset = 0
|
||||
|
||||
async def seek(self, offset: int) -> None:
|
||||
self._offset = offset
|
||||
|
||||
async def read(self, size: int = -1) -> bytes:
|
||||
if self._offset >= len(self._content):
|
||||
return b""
|
||||
end = len(self._content) if size < 0 else min(len(self._content), self._offset + size)
|
||||
chunk = self._content[self._offset : end]
|
||||
self._offset = end
|
||||
return chunk
|
||||
|
||||
|
||||
class FakeMinioClient:
|
||||
KB_BUCKETS = {"documents": "knowledgebases"}
|
||||
|
||||
def __init__(self):
|
||||
self.objects: dict[tuple[str, str], bytes] = {}
|
||||
self.uploads: list[dict] = []
|
||||
|
||||
async def aupload_file(self, bucket_name: str, object_name: str, data: bytes, content_type: str | None = None):
|
||||
self.objects[(bucket_name, object_name)] = data
|
||||
self.uploads.append(
|
||||
{
|
||||
"bucket_name": bucket_name,
|
||||
"object_name": object_name,
|
||||
"data": data,
|
||||
"content_type": content_type,
|
||||
}
|
||||
)
|
||||
return SimpleNamespace(
|
||||
bucket_name=bucket_name,
|
||||
object_name=object_name,
|
||||
url=f"http://minio:9000/{bucket_name}/{object_name}",
|
||||
)
|
||||
|
||||
async def adownload_file(self, bucket_name: str, object_name: str) -> bytes:
|
||||
try:
|
||||
return self.objects[(bucket_name, object_name)]
|
||||
except KeyError as exc:
|
||||
raise service.StorageError("missing object") from exc
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeConversation:
|
||||
id: int = 1
|
||||
uid: str = "user-1"
|
||||
agent_id: str = "agent-1"
|
||||
status: str = "active"
|
||||
extra_metadata: dict | None = None
|
||||
|
||||
|
||||
class FakeConversationRepository:
|
||||
def __init__(self, db):
|
||||
self.conversation = FakeConversation()
|
||||
self.attachments: list[dict] = []
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id: str):
|
||||
return self.conversation
|
||||
|
||||
async def add_attachment(self, conversation_id: int, attachment_info: dict):
|
||||
self.attachments.append(attachment_info)
|
||||
return attachment_info
|
||||
|
||||
async def add_attachments(self, conversation_id: int, attachment_infos: list[dict]):
|
||||
self.attachments.extend(attachment_infos)
|
||||
return attachment_infos
|
||||
|
||||
async def get_attachments(self, conversation_id: int):
|
||||
return list(self.attachments)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_tmp_attachment_writes_user_scoped_minio_object(monkeypatch):
|
||||
fake_minio = FakeMinioClient()
|
||||
monkeypatch.setattr(service, "get_minio_client", lambda: fake_minio)
|
||||
|
||||
response = await service.upload_tmp_attachment_view(
|
||||
file=FakeUpload("demo.pdf", b"pdf-bytes", "application/pdf"),
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
assert response["bucket_name"] == "knowledgebases"
|
||||
assert response["object_name"].startswith("tmp/chat_attachments/user-1/")
|
||||
assert response["parse_methods"][0] == "disable"
|
||||
assert fake_minio.objects[("knowledgebases", response["object_name"])] == b"pdf-bytes"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_tmp_attachment_uses_selected_method_and_uploads_markdown(monkeypatch):
|
||||
fake_minio = FakeMinioClient()
|
||||
object_name = "tmp/chat_attachments/user-1/tmp-1/original/demo.pdf"
|
||||
fake_minio.objects[("knowledgebases", object_name)] = b"pdf-bytes"
|
||||
monkeypatch.setattr(service, "get_minio_client", lambda: fake_minio)
|
||||
|
||||
parse_calls = []
|
||||
|
||||
async def fake_parse(source: str, params: dict | None = None) -> str:
|
||||
parse_calls.append({"source": source, "params": params})
|
||||
return "# parsed"
|
||||
|
||||
monkeypatch.setattr(service.Parser, "aparse", staticmethod(fake_parse))
|
||||
|
||||
response = await service.parse_tmp_attachment_view(
|
||||
object_name=object_name,
|
||||
file_name="demo.pdf",
|
||||
parse_method="disable",
|
||||
bucket_name="knowledgebases",
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
assert parse_calls == [
|
||||
{
|
||||
"source": f"minio://knowledgebases/{object_name}",
|
||||
"params": {"ocr_engine": "disable"},
|
||||
}
|
||||
]
|
||||
assert response["parsed_object_name"] == "tmp/chat_attachments/user-1/tmp-1/parsed/demo.md"
|
||||
assert fake_minio.objects[("knowledgebases", response["parsed_object_name"])] == b"# parsed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm_tmp_thread_attachments_materializes_original_and_parsed_files(monkeypatch, tmp_path: Path):
|
||||
fake_minio = FakeMinioClient()
|
||||
original_object = "tmp/chat_attachments/user-1/tmp-1/original/demo.pdf"
|
||||
parsed_object = "tmp/chat_attachments/user-1/tmp-1/parsed/demo.md"
|
||||
fake_minio.objects[("knowledgebases", original_object)] = b"pdf-bytes"
|
||||
fake_minio.objects[("knowledgebases", parsed_object)] = b"# parsed"
|
||||
fake_repo = FakeConversationRepository(db=None)
|
||||
|
||||
monkeypatch.setattr(service, "get_minio_client", lambda: fake_minio)
|
||||
monkeypatch.setattr(service, "ConversationRepository", lambda db: fake_repo)
|
||||
monkeypatch.setattr(service.app_config, "save_dir", str(tmp_path))
|
||||
|
||||
def fake_uploads_dir(thread_id: str) -> Path:
|
||||
path = tmp_path / "threads" / thread_id / "user-data" / "uploads"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
monkeypatch.setattr(service, "ensure_thread_dirs", lambda thread_id, user_id: fake_uploads_dir(thread_id))
|
||||
monkeypatch.setattr(service, "sandbox_uploads_dir", fake_uploads_dir)
|
||||
|
||||
async def noop_sync(**kwargs):
|
||||
return None
|
||||
|
||||
async def noop_invalidate(thread_id: str):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(service, "_sync_thread_upload_state", noop_sync)
|
||||
monkeypatch.setattr(service, "invalidate_mention_cache", noop_invalidate)
|
||||
|
||||
response = await service.confirm_tmp_thread_attachments_view(
|
||||
thread_id="thread-1",
|
||||
attachments=[
|
||||
{
|
||||
"file_name": "demo.pdf",
|
||||
"file_type": "application/pdf",
|
||||
"bucket_name": "knowledgebases",
|
||||
"object_name": original_object,
|
||||
"parsed_object_name": parsed_object,
|
||||
"truncated": False,
|
||||
}
|
||||
],
|
||||
db=None,
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
[attachment] = response["attachments"]
|
||||
assert attachment["status"] == "parsed"
|
||||
original_name = Path(attachment["original_path"]).name
|
||||
markdown_name = Path(attachment["path"]).name
|
||||
assert original_name.endswith("_demo.pdf")
|
||||
assert markdown_name.endswith("_demo.md")
|
||||
assert (tmp_path / "threads" / "thread-1" / "user-data" / "uploads" / original_name).read_bytes() == b"pdf-bytes"
|
||||
assert (
|
||||
tmp_path / "threads" / "thread-1" / "user-data" / "uploads" / "attachments" / markdown_name
|
||||
).read_text(encoding="utf-8") == "# parsed"
|
||||
assert Path(fake_repo.attachments[0]["original_path"]).name == original_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_tmp_attachment_uses_object_name_for_type_validation(monkeypatch):
|
||||
fake_minio = FakeMinioClient()
|
||||
object_name = "tmp/chat_attachments/user-1/tmp-1/original/demo.docx"
|
||||
fake_minio.objects[("knowledgebases", object_name)] = b"docx-bytes"
|
||||
monkeypatch.setattr(service, "get_minio_client", lambda: fake_minio)
|
||||
|
||||
with pytest.raises(service.HTTPException) as exc_info:
|
||||
await service.parse_tmp_attachment_view(
|
||||
object_name=object_name,
|
||||
file_name="demo.pdf",
|
||||
parse_method="disable",
|
||||
bucket_name="knowledgebases",
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "PDF 和图片" in exc_info.value.detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_tmp_attachment_handles_url_metacharacters(monkeypatch):
|
||||
fake_minio = FakeMinioClient()
|
||||
object_name = "tmp/chat_attachments/user-1/tmp-1/original/q1?.pdf"
|
||||
fake_minio.objects[("knowledgebases", object_name)] = b"pdf-bytes"
|
||||
monkeypatch.setattr(service, "get_minio_client", lambda: fake_minio)
|
||||
|
||||
parse_calls = []
|
||||
|
||||
async def fake_parse(source: str, params: dict | None = None) -> str:
|
||||
parse_calls.append(source)
|
||||
return "# parsed"
|
||||
|
||||
monkeypatch.setattr(service.Parser, "aparse", staticmethod(fake_parse))
|
||||
|
||||
response = await service.parse_tmp_attachment_view(
|
||||
object_name=object_name,
|
||||
file_name="ignored.pdf",
|
||||
parse_method="disable",
|
||||
bucket_name="knowledgebases",
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
assert parse_calls == ["minio://knowledgebases/tmp/chat_attachments/user-1/tmp-1/original/q1%3F.pdf"]
|
||||
assert response["parsed_object_name"] == "tmp/chat_attachments/user-1/tmp-1/parsed/q1?.md"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm_tmp_thread_attachments_rejects_non_parsed_object(monkeypatch):
|
||||
fake_minio = FakeMinioClient()
|
||||
original_object = "tmp/chat_attachments/user-1/tmp-1/original/demo.pdf"
|
||||
fake_minio.objects[("knowledgebases", original_object)] = b"pdf-bytes"
|
||||
fake_repo = FakeConversationRepository(db=None)
|
||||
|
||||
monkeypatch.setattr(service, "get_minio_client", lambda: fake_minio)
|
||||
monkeypatch.setattr(service, "ConversationRepository", lambda db: fake_repo)
|
||||
|
||||
with pytest.raises(service.HTTPException) as exc_info:
|
||||
await service.confirm_tmp_thread_attachments_view(
|
||||
thread_id="thread-1",
|
||||
attachments=[
|
||||
{
|
||||
"file_name": "demo.pdf",
|
||||
"file_type": "application/pdf",
|
||||
"bucket_name": "knowledgebases",
|
||||
"object_name": original_object,
|
||||
"parsed_object_name": original_object,
|
||||
}
|
||||
],
|
||||
db=None,
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert fake_repo.attachments == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm_tmp_thread_attachments_validates_batch_before_commit(monkeypatch):
|
||||
fake_minio = FakeMinioClient()
|
||||
valid_object = "tmp/chat_attachments/user-1/tmp-1/original/valid.pdf"
|
||||
missing_object = "tmp/chat_attachments/user-1/tmp-2/original/missing.pdf"
|
||||
fake_minio.objects[("knowledgebases", valid_object)] = b"pdf-bytes"
|
||||
fake_repo = FakeConversationRepository(db=None)
|
||||
|
||||
monkeypatch.setattr(service, "get_minio_client", lambda: fake_minio)
|
||||
monkeypatch.setattr(service, "ConversationRepository", lambda db: fake_repo)
|
||||
|
||||
with pytest.raises(service.HTTPException) as exc_info:
|
||||
await service.confirm_tmp_thread_attachments_view(
|
||||
thread_id="thread-1",
|
||||
attachments=[
|
||||
{"file_name": "valid.pdf", "bucket_name": "knowledgebases", "object_name": valid_object},
|
||||
{"file_name": "missing.pdf", "bucket_name": "knowledgebases", "object_name": missing_object},
|
||||
],
|
||||
db=None,
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert fake_repo.attachments == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm_tmp_thread_attachments_keeps_duplicate_names_separate(monkeypatch, tmp_path: Path):
|
||||
fake_minio = FakeMinioClient()
|
||||
first_object = "tmp/chat_attachments/user-1/tmp-1/original/report.pdf"
|
||||
second_object = "tmp/chat_attachments/user-1/tmp-2/original/report.pdf"
|
||||
fake_minio.objects[("knowledgebases", first_object)] = b"first"
|
||||
fake_minio.objects[("knowledgebases", second_object)] = b"second"
|
||||
fake_repo = FakeConversationRepository(db=None)
|
||||
|
||||
monkeypatch.setattr(service, "get_minio_client", lambda: fake_minio)
|
||||
monkeypatch.setattr(service, "ConversationRepository", lambda db: fake_repo)
|
||||
monkeypatch.setattr(service.app_config, "save_dir", str(tmp_path))
|
||||
|
||||
def fake_uploads_dir(thread_id: str) -> Path:
|
||||
path = tmp_path / "threads" / thread_id / "user-data" / "uploads"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
monkeypatch.setattr(service, "ensure_thread_dirs", lambda thread_id, uid: fake_uploads_dir(thread_id))
|
||||
monkeypatch.setattr(service, "sandbox_uploads_dir", fake_uploads_dir)
|
||||
|
||||
async def noop_sync(**kwargs):
|
||||
return None
|
||||
|
||||
async def noop_invalidate(thread_id: str):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(service, "_sync_thread_upload_state", noop_sync)
|
||||
monkeypatch.setattr(service, "invalidate_mention_cache", noop_invalidate)
|
||||
|
||||
response = await service.confirm_tmp_thread_attachments_view(
|
||||
thread_id="thread-1",
|
||||
attachments=[
|
||||
{"file_name": "report.pdf", "bucket_name": "knowledgebases", "object_name": first_object},
|
||||
{"file_name": "report.pdf", "bucket_name": "knowledgebases", "object_name": second_object},
|
||||
],
|
||||
db=None,
|
||||
current_uid="user-1",
|
||||
)
|
||||
|
||||
first, second = response["attachments"]
|
||||
assert first["original_path"] != second["original_path"]
|
||||
assert (tmp_path / "threads" / "thread-1" / "user-data" / "uploads" / Path(first["original_path"]).name).read_bytes() == b"first"
|
||||
assert (tmp_path / "threads" / "thread-1" / "user-data" / "uploads" / Path(second["original_path"]).name).read_bytes() == b"second"
|
||||
@@ -0,0 +1,302 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.skills import service as skill_service
|
||||
from yuxi.agents.toolkits.buildin import install_skill as exported_install_skill
|
||||
|
||||
install_skill_module = importlib.import_module("yuxi.agents.toolkits.buildin.install_skill")
|
||||
|
||||
|
||||
class _AsyncSessionContext:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def __aenter__(self):
|
||||
return self.db
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
def _runtime(**context_values):
|
||||
return SimpleNamespace(context=SimpleNamespace(**context_values))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_skill_from_sandbox_installs_as_current_user_private_skill(monkeypatch, tmp_path: Path):
|
||||
assert exported_install_skill.name == "install_skill"
|
||||
|
||||
calls = {}
|
||||
db = SimpleNamespace()
|
||||
source_dir = tmp_path / "demo-skill"
|
||||
|
||||
def prepare_skill_from_sandbox(source, thread_id, uid, staging_root):
|
||||
calls["prepare"] = {
|
||||
"source": source,
|
||||
"thread_id": thread_id,
|
||||
"uid": uid,
|
||||
"staging_root": staging_root,
|
||||
}
|
||||
return source_dir, "demo-skill"
|
||||
|
||||
async def import_skill_dir(db_arg, **kwargs):
|
||||
calls["import"] = {"db": db_arg, **kwargs}
|
||||
return SimpleNamespace(slug="demo-skill")
|
||||
|
||||
async def enable_skills(db_arg, thread_id, uid, skill_slugs):
|
||||
calls["enable"] = {"db": db_arg, "thread_id": thread_id, "uid": uid, "skill_slugs": skill_slugs}
|
||||
return True
|
||||
|
||||
def sync_thread_readable_skills(thread_id, skills):
|
||||
calls["sync"] = {"thread_id": thread_id, "skills": skills}
|
||||
|
||||
monkeypatch.setattr(
|
||||
install_skill_module,
|
||||
"_prepare_skill_from_sandbox",
|
||||
prepare_skill_from_sandbox,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
install_skill_module,
|
||||
"_enable_skills_in_current_config",
|
||||
enable_skills,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
install_skill_module.pg_manager,
|
||||
"get_async_session_context",
|
||||
lambda: _AsyncSessionContext(db),
|
||||
)
|
||||
monkeypatch.setattr(skill_service, "import_skill_dir", import_skill_dir)
|
||||
monkeypatch.setattr(skill_service, "sync_thread_readable_skills", sync_thread_readable_skills)
|
||||
|
||||
result = await install_skill_module._run_install_task(
|
||||
" /home/gem/user-data/workspace/demo-skill ",
|
||||
_runtime(uid="normal-user", thread_id="thread-1", skills=["existing-skill"]),
|
||||
"tool-1",
|
||||
)
|
||||
|
||||
assert result.update["activated_skills"] == ["demo-skill"]
|
||||
assert "成功安装并激活技能" in result.update["messages"][0].content
|
||||
assert calls["prepare"]["uid"] == "normal-user"
|
||||
assert calls["import"]["created_by"] == "normal-user"
|
||||
assert calls["import"]["share_config"] == {
|
||||
"access_level": "user",
|
||||
"department_ids": [],
|
||||
"user_uids": ["normal-user"],
|
||||
}
|
||||
assert calls["prepare"]["source"] == "/home/gem/user-data/workspace/demo-skill"
|
||||
assert calls["enable"] == {
|
||||
"db": db,
|
||||
"thread_id": "thread-1",
|
||||
"uid": "normal-user",
|
||||
"skill_slugs": ["demo-skill"],
|
||||
}
|
||||
assert calls["sync"] == {"thread_id": "thread-1", "skills": ["existing-skill", "demo-skill"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_skill_rejects_subagent_runtime_before_install(monkeypatch):
|
||||
def fail_get_session():
|
||||
raise AssertionError("子智能体运行态不应访问数据库或执行安装")
|
||||
|
||||
monkeypatch.setattr(
|
||||
install_skill_module.pg_manager,
|
||||
"get_async_session_context",
|
||||
fail_get_session,
|
||||
)
|
||||
|
||||
result = await install_skill_module._run_install_task(
|
||||
"/home/gem/user-data/workspace/demo-skill",
|
||||
_runtime(uid="user-1", thread_id="child-thread", is_subagent_runtime=True),
|
||||
"tool-1",
|
||||
)
|
||||
|
||||
assert "只能在主智能体中使用" in result.update["messages"][0].content
|
||||
assert "activated_skills" not in result.update
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_skill_git_source_requires_skill_names():
|
||||
result = await install_skill_module._run_install_task(
|
||||
"owner/repo",
|
||||
_runtime(uid="user-1", thread_id="thread-1"),
|
||||
"tool-1",
|
||||
)
|
||||
|
||||
assert "必须通过 skill_names 指定技能名称" in result.update["messages"][0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_skill_rejects_empty_source():
|
||||
result = await install_skill_module._run_install_task(
|
||||
" ",
|
||||
_runtime(uid="user-1", thread_id="thread-1"),
|
||||
"tool-1",
|
||||
)
|
||||
|
||||
assert "Skill 来源不能为空" in result.update["messages"][0].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enable_skills_updates_current_user_owned_agent_config(monkeypatch):
|
||||
conv = SimpleNamespace(uid="user-1", agent_id="agent-1")
|
||||
agent = SimpleNamespace(
|
||||
created_by="user-1",
|
||||
config_json={"context": {"skills": ["existing-skill"], "model": "provider:model"}},
|
||||
)
|
||||
calls = {}
|
||||
|
||||
class FakeConversationRepository:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_conversation_by_thread_id(self, thread_id):
|
||||
calls["thread_id"] = thread_id
|
||||
return conv
|
||||
|
||||
class FakeAgentRepository:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_by_slug(self, slug):
|
||||
calls["agent_slug"] = slug
|
||||
return agent
|
||||
|
||||
async def update(self, agent_arg, **kwargs):
|
||||
calls["update"] = {"agent": agent_arg, **kwargs}
|
||||
return agent_arg
|
||||
|
||||
monkeypatch.setattr(install_skill_module, "ConversationRepository", FakeConversationRepository)
|
||||
monkeypatch.setattr(install_skill_module, "AgentRepository", FakeAgentRepository)
|
||||
|
||||
result = await install_skill_module._enable_skills_in_current_config(
|
||||
SimpleNamespace(),
|
||||
"thread-1",
|
||||
"user-1",
|
||||
["existing-skill", "new-skill"],
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert calls["thread_id"] == "thread-1"
|
||||
assert calls["agent_slug"] == "agent-1"
|
||||
assert calls["update"]["updated_by"] == "user-1"
|
||||
assert calls["update"]["config_json"] == {
|
||||
"context": {"skills": ["existing-skill", "new-skill"], "model": "provider:model"}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enable_skills_does_not_update_agent_not_owned_by_current_user(monkeypatch):
|
||||
conv = SimpleNamespace(uid="user-1", agent_id="shared-agent")
|
||||
agent = SimpleNamespace(created_by="admin", config_json={"context": {}})
|
||||
calls = {}
|
||||
|
||||
class FakeConversationRepository:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_conversation_by_thread_id(self, _thread_id):
|
||||
return conv
|
||||
|
||||
class FakeAgentRepository:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_by_slug(self, _slug):
|
||||
return agent
|
||||
|
||||
async def update(self, *_args, **_kwargs):
|
||||
calls["updated"] = True
|
||||
|
||||
monkeypatch.setattr(install_skill_module, "ConversationRepository", FakeConversationRepository)
|
||||
monkeypatch.setattr(install_skill_module, "AgentRepository", FakeAgentRepository)
|
||||
|
||||
result = await install_skill_module._enable_skills_in_current_config(
|
||||
SimpleNamespace(),
|
||||
"thread-1",
|
||||
"user-1",
|
||||
["new-skill"],
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert "updated" not in calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enable_skills_does_not_update_mismatched_runtime_uid(monkeypatch):
|
||||
conv = SimpleNamespace(uid="user-1", agent_id="agent-1")
|
||||
calls = {}
|
||||
|
||||
class FakeConversationRepository:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_conversation_by_thread_id(self, _thread_id):
|
||||
return conv
|
||||
|
||||
class FakeAgentRepository:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def get_by_slug(self, *_args):
|
||||
calls["loaded_agent"] = True
|
||||
|
||||
monkeypatch.setattr(install_skill_module, "ConversationRepository", FakeConversationRepository)
|
||||
monkeypatch.setattr(install_skill_module, "AgentRepository", FakeAgentRepository)
|
||||
|
||||
result = await install_skill_module._enable_skills_in_current_config(
|
||||
SimpleNamespace(),
|
||||
"thread-1",
|
||||
"other-user",
|
||||
["new-skill"],
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert "loaded_agent" not in calls
|
||||
|
||||
|
||||
def test_prepare_skill_invalid_virtual_path_does_not_fallback_to_sandbox(monkeypatch, tmp_path: Path):
|
||||
calls = {}
|
||||
|
||||
def resolve_virtual_path(*_args, **_kwargs):
|
||||
raise ValueError("path traversal detected")
|
||||
|
||||
class FakeProvisionerSandboxBackend:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
calls["fallback"] = True
|
||||
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.sandbox.resolve_virtual_path",
|
||||
resolve_virtual_path,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"yuxi.agents.backends.sandbox.ProvisionerSandboxBackend",
|
||||
FakeProvisionerSandboxBackend,
|
||||
)
|
||||
monkeypatch.setattr(skill_service, "is_valid_skill_slug", lambda _slug: True)
|
||||
|
||||
with pytest.raises(ValueError, match="path traversal detected"):
|
||||
install_skill_module._prepare_skill_from_sandbox(
|
||||
"/home/gem/user-data/workspace/demo-skill",
|
||||
"thread-1",
|
||||
"user-1",
|
||||
tmp_path,
|
||||
)
|
||||
|
||||
assert "fallback" not in calls
|
||||
|
||||
|
||||
def test_collect_sandbox_file_paths_rejects_more_than_1000_files():
|
||||
class FakeBackend:
|
||||
def ls(self, _remote_dir):
|
||||
return SimpleNamespace(
|
||||
error=None,
|
||||
entries=[{"path": f"/skill/file-{idx}.txt", "is_dir": False} for idx in range(1001)],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="最多 1000 个文件"):
|
||||
install_skill_module._collect_sandbox_file_paths(FakeBackend(), "/skill")
|
||||
@@ -0,0 +1,581 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.toolkits.kbs import tools
|
||||
|
||||
|
||||
def _tool_callable(tool):
|
||||
callback = getattr(tool, "coroutine", None)
|
||||
if callback is not None:
|
||||
return callback
|
||||
|
||||
callback = getattr(tool, "func", None)
|
||||
if callback is not None:
|
||||
return callback
|
||||
|
||||
raise AssertionError(f"{tool.name} tool has no callable entry")
|
||||
|
||||
|
||||
def _query_kb_callable():
|
||||
return _tool_callable(tools.query_kb)
|
||||
|
||||
|
||||
def _find_kb_document_callable():
|
||||
return _tool_callable(tools.find_kb_document)
|
||||
|
||||
|
||||
def _open_kb_document_callable():
|
||||
return _tool_callable(tools.open_kb_document)
|
||||
|
||||
|
||||
async def _run_tool(callback, **kwargs):
|
||||
result = callback(**kwargs)
|
||||
if inspect.isawaitable(result):
|
||||
return await result
|
||||
return result
|
||||
|
||||
|
||||
async def _run_query_kb(**kwargs):
|
||||
return await _run_tool(_query_kb_callable(), **kwargs)
|
||||
|
||||
|
||||
async def _run_find_kb_document(**kwargs):
|
||||
return await _run_tool(_find_kb_document_callable(), **kwargs)
|
||||
|
||||
|
||||
async def _run_open_kb_document(**kwargs):
|
||||
return await _run_tool(_open_kb_document_callable(), **kwargs)
|
||||
|
||||
|
||||
def _build_test_window(content: str, offset: int = 0, limit: int = 1800) -> dict:
|
||||
lines = content.splitlines()
|
||||
start = min(max(offset, 0), len(lines))
|
||||
selected = lines[start : start + limit]
|
||||
end = start + len(selected)
|
||||
return {
|
||||
"start_line": start + 1 if selected else 0,
|
||||
"end_line": end,
|
||||
"total_lines": len(lines),
|
||||
"offset": start,
|
||||
"window_size": limit,
|
||||
"has_more_before": start > 0,
|
||||
"has_more_after": end < len(lines),
|
||||
"next_offset": end if end < len(lines) else None,
|
||||
"content": "\n".join(f"{start + idx + 1:6d}\t{line}" for idx, line in enumerate(selected)),
|
||||
}
|
||||
|
||||
|
||||
def _patch_retrievers(monkeypatch, *, kb_type: str = "milvus", retriever=None):
|
||||
async def _not_configured(*args, **kwargs):
|
||||
del args, kwargs
|
||||
raise AssertionError("knowledge base method is not configured for this test")
|
||||
|
||||
manager = SimpleNamespace(
|
||||
get_retrievers=lambda: {
|
||||
"db-1": {
|
||||
"name": "FAQ",
|
||||
"retriever": retriever or object(),
|
||||
"metadata": {"kb_type": kb_type},
|
||||
}
|
||||
},
|
||||
find_file_content=_not_configured,
|
||||
open_file_content=_not_configured,
|
||||
)
|
||||
monkeypatch.setattr(tools, "_get_knowledge_base", lambda: manager)
|
||||
monkeypatch.setattr(tools, "knowledge_base", manager, raising=False)
|
||||
return manager
|
||||
|
||||
|
||||
async def _fake_visible_kbs(runtime):
|
||||
del runtime
|
||||
return [{"kb_id": "db-1", "name": "FAQ"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_kb_returns_search_schema_without_sandbox_paths(monkeypatch) -> None:
|
||||
async def _fake_retriever(query_text: str, **kwargs):
|
||||
assert query_text == "auth"
|
||||
assert kwargs == {}
|
||||
return [
|
||||
{
|
||||
"content": "auth guide",
|
||||
"metadata": {
|
||||
"file_id": "file-1",
|
||||
"source": "auth-guide.pdf",
|
||||
"filepath": "/tmp/sandbox/auth-guide.pdf",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
_patch_retrievers(monkeypatch, retriever=_fake_retriever)
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_query_kb(kb_id="db-1", query_text="auth", runtime=runtime)
|
||||
|
||||
assert result["kb_id"] == "db-1"
|
||||
assert result["results"][0]["id"] == "file-1:1"
|
||||
assert result["results"][0]["kb_id"] == "db-1"
|
||||
assert result["results"][0]["file_id"] == "file-1"
|
||||
assert result["results"][0]["content"] == "auth guide"
|
||||
assert result["results"][0]["metadata"]["source"] == "auth-guide.pdf"
|
||||
assert "filepath" not in result["results"][0]["metadata"]
|
||||
assert "parsed_path" not in result["results"][0]["metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_kb_allows_dify_knowledge_base(monkeypatch) -> None:
|
||||
async def _fake_retriever(query_text: str, **kwargs):
|
||||
assert query_text == "auth"
|
||||
return [
|
||||
{
|
||||
"content": "auth guide",
|
||||
"score": 0.98,
|
||||
"metadata": {
|
||||
"file_id": "dify-doc-1",
|
||||
"chunk_id": "dify-segment-1",
|
||||
"source": "Dify Doc",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
_patch_retrievers(monkeypatch, kb_type="dify", retriever=_fake_retriever)
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_query_kb(kb_id="db-1", query_text="auth", runtime=runtime)
|
||||
|
||||
assert result == {
|
||||
"kb_id": "db-1",
|
||||
"results": [
|
||||
{
|
||||
"id": "dify-segment-1",
|
||||
"kb_id": "db-1",
|
||||
"file_id": "dify-doc-1",
|
||||
"content": "auth guide",
|
||||
"metadata": {
|
||||
"file_id": "dify-doc-1",
|
||||
"chunk_id": "dify-segment-1",
|
||||
"source": "Dify Doc",
|
||||
"score": 0.98,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_kb_returns_plain_result_without_path_injection(monkeypatch) -> None:
|
||||
async def _fake_retriever(query_text: str, **kwargs):
|
||||
assert query_text == "auth"
|
||||
return "Milvus context"
|
||||
|
||||
_patch_retrievers(monkeypatch, retriever=_fake_retriever)
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_query_kb(kb_id="db-1", query_text="auth", runtime=runtime)
|
||||
|
||||
assert result == "Milvus context"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_kb_maps_full_doc_id_and_chunk_metadata(monkeypatch) -> None:
|
||||
async def _fake_retriever(query_text: str, **kwargs):
|
||||
assert query_text == "auth"
|
||||
return [
|
||||
{
|
||||
"content": "auth guide",
|
||||
"full_doc_id": "file-1",
|
||||
"chunk_id": "chunk-1",
|
||||
"chunk_index": 3,
|
||||
}
|
||||
]
|
||||
|
||||
_patch_retrievers(monkeypatch, retriever=_fake_retriever)
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_query_kb(kb_id="db-1", query_text="auth", runtime=runtime)
|
||||
|
||||
assert result["results"][0] == {
|
||||
"id": "chunk-1",
|
||||
"kb_id": "db-1",
|
||||
"file_id": "file-1",
|
||||
"content": "auth guide",
|
||||
"metadata": {"chunk_index": 3},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_kb_document_returns_context_windows(monkeypatch) -> None:
|
||||
_patch_retrievers(monkeypatch)
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
async def _fake_find_file_content(
|
||||
kb_id: str,
|
||||
file_id: str,
|
||||
patterns: list[str],
|
||||
*,
|
||||
use_regex: bool = False,
|
||||
case_sensitive: bool = False,
|
||||
max_windows: int = 5,
|
||||
window_size: int = 80,
|
||||
):
|
||||
assert kb_id == "db-1"
|
||||
assert file_id == "file-1"
|
||||
assert patterns == ["token"]
|
||||
assert use_regex is False
|
||||
assert case_sensitive is False
|
||||
assert max_windows == 5
|
||||
assert window_size == 80
|
||||
return {
|
||||
"semantic": False,
|
||||
"match_mode": "keyword",
|
||||
"total_matches": 2,
|
||||
"windows": [
|
||||
{
|
||||
"start_line": 1,
|
||||
"end_line": 3,
|
||||
"matched_lines": [2],
|
||||
"content": " 1\tintro\n 2\ttoken value\n 3\toutro",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
monkeypatch.setattr(tools.knowledge_base, "find_file_content", _fake_find_file_content)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_find_kb_document(
|
||||
kb_id="db-1",
|
||||
file_id="file-1",
|
||||
patterns=["token"],
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"kb_id": "db-1",
|
||||
"file_id": "file-1",
|
||||
"semantic": False,
|
||||
"match_mode": "keyword",
|
||||
"total_matches": 2,
|
||||
"windows": [
|
||||
{
|
||||
"start_line": 1,
|
||||
"end_line": 3,
|
||||
"matched_lines": [2],
|
||||
"content": " 1\tintro\n 2\ttoken value\n 3\toutro",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_kb_document_rejects_dify(monkeypatch) -> None:
|
||||
_patch_retrievers(monkeypatch, kb_type="dify")
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_find_kb_document(
|
||||
kb_id="db-1",
|
||||
file_id="file-1",
|
||||
patterns=["token"],
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
assert "Dify 知识库" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_kb_document_reads_markdown_content_by_default_window(monkeypatch) -> None:
|
||||
lines = [f"line {index}" for index in range(1, 2001)]
|
||||
|
||||
_patch_retrievers(monkeypatch)
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
async def _fake_open_file_content(kb_id: str, file_id: str, offset: int = 0, limit: int = 1800):
|
||||
assert kb_id == "db-1"
|
||||
assert file_id == "file-1"
|
||||
return _build_test_window("\n".join(lines), offset=offset, limit=limit)
|
||||
|
||||
monkeypatch.setattr(tools.knowledge_base, "open_file_content", _fake_open_file_content)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_open_kb_document(kb_id="db-1", file_id="file-1", runtime=runtime)
|
||||
|
||||
assert result["kb_id"] == "db-1"
|
||||
assert result["file_id"] == "file-1"
|
||||
assert result["start_line"] == 1
|
||||
assert result["end_line"] == 1800
|
||||
assert result["total_lines"] == 2000
|
||||
assert result["window_size"] == 1800
|
||||
assert result["has_more_before"] is False
|
||||
assert result["has_more_after"] is True
|
||||
assert result["next_offset"] == 1800
|
||||
assert " 1\tline 1" in result["content"]
|
||||
assert " 1800\tline 1800" in result["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_kb_document_prefers_line_over_offset(monkeypatch) -> None:
|
||||
lines = [f"line {index}" for index in range(1, 1001)]
|
||||
|
||||
_patch_retrievers(monkeypatch)
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
async def _fake_open_file_content(kb_id: str, file_id: str, offset: int = 0, limit: int = 1800):
|
||||
assert kb_id == "db-1"
|
||||
assert file_id == "file-1"
|
||||
return _build_test_window("\n".join(lines), offset=offset, limit=limit)
|
||||
|
||||
monkeypatch.setattr(tools.knowledge_base, "open_file_content", _fake_open_file_content)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_open_kb_document(
|
||||
kb_id="db-1",
|
||||
file_id="file-1",
|
||||
line=801,
|
||||
offset=0,
|
||||
window_size=10,
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
assert result["offset"] == 800
|
||||
assert result["start_line"] == 801
|
||||
assert result["end_line"] == 810
|
||||
assert result["has_more_before"] is True
|
||||
assert result["has_more_after"] is True
|
||||
assert result["next_offset"] == 810
|
||||
assert " 801\tline 801" in result["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_kb_document_rejects_invisible_resource(monkeypatch) -> None:
|
||||
async def _fake_visible_kbs(runtime):
|
||||
del runtime
|
||||
return [{"kb_id": "db-2", "name": "FAQ"}]
|
||||
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_open_kb_document(kb_id="db-1", file_id="file-1", runtime=runtime)
|
||||
|
||||
assert "不存在或当前会话未启用" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_kb_document_requires_markdown_content(monkeypatch) -> None:
|
||||
_patch_retrievers(monkeypatch)
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
async def _fake_open_file_content(kb_id: str, file_id: str, offset: int = 0, limit: int = 1800):
|
||||
del kb_id, file_id, offset, limit
|
||||
raise Exception("文件 file-1 没有解析后的 Markdown 内容")
|
||||
|
||||
monkeypatch.setattr(tools.knowledge_base, "open_file_content", _fake_open_file_content)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_open_kb_document(kb_id="db-1", file_id="file-1", runtime=runtime)
|
||||
|
||||
assert "没有解析后的 Markdown 内容" in result
|
||||
|
||||
|
||||
def _search_file_callable():
|
||||
return _tool_callable(tools.search_file)
|
||||
|
||||
|
||||
async def _run_search_file(**kwargs):
|
||||
return await _run_tool(_search_file_callable(), **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_file_requires_kb_name_or_query(monkeypatch) -> None:
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_search_file(runtime=runtime)
|
||||
|
||||
assert "不能同时为空" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_file_returns_files_by_query(monkeypatch) -> None:
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
from types import SimpleNamespace as SN
|
||||
|
||||
fake_files = [
|
||||
SN(
|
||||
file_id="file-1",
|
||||
filename="test.pdf",
|
||||
file_type="file",
|
||||
status="indexed",
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=1024,
|
||||
),
|
||||
SN(
|
||||
file_id="file-2",
|
||||
filename="test2.pdf",
|
||||
file_type="file",
|
||||
status="indexed",
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=2048,
|
||||
),
|
||||
SN(
|
||||
file_id="file-3",
|
||||
filename="other.pdf",
|
||||
file_type="file",
|
||||
status="indexed",
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=512,
|
||||
),
|
||||
]
|
||||
|
||||
async def _fake_list_by_kb_id_after(self, kb_id, *, after_file_id=None, limit=500, files_only=False):
|
||||
return fake_files
|
||||
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
|
||||
monkeypatch.setattr(KnowledgeFileRepository, "list_by_kb_id_after", _fake_list_by_kb_id_after)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_search_file(query="test", runtime=runtime)
|
||||
|
||||
assert result["total"] == 2
|
||||
assert len(result["files"]) == 2
|
||||
assert result["files"][0]["filename"] == "test.pdf"
|
||||
assert result["files"][1]["filename"] == "test2.pdf"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_file_returns_all_files_when_query_empty(monkeypatch) -> None:
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
from types import SimpleNamespace as SN
|
||||
|
||||
fake_files = [
|
||||
SN(
|
||||
file_id="file-1",
|
||||
filename="test.pdf",
|
||||
file_type="file",
|
||||
status="indexed",
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=1024,
|
||||
),
|
||||
SN(
|
||||
file_id="file-2",
|
||||
filename="other.pdf",
|
||||
file_type="file",
|
||||
status="indexed",
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=2048,
|
||||
),
|
||||
]
|
||||
|
||||
async def _fake_list_by_kb_id_after(self, kb_id, *, after_file_id=None, limit=500, files_only=False):
|
||||
return fake_files
|
||||
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
|
||||
monkeypatch.setattr(KnowledgeFileRepository, "list_by_kb_id_after", _fake_list_by_kb_id_after)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_search_file(kb_name="FAQ", runtime=runtime)
|
||||
|
||||
assert result["total"] == 2
|
||||
assert len(result["files"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_file_pagination(monkeypatch) -> None:
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
from types import SimpleNamespace as SN
|
||||
|
||||
fake_files = [
|
||||
SN(
|
||||
file_id=f"file-{i}",
|
||||
filename=f"file{i}.pdf",
|
||||
file_type="file",
|
||||
status="indexed",
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=1024 * i,
|
||||
)
|
||||
for i in range(10)
|
||||
]
|
||||
|
||||
async def _fake_list_by_kb_id_after(self, kb_id, *, after_file_id=None, limit=500, files_only=False):
|
||||
return fake_files
|
||||
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
|
||||
monkeypatch.setattr(KnowledgeFileRepository, "list_by_kb_id_after", _fake_list_by_kb_id_after)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_search_file(kb_name="FAQ", offset=2, limit=3, runtime=runtime)
|
||||
|
||||
assert result["total"] == 10
|
||||
assert len(result["files"]) == 3
|
||||
assert result["offset"] == 2
|
||||
assert result["limit"] == 3
|
||||
assert result["has_more"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_file_rejects_invisible_kb(monkeypatch) -> None:
|
||||
async def _fake_visible_kbs(runtime):
|
||||
del runtime
|
||||
return [{"kb_id": "db-2", "name": "Other"}]
|
||||
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_search_file(kb_name="FAQ", query="test", runtime=runtime)
|
||||
|
||||
assert "不存在或当前会话未启用" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_file_total_reflects_full_set_not_page(monkeypatch) -> None:
|
||||
"""total/has_more 必须基于全量文件,而非按 limit/offset 截断的窗口。"""
|
||||
monkeypatch.setattr(tools, "_resolve_visible_knowledge_bases_for_query", _fake_visible_kbs)
|
||||
|
||||
from types import SimpleNamespace as SN
|
||||
|
||||
fake_files = [
|
||||
SN(
|
||||
file_id=f"file-{i:02d}",
|
||||
filename=f"file{i}.pdf",
|
||||
file_type="file",
|
||||
status="indexed",
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
file_size=1024,
|
||||
)
|
||||
for i in range(50)
|
||||
]
|
||||
|
||||
async def _fake_list_by_kb_id_after(self, kb_id, *, after_file_id=None, limit=500, files_only=False):
|
||||
# 真实仓储会按 limit 截断;此 mock 同样遵守 limit,以暴露按 limit+offset 取数导致的 total 失真。
|
||||
return fake_files[:limit]
|
||||
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
|
||||
monkeypatch.setattr(KnowledgeFileRepository, "list_by_kb_id_after", _fake_list_by_kb_id_after)
|
||||
|
||||
runtime = SimpleNamespace(context=SimpleNamespace())
|
||||
result = await _run_search_file(kb_name="FAQ", offset=0, limit=10, runtime=runtime)
|
||||
|
||||
assert result["total"] == 50
|
||||
assert len(result["files"]) == 10
|
||||
assert result["has_more"] is True
|
||||
@@ -0,0 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from yuxi.agents.backends.sandbox.paths import (
|
||||
ensure_thread_dirs,
|
||||
sandbox_outputs_dir,
|
||||
sandbox_uploads_dir,
|
||||
sandbox_workspace_dir,
|
||||
virtual_path_for_thread_file,
|
||||
)
|
||||
from yuxi.agents.toolkits.buildin.tools import ocr_parse_file
|
||||
from yuxi.knowledge.parser import Parser
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _runtime(
|
||||
*,
|
||||
thread_id: str = "thread-1",
|
||||
uid: str = "user-1",
|
||||
file_thread_id: str | None = None,
|
||||
) -> SimpleNamespace:
|
||||
configurable = {"thread_id": thread_id, "uid": uid}
|
||||
if file_thread_id:
|
||||
configurable["file_thread_id"] = file_thread_id
|
||||
return SimpleNamespace(
|
||||
config={"configurable": configurable},
|
||||
context=SimpleNamespace(thread_id=thread_id, uid=uid, file_thread_id=file_thread_id),
|
||||
state={},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_parse_file_writes_markdown_to_outputs(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr("yuxi.config.save_dir", str(tmp_path))
|
||||
thread_id = "thread-1"
|
||||
uid = "user-1"
|
||||
ensure_thread_dirs(thread_id, uid)
|
||||
source_path = sandbox_workspace_dir(thread_id, uid) / "scan.png"
|
||||
source_path.write_bytes(b"fake image")
|
||||
source_virtual_path = virtual_path_for_thread_file(thread_id, source_path, uid=uid)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_aparse(source: str, params: dict | None = None) -> str:
|
||||
captured["source"] = source
|
||||
captured["params"] = params
|
||||
return "识别结果\n" + ("长文本" * 500)
|
||||
|
||||
monkeypatch.setattr(Parser, "aparse", fake_aparse)
|
||||
|
||||
result = await ocr_parse_file.coroutine(
|
||||
file_path=source_virtual_path,
|
||||
ocr_engine="mineru_ocr",
|
||||
runtime=_runtime(thread_id=thread_id, uid=uid),
|
||||
)
|
||||
|
||||
output_root = sandbox_outputs_dir(thread_id) / "ocr"
|
||||
output_path = output_root / "scan.md"
|
||||
assert output_path.exists()
|
||||
assert output_path.read_text(encoding="utf-8").startswith("识别结果")
|
||||
assert result["source_path"] == source_virtual_path
|
||||
assert result["parsed_path"] == virtual_path_for_thread_file(thread_id, output_path, uid=uid)
|
||||
assert result["ocr_engine"] == "mineru_ocr"
|
||||
assert result["char_count"] == len(output_path.read_text(encoding="utf-8"))
|
||||
assert result["truncated"] is True
|
||||
assert len(result["preview"]) <= 1200
|
||||
assert captured["source"] == str(source_path)
|
||||
assert captured["params"] == {"ocr_engine": "mineru_ocr"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_parse_file_uses_default_engine(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr("yuxi.config.save_dir", str(tmp_path))
|
||||
monkeypatch.setattr("yuxi.config.default_ocr_engine", "rapid_ocr")
|
||||
thread_id = "thread-1"
|
||||
uid = "user-1"
|
||||
ensure_thread_dirs(thread_id, uid)
|
||||
source_path = sandbox_uploads_dir(thread_id) / "upload.pdf"
|
||||
source_path.write_bytes(b"fake pdf")
|
||||
source_virtual_path = virtual_path_for_thread_file(thread_id, source_path, uid=uid)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_aparse(source: str, params: dict | None = None) -> str:
|
||||
captured["params"] = params
|
||||
return "OCR content"
|
||||
|
||||
monkeypatch.setattr(Parser, "aparse", fake_aparse)
|
||||
|
||||
result = await ocr_parse_file.coroutine(
|
||||
file_path=source_virtual_path,
|
||||
runtime=_runtime(thread_id=thread_id, uid=uid),
|
||||
)
|
||||
|
||||
assert result["ocr_engine"] == "rapid_ocr"
|
||||
assert captured["params"] == {"ocr_engine": "rapid_ocr"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_parse_file_rejects_non_user_data_path(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr("yuxi.config.save_dir", str(tmp_path))
|
||||
|
||||
with pytest.raises(ValueError, match="只允许解析"):
|
||||
await ocr_parse_file.coroutine(file_path="/etc/passwd", runtime=_runtime())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_parse_file_rejects_directory(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr("yuxi.config.save_dir", str(tmp_path))
|
||||
thread_id = "thread-1"
|
||||
uid = "user-1"
|
||||
ensure_thread_dirs(thread_id, uid)
|
||||
dir_virtual_path = virtual_path_for_thread_file(thread_id, sandbox_workspace_dir(thread_id, uid), uid=uid)
|
||||
|
||||
with pytest.raises(ValueError, match="路径不是普通文件"):
|
||||
await ocr_parse_file.coroutine(file_path=dir_virtual_path, runtime=_runtime(thread_id=thread_id, uid=uid))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_parse_file_rejects_path_traversal(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr("yuxi.config.save_dir", str(tmp_path))
|
||||
|
||||
with pytest.raises(ValueError, match="只允许解析"):
|
||||
await ocr_parse_file.coroutine(
|
||||
file_path="/home/gem/user-data/../secrets.png",
|
||||
runtime=_runtime(),
|
||||
)
|
||||
@@ -0,0 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from yuxi.agents.toolkits.registry import tool
|
||||
|
||||
|
||||
def test_tool_decorator_sets_handle_tool_error():
|
||||
"""测试通过 @tool 装饰器注册的工具是否自动设置了 handle_tool_error 为 True"""
|
||||
|
||||
@tool(
|
||||
category="test",
|
||||
display_name="测试工具",
|
||||
description="这是一个单元测试工具",
|
||||
)
|
||||
def my_test_tool(arg: str) -> str:
|
||||
return f"hello {arg}"
|
||||
|
||||
assert my_test_tool.name == "my_test_tool"
|
||||
assert my_test_tool.handle_tool_error is True
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user