chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,28 @@
|
||||
# 忽略 Git 相关的文件夹
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
# 忽略 Python 的缓存文件
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
|
||||
# 忽略虚拟环境目录
|
||||
venv/
|
||||
env/
|
||||
backend/.venv/
|
||||
backend/package/.venv/
|
||||
|
||||
# 忽略 node_modules 文件夹（前端项目）
|
||||
web/node_modules/
|
||||
|
||||
# 忽略 Docker 运行时数据卷
|
||||
docker/volumes/
|
||||
|
||||
# 忽略日志和临时文件
|
||||
*.log
|
||||
*.tmp
|
||||
|
||||
# 忽略 Docker 本身的文件
|
||||
Dockerfile
|
||||
docker-compose.yml
|
||||
@@ -0,0 +1,78 @@
|
||||
# region model_provider
|
||||
SILICONFLOW_API_KEY= # 推荐使用硅基流动免费服务 https://cloud.siliconflow.cn/i/Eo5yTHGJ
|
||||
TAVILY_API_KEY= # 获取搜索服务的 api key 请访问 https://app.tavily.com/
|
||||
|
||||
# JWT 安全配置：首次部署请生成随机值，初始化脚本会在留空时自动生成
|
||||
YUXI_ENV=development
|
||||
JWT_SECRET_KEY=
|
||||
YUXI_INSTANCE_ID=
|
||||
|
||||
# CORS allowed origins for browser clients, comma-separated.
|
||||
# Development defaults to http://localhost:5173,http://127.0.0.1:5173 when unset.
|
||||
# Production does not allow cross-origin browser requests unless this is explicitly set.
|
||||
YUXI_CORS_ORIGINS=
|
||||
|
||||
# # 其余可选配置
|
||||
# OPENAI_API_KEY=
|
||||
# OPENAI_API_BASE=
|
||||
# ZHIPUAI_API_KEY=
|
||||
# DASHSCOPE_API_KEY=
|
||||
# DEEPSEEK_API_KEY=
|
||||
# ARK_API_KEY=
|
||||
# MINIMAX_API_KEY= # MiniMax 大模型 https://platform.minimaxi.com/
|
||||
# TOGETHER_API_KEY=
|
||||
# # endregion model_provider
|
||||
|
||||
# # region neo4j
|
||||
# NEO4J_URI=
|
||||
# NEO4J_USERNAME=
|
||||
# NEO4J_PASSWORD=
|
||||
# # endregion neo4j
|
||||
|
||||
# # Servies
|
||||
# YUXI_SUPER_ADMIN_NAME=
|
||||
# YUXI_SUPER_ADMIN_PASSWORD=
|
||||
|
||||
# # URL Whitelist (comma-separated domains/IPs, empty to disable URL parsing)
|
||||
# YUXI_URL_WHITELIST=github.com,docs.example.com,gitlab.example.com,127.0.0.1
|
||||
|
||||
# # MinerU
|
||||
# MINERU_API_KEY=
|
||||
|
||||
# # PaddleOCR API
|
||||
# PADDLEOCR_API_TOKEN=
|
||||
# PADDLEOCR_API_URL=https://paddleocr.aistudio-app.com/api/v2/ocr/jobs
|
||||
|
||||
# Sandbox (deerFlow-style provisioner)
|
||||
# SANDBOX_PROVIDER=provisioner
|
||||
# SANDBOX_PROVISIONER_URL=http://sandbox-provisioner:8002
|
||||
# SANDBOX_VIRTUAL_PATH_PREFIX=/home/gem/user-data
|
||||
# SANDBOX_EXEC_TIMEOUT_SECONDS=180
|
||||
# SANDBOX_MAX_OUTPUT_BYTES=262144
|
||||
# SANDBOX_KEEPALIVE_INTERVAL_SECONDS=30
|
||||
# SANDBOX_IDLE_TIMEOUT_SECONDS=120
|
||||
# SANDBOX_IDLE_CHECK_INTERVAL_SECONDS=10
|
||||
# # sandbox-provisioner backend: memory | docker | kubernetes
|
||||
# SANDBOX_PROVISIONER_BACKEND=docker
|
||||
|
||||
# sandbox-provisioner 通用配置
|
||||
# SANDBOX_IMAGE=enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest
|
||||
# SANDBOX_CONTAINER_PORT=8080
|
||||
# SANDBOX_HEALTH_TIMEOUT_SECONDS=300
|
||||
# MEMORY_SANDBOX_URL_TEMPLATE=http://agent-sandbox:8000
|
||||
# SANDBOX_HTTP_PROXY=http://host.docker.internal:7897
|
||||
# SANDBOX_HTTPS_PROXY=http://host.docker.internal:7897
|
||||
|
||||
# Docker backend 专用 (used when SANDBOX_PROVISIONER_BACKEND=docker)
|
||||
# 注意：网络名称已在 docker-compose.yml 中固定为 yuxi-know_app-network
|
||||
# SANDBOX_DOCKER_NETWORK=yuxi-know_app-network
|
||||
# SANDBOX_DOCKER_THREADS_HOST_PATH=
|
||||
# SANDBOX_DOCKER_SANDBOX_PREFIX=yuxi-sandbox
|
||||
# SANDBOX_DOCKER_SANDBOX_HOST=host.docker.internal
|
||||
|
||||
# Kubernetes backend 专用 (used when SANDBOX_PROVISIONER_BACKEND=kubernetes)
|
||||
# SANDBOX_K8S_NAMESPACE=yuxi-know
|
||||
# SANDBOX_NODE_HOST=host.docker.internal
|
||||
# KUBECONFIG_PATH=/root/.kube/config
|
||||
# THREAD_PVC=yuxi-thread
|
||||
# SKILLS_PVC=yuxi-skills # 当前代码会读取，但 Pod 挂载实际仍只使用 THREAD_PVC
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
name: 提交一个BUG或者报错
|
||||
about: 无法启动、无法回答、后端报错等等
|
||||
title: 'Error: '
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
1️⃣ 描述一下问题
|
||||
|
||||
<!-- 简单描述一下问题（如何产生的，什么情况下，进行什么操作的时候）-->
|
||||
|
||||
|
||||
|
||||
2️⃣ 报错日志
|
||||
|
||||
请运行以下命令，并提供部分相关日志：
|
||||
|
||||
```sh
|
||||
# macOS / Linux
|
||||
make logs
|
||||
|
||||
# Windows
|
||||
docker logs --tail=100 api-dev
|
||||
git rev-parse HEAD
|
||||
```
|
||||
|
||||
<!-- 在下面粘贴日志的输出 -->
|
||||
|
||||
```
|
||||
make logs 的输出：
|
||||
|
||||
|
||||
|
||||
```
|
||||
|
||||
|
||||
3️⃣ 相关截图
|
||||
|
||||
|
||||
|
||||
|
||||
#️⃣ 其他相关信息
|
||||
|
||||
|
||||
✅ 如果问题与模型调用相关，请尝试切换到其他在线模型
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
name: 提交一个启动问题
|
||||
about: Docker镜像拉取、服务启动、端口占用等相关问题
|
||||
title: 'Startup: '
|
||||
labels: startup
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## 1️⃣ 问题描述
|
||||
|
||||
请清晰描述您在使用 Docker 或启动服务时遇到的问题：
|
||||
- 操作步骤：您执行了什么操作？
|
||||
- 预期结果：您期望看到什么？
|
||||
- 实际结果：实际发生了什么？
|
||||
|
||||
例如："执行 `docker compose up -d` 后，api-dev 服务一直重启，查看日志显示无法连接到 Milvus"
|
||||
|
||||
|
||||
## 2️⃣ 环境信息
|
||||
|
||||
请提供以下信息，帮助我们快速定位问题：
|
||||
- 操作系统：Windows/macOS/Linux 及版本
|
||||
- Docker 版本：执行 `docker --version` 输出
|
||||
- Docker Compose 版本：执行 `docker compose --version` 输出
|
||||
- 项目版本：执行 `git rev-parse HEAD` 输出
|
||||
|
||||
|
||||
## 3️⃣ 启动命令
|
||||
|
||||
请提供您使用的完整启动命令：
|
||||
```bash
|
||||
# 例如
|
||||
docker compose up -d
|
||||
# 或
|
||||
make up
|
||||
```
|
||||
|
||||
|
||||
## 4️⃣ 日志信息
|
||||
|
||||
请提供相关服务的日志（至少包含最近 100 行）：
|
||||
|
||||
```bash
|
||||
# 查看所有服务状态
|
||||
docker ps
|
||||
|
||||
# 查看 api-dev 服务日志
|
||||
docker logs --tail=100 api-dev
|
||||
|
||||
# 查看所有服务日志
|
||||
docker compose logs --tail=100
|
||||
```
|
||||
|
||||
将日志粘贴到下方（可根据问题相关性选择部分日志）：
|
||||
|
||||
```
|
||||
# api-dev 日志
|
||||
...
|
||||
|
||||
# 其他相关服务日志
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
## 5️⃣ 配置文件（可选）
|
||||
|
||||
如果您修改过 `docker-compose.yml` 或 `.env` 文件，请提供相关配置片段（注意隐藏敏感信息）：
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml 相关部分
|
||||
...
|
||||
|
||||
# .env 相关部分
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
## 6️⃣ 其他信息
|
||||
|
||||
您还可以提供以下信息帮助我们解决问题：
|
||||
- 是否已尝试过重启 Docker 服务？
|
||||
- 是否已清理过 Docker 缓存或旧容器？
|
||||
- 网络环境是否有特殊配置（如代理、防火墙等）？
|
||||
- 是否有其他相关的错误提示或截图？
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
name: 提交一个提问
|
||||
about: 关于项目使用的问题
|
||||
title: 'Question: '
|
||||
labels: question
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**问题描述**
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
name: 提交一个需求建议
|
||||
about: 为这个项目提个建议
|
||||
title: 'Feat: '
|
||||
labels: enhancement
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**#️⃣ 您的功能请求是否与某个问题相关？请描述。**
|
||||
<!--清晰简明地描述问题所在。例如：当[...]时，我感觉不方便。-->
|
||||
|
||||
**#️⃣ 描述您期望的解决方案**
|
||||
<!--清晰简明地描述您希望看到的结果。-->
|
||||
|
||||
**#️⃣ 附加背景信息**
|
||||
<!--在此处添加关于该功能请求的其他背景信息或截图。-->
|
||||
@@ -0,0 +1,28 @@
|
||||
## 变更描述
|
||||
|
||||
> 🔥 务必注意，本项目不允许提交未经 “Human Review” 过的代码，不允许 Agent 未经人类审阅，直接提交 PR。不允许 Agent 直接提交回复。
|
||||
>
|
||||
> Code is Cheap, Show Me Your Thoughts!
|
||||
|
||||
简要描述这个 PR 做了什么
|
||||
|
||||
|
||||
## 变更类型
|
||||
- [ ] 新功能
|
||||
- [ ] Bug 修复
|
||||
- [ ] 文档更新
|
||||
- [ ] 其他
|
||||
|
||||
## 测试
|
||||
- [ ] 已在 Docker 环境测试
|
||||
- [ ] 相关功能正常工作
|
||||
|
||||
**相关日志或者截图**
|
||||
|
||||
|
||||
## 说明
|
||||
（可选）有什么需要特别说明的吗？
|
||||
|
||||
---
|
||||
|
||||
💡 **提示**: 提交前可以运行 `make lint` 和 `make format` 检查代码规范
|
||||
@@ -0,0 +1,29 @@
|
||||
# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
|
||||
#
|
||||
# You can adjust the behavior by modifying this file.
|
||||
# For more information, see:
|
||||
# https://github.com/actions/stale
|
||||
name: Mark stale issues and pull requests
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '40 9 * * *'
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- uses: actions/stale@v5
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
stale-issue-message: '此 Issue 已标记为待关闭，若 14 天内无进一步活动将被自动关闭。'
|
||||
close-issue-message: '因 14 天无响应，此 Issue 已自动关闭。如有需要，请重新打开。'
|
||||
stale-issue-label: 'no-issue-activity'
|
||||
only-labels: 'to-be-closed'
|
||||
days-before-stale: 30
|
||||
days-before-close: 14
|
||||
@@ -0,0 +1,72 @@
|
||||
# 构建 VitePress 站点并将其部署到 GitHub Pages 的示例工作流程
|
||||
#
|
||||
name: Deploy VitePress site to Pages
|
||||
|
||||
on:
|
||||
# 在 docs 相关变更推送时运行构建（任意分支）
|
||||
push:
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- '.github/workflows/deploy.yml'
|
||||
|
||||
# 在面向 main 的 PR 中提前验证 docs 构建
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- '.github/workflows/deploy.yml'
|
||||
|
||||
# 允许你从 Actions 选项卡手动运行此工作流程
|
||||
workflow_dispatch:
|
||||
|
||||
# 设置 GITHUB_TOKEN 的权限，以允许部署到 GitHub Pages
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
# 只允许同时进行一次部署，跳过正在运行和最新队列之间的运行队列
|
||||
# 但是，不要取消正在进行的运行，因为我们希望允许这些生产部署完成
|
||||
concurrency:
|
||||
group: pages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
# 构建工作
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # 如果未启用 lastUpdated，则不需要
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v4
|
||||
- name: Install dependencies
|
||||
run: npm install
|
||||
working-directory: docs
|
||||
- name: Build with VitePress
|
||||
run: npm run build
|
||||
working-directory: docs
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: docs/.vitepress/dist
|
||||
|
||||
# 部署工作
|
||||
deploy:
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
name: Deploy
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
@@ -0,0 +1,49 @@
|
||||
name: Publish yuxi-cli
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Build and publish to PyPI
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
defaults:
|
||||
run:
|
||||
working-directory: packages/yuxi-cli
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
with:
|
||||
version: "0.7.2"
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --group test
|
||||
|
||||
- name: Run tests
|
||||
run: uv run pytest
|
||||
|
||||
- name: Build package
|
||||
run: uv build
|
||||
|
||||
- name: Publish package
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
packages-dir: packages/yuxi-cli/dist/
|
||||
@@ -0,0 +1,157 @@
|
||||
# GitHub Actions workflow for Ruff code formatting and linting
|
||||
# 自动运行 Ruff 格式检查和修复
|
||||
#
|
||||
# 触发条件：
|
||||
# 1. Push 到 main 分支时：自动格式化并提交
|
||||
|
||||
name: Ruff Format Check
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'backend/**/*.py'
|
||||
- 'backend/pyproject.toml'
|
||||
- 'Makefile'
|
||||
- '.github/workflows/ruff.yml'
|
||||
|
||||
# 设置写入权限，允许自动提交格式化代码
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
ruff:
|
||||
name: Ruff Format & Lint
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: backend
|
||||
|
||||
steps:
|
||||
# 1. 检出代码
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# 2. 安装 uv（项目使用 uv 作为包管理器）
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
with:
|
||||
version: "0.7.2"
|
||||
|
||||
# 3. 设置 Python 环境
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
# 4. 安装 ruff 依赖（根据项目配置，ruff 在 dev 依赖组中）
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv sync --group dev
|
||||
|
||||
# 5. 运行 Ruff 格式检查（与项目的 make lint 命令一致）
|
||||
- name: Run Ruff format check
|
||||
id: ruff-format
|
||||
run: |
|
||||
set +e # 不立即退出失败
|
||||
|
||||
echo "Running ruff check (uv run python -m ruff check package)..."
|
||||
uv run python -m ruff check package
|
||||
ruff_check_result=$?
|
||||
|
||||
echo "Running ruff format diff check (uv run python -m ruff format package --diff)..."
|
||||
uv run python -m ruff format package --diff
|
||||
ruff_format_result=$?
|
||||
|
||||
echo "Running import sorting check (uv run python -m ruff check --select I package)..."
|
||||
uv run python -m ruff check --select I package
|
||||
ruff_import_result=$?
|
||||
|
||||
# 检查是否有任何错误
|
||||
if [ $ruff_check_result -eq 0 ] && [ $ruff_format_result -eq 0 ] && [ $ruff_import_result -eq 0 ]; then
|
||||
echo "ruff_format_passed=true" >> $GITHUB_OUTPUT
|
||||
echo "✅ Ruff format check passed"
|
||||
else
|
||||
echo "ruff_format_passed=false" >> $GITHUB_OUTPUT
|
||||
echo "❌ Ruff format check failed"
|
||||
echo "::warning::Ruff format check found issues that should be addressed"
|
||||
|
||||
# 汇总错误信息
|
||||
echo "## Ruff Format Issues Summary" > $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "The following formatting issues were found:" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
# 运行 ruff check 以获取详细错误信息
|
||||
echo "### Detailed Issues:" >> $GITHUB_STEP_SUMMARY
|
||||
uv run python -m ruff check package --output-format=github >> $GITHUB_STEP_SUMMARY 2>&1 || true
|
||||
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "To fix these issues locally, run:" >> $GITHUB_STEP_SUMMARY
|
||||
echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
|
||||
echo "make format" >> $GITHUB_STEP_SUMMARY
|
||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
|
||||
# 6. 针对不同事件类型执行不同操作
|
||||
- name: Process Ruff Results
|
||||
if: ${{ github.event_name == 'push' }}
|
||||
env:
|
||||
GIT_AUTHOR_NAME: "GitHub Actions"
|
||||
GIT_AUTHOR_EMAIL: "actions@github.com"
|
||||
GIT_COMMITTER_NAME: "GitHub Actions"
|
||||
GIT_COMMITTER_EMAIL: "actions@github.com"
|
||||
run: |
|
||||
echo "Processing push event to main branch..."
|
||||
|
||||
# 检查当前是否有未提交的修改
|
||||
if ! git diff --exit-code --quiet; then
|
||||
echo "⚠️ Warning: There are uncommitted changes before formatting"
|
||||
echo "The formatting process might need to handle this specially."
|
||||
fi
|
||||
|
||||
# 先检查是否有格式问题
|
||||
echo "Checking for formatting issues..."
|
||||
set +e
|
||||
uv run python -m ruff check package --quiet
|
||||
has_ruff_issues=$?
|
||||
set -e
|
||||
|
||||
if [ $has_ruff_issues -eq 0 ]; then
|
||||
echo "✅ No formatting issues found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Formatting issues detected, running automatic fixes..."
|
||||
|
||||
# 自动应用格式修复（与项目的 make format 命令一致）
|
||||
echo "Running uv run ruff format package"
|
||||
uv run ruff format package
|
||||
|
||||
echo "Running uv run ruff check package --fix"
|
||||
uv run ruff check package --fix
|
||||
|
||||
echo "Running uv run python -m ruff check --select I package --fix"
|
||||
uv run python -m ruff check --select I package --fix
|
||||
|
||||
# 检查是否有格式变更
|
||||
echo "Checking for formatting changes..."
|
||||
if git diff --exit-code --quiet; then
|
||||
echo "✅ No formatting changes needed"
|
||||
else
|
||||
echo "📝 Formatting changes detected"
|
||||
echo "Changed files:"
|
||||
git diff --name-only
|
||||
|
||||
# 添加所有变更并提交
|
||||
git add .
|
||||
git commit -m "style: auto-format with ruff [skip ci]"
|
||||
|
||||
echo "📤 Pushing formatting changes..."
|
||||
git push
|
||||
echo "✅ Formatting changes committed and pushed"
|
||||
fi
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
.yuxi/
|
||||
|
||||
*.egg-info
|
||||
|
||||
### python gitignore
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.venv
|
||||
.ruff_cache
|
||||
|
||||
### Vue gitignore
|
||||
dist
|
||||
node_modules
|
||||
|
||||
### common gitignore
|
||||
.DS_Store
|
||||
.env
|
||||
.env.prod
|
||||
.env.test
|
||||
.env.local
|
||||
.env.*.local
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
*.log
|
||||
log
|
||||
logs
|
||||
*.log.*
|
||||
*.db
|
||||
*.lock
|
||||
!uv.lock
|
||||
tmp
|
||||
cache
|
||||
.cache
|
||||
.backup
|
||||
*.bak
|
||||
*._*
|
||||
|
||||
# test
|
||||
.pytest_cache
|
||||
.playwright
|
||||
.playwright-cli
|
||||
.playwright-mcp
|
||||
.opencode
|
||||
|
||||
|
||||
### IDE
|
||||
.vscode
|
||||
.idea
|
||||
.vibe
|
||||
.qoder
|
||||
.claude
|
||||
.cursor
|
||||
.trae
|
||||
.codex
|
||||
.sisyphus
|
||||
.pytest_cache
|
||||
|
||||
*.secret*
|
||||
*.nogit*
|
||||
*_private
|
||||
*.private
|
||||
# *.local* 保留用于本地配置文件
|
||||
*.local.py
|
||||
*.local.js
|
||||
*.local.yaml
|
||||
*.local/*
|
||||
|
||||
*.pdf
|
||||
src/data
|
||||
*/package-lock.json
|
||||
web/package-lock.json
|
||||
saves
|
||||
saves_dev
|
||||
notebooks
|
||||
graphrag
|
||||
docker/volumes
|
||||
docs/vibe
|
||||
.cursorrules
|
||||
|
||||
/models
|
||||
|
||||
.taskr/
|
||||
@@ -0,0 +1,139 @@
|
||||
|
||||
# 项目目录结构 (Project Overview)
|
||||
|
||||
Yuxi 是一个基于大模型的智能知识库与知识图谱智能体开发平台，融合了 RAG 技术与知识图谱技术，基于 LangGraph v1 + Vue.js + FastAPI + LightRAG 架构构建。项目完全通过 Docker Compose 进行管理，支持热重载开发。
|
||||
|
||||
架构代码地图见 [ARCHITECTURE.md](ARCHITECTURE.md)。修改不熟悉的模块前，先阅读其中的后端、前端、运行链路和架构不变量说明，再用符号搜索定位具体实现；该文档只维护相对稳定的系统边界，不替代细节文档或源码注释。
|
||||
|
||||
## 开发准则
|
||||
|
||||
Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed.
|
||||
|
||||
**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment.
|
||||
|
||||
## 1. Think Before Coding
|
||||
|
||||
**Don't assume. Don't hide confusion. Surface tradeoffs.**
|
||||
|
||||
Before implementing:
|
||||
- Restate the request as the smallest acceptance criteria you are about to satisfy. If you cannot state it simply, you do not understand the request yet.
|
||||
- State your assumptions explicitly. If uncertain, ask.
|
||||
- If multiple interpretations exist, present them - don't pick silently.
|
||||
- If a simpler approach exists, say so. Push back when warranted.
|
||||
- If something is unclear, stop. Name what's confusing. Ask.
|
||||
- Treat phrases like "可以", "也可以", "类似这样", or "for example" as acceptable simple directions, not permission to design a larger mechanism.
|
||||
|
||||
## 2. Simplicity First
|
||||
|
||||
**Minimum code that solves the problem. Nothing speculative.**
|
||||
|
||||
- No features beyond what was asked.
|
||||
- No abstractions for single-use code.
|
||||
- No "flexibility" or "configurability" that wasn't requested.
|
||||
- No error handling for impossible scenarios.
|
||||
- Do not fill in imagined requirements. If you start adding aggregation, priority rules, fallback layers, protocol interpreters, or generic frameworks that were not explicitly asked for, stop and reduce the solution to the acceptance criteria.
|
||||
- For small status/progress/summary changes, prefer a direct projection: read the source data, select the needed items, return the smallest useful shape. Do not rebuild an event stream or debug view unless that is the request.
|
||||
- If you write 200 lines and it could be 50, rewrite it.
|
||||
|
||||
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify.
|
||||
|
||||
## 3. Surgical Changes
|
||||
|
||||
**Touch only what you must. Clean up only your own mess.**
|
||||
|
||||
When editing existing code:
|
||||
- Don't "improve" adjacent code, comments, or formatting.
|
||||
- Don't refactor things that aren't broken.
|
||||
- Match existing style, even if you'd do it differently.
|
||||
- If you notice unrelated dead code, mention it - don't delete it.
|
||||
|
||||
When your changes create orphans:
|
||||
- Remove imports/variables/functions that YOUR changes made unused.
|
||||
- Don't remove pre-existing dead code unless asked.
|
||||
|
||||
The test: Every changed line should trace directly to the user's request.
|
||||
|
||||
## 4. Goal-Driven Execution
|
||||
|
||||
**Define success criteria. Loop until verified.**
|
||||
|
||||
Transform tasks into verifiable goals:
|
||||
- "Add validation" → "Write tests for invalid inputs, then make them pass"
|
||||
- "Fix the bug" → "Write a test that reproduces it, then make it pass"
|
||||
- "Refactor X" → "Ensure tests pass before and after"
|
||||
|
||||
For multi-step tasks, state a brief plan:
|
||||
```
|
||||
1. [Step] → verify: [check]
|
||||
2. [Step] → verify: [check]
|
||||
3. [Step] → verify: [check]
|
||||
```
|
||||
|
||||
Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification.
|
||||
|
||||
---
|
||||
|
||||
**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes.
|
||||
|
||||
## 代码 Review 准则
|
||||
|
||||
进行代码 Review 时，按以下顺序审查：
|
||||
|
||||
1. 首先确认代码是否能够完成基本功能，并覆盖主要使用场景；如果主路径或关键场景没有验证清楚，应优先指出。
|
||||
2. 审查当前实施方案是否是上下文中的最优解，是否会增加用户或维护者的理解负担；如果存在更简洁、更容易理解但改动面更大的方案，不要直接重写，先向用户说明取舍并确认。
|
||||
3. 检查是否存在过度设计、过度防御或过度嵌套：过度设计通常表现为加入无关功能；过度防御通常表现为用非预期的回退或保底掩盖设计问题；过度嵌套通常表现为 helper 过多、调用链绕、没有遵循从上到下的阅读顺序。
|
||||
4. 认真评估测试脚本和测试用例的价值。对繁琐但只是在“给出靶子后评估靶子”的低价值测试，应建议清理或合并；保留能验证真实行为、关键路径和回归风险的测试。
|
||||
|
||||
## 开发与调试工作流 (Development & Debugging Workflow)
|
||||
|
||||
本项目完全通过 Docker Compose 进行管理。所有开发和调试都应在运行的容器环境中进行。使用 `docker compose up -d` 命令进行构建和启动。
|
||||
|
||||
**核心原则**:
|
||||
|
||||
1. 由于 Compose 服务 `api` / `web`（容器名 `api-dev` / `web-dev`）均配置了热重载 (hot-reloading)，本地修改代码后无需重启容器，服务会自动更新。应该先检查项目是否已经在后台启动（`docker ps`），查看日志（`docker logs api-dev --tail 100`）具体的可以阅读 [docker-compose.yml](docker-compose.yml).
|
||||
2. 开发完成之后必须按改动范围进行 检查 -> 测试 -> Lint：相关单元测试必跑；涉及接口时跑集成测试；涉及关键主链路时补跑端到端测试。测试脚本不完善时应完善脚本。
|
||||
3. 测试规范务必遵守 [testing-guidelines.md](docs/develop-guides/testing-guidelines.md) 中的规范，测试脚本务必放在 backend/test/unit、backend/test/integration 或 backend/test/e2e 对应目录下，并且在提交前确保测试通过。
|
||||
4. 非常重要！千万不要使用过度的防御/回退机制来掩盖设计上的缺陷，良好的软件应该在预设的条件下运行，其余情况均应该及时发现问题/错误并修复，而不是通过增加冗余代码来掩盖问题。
|
||||
|
||||
### 需求沟通规范
|
||||
|
||||
在沟通需求的时候，当需求不明确的时候，需要主动挖掘需求细节，对齐需求的验收标准，明确需求的优先级和范围，避免模糊需求导致的过度设计和不必要的工作。
|
||||
|
||||
- 需求/修改 明确之后，如果改动较大，则需要在 docs/vibe 目录下创建一个包含日期的文档，记录需求的细节和验收标准
|
||||
- 该需求文档中，还应该包括本次任务的目标以及 checklist（简要）
|
||||
|
||||
### 前端开发规范
|
||||
- 使用 pnpm 管理
|
||||
- API 接口规范：所有的 API 接口都应该定义在 web/src/apis 下面
|
||||
- Icon 应该优先从 lucide-vue-next （推荐，但是需要注意尺寸）
|
||||
- 样式使用 less，非特殊情况必须使用 [base.css](web/src/assets/css/base.css) 中的颜色变量
|
||||
- UI 设计规范详见 [design](docs/develop-guides/design.md)
|
||||
|
||||
|
||||
### 后端开发规范
|
||||
|
||||
```bash
|
||||
# 代码检查和格式化
|
||||
make format # 格式化代码
|
||||
|
||||
```
|
||||
注意：
|
||||
- Python 代码要符合 pythonic 风格
|
||||
- 尽量使用较新的语法，避免使用旧版本的语法（版本兼容到 3.12+）
|
||||
- 更新 [changelog.md](docs/develop-guides/changelog.md) 文档记录本次修改，多个类似的功能更新已经补充在一起
|
||||
- 开发完成后务必在 docker 中进行测试，可以读取 .env 获取管理员账户和密码；敏感值仅用于本地测试命令，不要输出到回复、日志摘录、测试文件或文档中
|
||||
- 不允许把代码写得稀碎：不要为简单线性逻辑拆出一堆细碎 helper；优先写成职责清晰、结构完整、可一眼读懂的实现。
|
||||
- 拆函数必须服务于明确的复用、隔离副作用或降低认知负担；如果拆分后调用链更绕、上下文更分散，就应合并回更直接的实现。
|
||||
- 遵循向下规则（The Stepdown Rule）：公开的、高层次的方法放在文件顶部，细节逐层下沉。读者从上往下阅读时，每一层只调用紧接着的下一层实现，像读报纸标题一样逐级展开细节，无需跳跃。
|
||||
|
||||
**其他**：
|
||||
|
||||
- 如果需要新建说明文档（仅开发者可见，非必要不创建），则保存在 `docs/vibe` 文件夹下面
|
||||
- 代码更新后要检查文档部分是否有需要更新的地方，文档的目录定义在 `docs/.vitepress/config.mts` 中
|
||||
- 如果新增面向用户的正式文档，除了补正文档内容外，还需要同步更新 `docs/.vitepress/config.mts` 的导航；Langfuse 集成说明归档在 `docs/agents` 分组下维护，并同步更新 `docs/develop-guides/changelog.md`
|
||||
|
||||
## 提交规范
|
||||
|
||||
1. 参考 [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) 规范编写提交信息。
|
||||
2. 使用中文提交信息，标题简洁明了，描述具体改动内容和原因。
|
||||
3. 创建 PR 必须参考 [contributing.md](docs/develop-guides/contributing.md) 以及 PR 模板[PULL_REQUEST_TEMPLATE.md](.github/PULL_REQUEST_TEMPLATE.md)，并在提交前完成其中的检查项。
|
||||
@@ -0,0 +1,79 @@
|
||||
# ARCHITECTURE.md
|
||||
|
||||
本文档是 Yuxi 的代码地图，参考 matklad 的 `ARCHITECTURE.md` 建议维护：只描述相对稳定的系统边界、目录职责和跨切面约束，避免同步易变的实现细节。新贡献者如果不确定“某个能力应该改哪里”，先读这里，再用符号搜索定位具体类型、函数或路由。
|
||||
|
||||
## 鸟瞰
|
||||
|
||||
Yuxi 是一个面向 RAG、知识图谱和多智能体工作流的知识库平台。用户在 Vue 前端中配置智能体、知识库、工具、Skills、MCP 与 SubAgents；前端通过 `/api` 调用 FastAPI；后端服务层协调数据库、对象存储、向量库、图数据库、LangGraph 运行态和沙盒；长耗时智能体运行交给 worker 异步执行，并通过事件流回到前端。
|
||||
|
||||
开发与运行拓扑以 `docker-compose.yml` 为准。核心开发服务包括：
|
||||
|
||||
- `web-dev`：Vue/Vite 前端，挂载 `web/src` 并热重载。
|
||||
- `api-dev`：FastAPI API 服务，挂载 `backend/server` 和 `backend/package` 并热重载。
|
||||
- `worker-dev`：ARQ 后台任务 worker，处理智能体运行等异步任务。
|
||||
- `sandbox-provisioner`：为智能体工具执行提供沙盒环境。
|
||||
- `postgres`、`redis`、`minio`、`milvus`、`graph`：分别承载业务/知识库元数据、运行事件与队列状态、对象存储、向量检索、Neo4j 图谱。
|
||||
- `mineru-*`、`paddlex`：按 `all` profile 启动的文档解析/OCR 能力。
|
||||
|
||||
## 后端代码地图
|
||||
|
||||
后端分成两个顶层边界：`backend/server` 是 Web 应用入口与 HTTP 适配层，`backend/package/yuxi` 是可复用业务包。新增业务逻辑通常优先放在 `yuxi` 包中，路由层只做请求解析、认证上下文和响应装配。
|
||||
|
||||
- `server/main.py` 创建 FastAPI 应用，注册中间件，并把所有业务接口统一挂到 `/api`。
|
||||
- `server/routers` 是 HTTP 路由边界。路由按领域拆分，集中在 `server/routers/__init__.py` 注册；知识库、图谱、评估和思维导图接口在 `LITE_MODE` 下不会注册。
|
||||
- `server/utils` 放 Web 层通用能力，例如生命周期、认证、日志与迁移辅助。
|
||||
- `server/worker_main.py` 是 worker 入口，实际 worker 设置来自 `yuxi.services.run_worker`。
|
||||
|
||||
`backend/package/yuxi` 是后端主体：
|
||||
|
||||
- `agents` 定义 LangGraph 智能体体系。`BaseAgent` 是智能体基类，`BaseContext` 是运行配置上下文；`buildin` 放内置智能体；`middlewares` 负责把知识库、Skills、MCP、附件、运行配置等能力挂到运行时；`toolkits` 放工具注册与内置工具；`backends` 对接沙盒和 Skills 等外部执行/资源后端。
|
||||
- `services` 是用例层，负责串联 repositories、agents、knowledge、storage 和外部系统。聊天、运行队列、文件视图、Skills、MCP、SubAgents、评估等跨模块流程都从这里找入口。
|
||||
- `repositories` 是数据库访问边界，封装业务对象和知识库元数据的 SQLAlchemy 查询。不要让路由绕过 repository 直接操作模型，除非已有局部模式要求这样做。
|
||||
- `storage` 放持久化基础设施。`storage/postgres` 管理业务表、知识库表和 LangGraph checkpoint 所需连接池；`storage/minio` 管理对象存储。
|
||||
- `knowledge` 是知识库和图谱领域。`KnowledgeBaseManager` 根据知识库类型分发到具体实现；`implementations` 放 Milvus、Dify 等知识库实现；`graphs` 放 Milvus 知识库图谱适配与构建服务；`chunking` 放文档分块策略。
|
||||
- `knowledge/parser` 是文档解析边界，统一封装 MinerU、PaddleX、RapidOCR、DeepSeek OCR 等解析实现。
|
||||
- `models` 封装 chat、embedding、rerank 模型适配；`config` 维护应用配置和内置模型信息；`utils` 放跨领域但足够通用的工具。
|
||||
|
||||
测试代码放在 `backend/test`，按 `unit`、`integration`、`e2e` 分层组织。新增或修改后端行为时，测试应落在最能覆盖风险的那一层。
|
||||
|
||||
## 前端代码地图
|
||||
|
||||
前端是 Vue 3 + Vite 应用，业务入口集中在 `web/src`。
|
||||
|
||||
- `main.js` 挂载应用，`App.vue` 是根组件。
|
||||
- `router` 定义页面路由和权限跳转。普通用户默认进入智能体对话，图谱、知识库、仪表盘、扩展管理等页面带管理员权限约束。
|
||||
- `apis` 是唯一推荐的后端接口封装位置。新增后端接口时，同步在这里补对应 API 方法，复用 `base.js` 的请求、鉴权和错误处理。
|
||||
- `stores` 放 Pinia 状态，例如用户、智能体配置、主题、图谱和任务状态。
|
||||
- `views` 是页面级入口，`components` 是可复用界面块；智能体对话、知识库、图谱、扩展管理等复杂页面由 view 组合多个 component。
|
||||
- `composables` 放可组合的前端运行逻辑，例如流式消息处理、运行事件订阅、审批、人机输入和智能体线程状态。
|
||||
- `utils` 放前端通用工具和轻量转换逻辑；样式集中在 `assets/css`，颜色和基础规范优先复用 `base.css` 与现有 less 文件。
|
||||
|
||||
## 运行链路
|
||||
|
||||
一次典型智能体对话大致经过以下边界：
|
||||
|
||||
1. `AgentView` 及相关组件收集输入、附件和智能体配置。
|
||||
2. `web/src/apis` 调用 `/api/chat` 相关接口。
|
||||
3. `server/routers/chat_router.py` 进入后端，委托 `yuxi.services.chat_service` 或 `agent_run_service`。
|
||||
4. 服务层读取 conversation、agent config、tools、skills、knowledge 等配置，必要时创建后台 run。
|
||||
5. `worker-dev` 执行 LangGraph 智能体；中间件按上下文挂载知识库工具、Skills、MCP、附件与沙盒能力。
|
||||
6. 运行事件写入 Redis，最终状态和业务记录写入 Postgres；文件和产物落到 `saves`、MinIO 或沙盒用户数据目录。
|
||||
7. 前端通过 SSE/轮询消费运行事件，渲染消息、工具调用、引用来源、产物卡片和文件预览。
|
||||
|
||||
## 架构不变量
|
||||
|
||||
- Docker Compose 是开发环境的事实来源。开发时优先检查容器、日志和热重载，不要默认要求本地裸跑服务。
|
||||
- HTTP 路由层应保持薄；领域流程放在 `yuxi.services`，持久化查询放在 `yuxi.repositories`。
|
||||
- 前端 API 调用应集中在 `web/src/apis`，组件不要散落拼接后端 URL。
|
||||
- 智能体能力通过 context、middleware、toolkits、backends 组合；知识库通过工具访问，不要把知识库、MCP、Skills 或沙盒逻辑硬编码进单个页面或路由。
|
||||
- LITE 模式必须允许跳过知识库、图谱、评估等重依赖能力；新增相关接口或初始化逻辑时要尊重这个边界。
|
||||
- 沙盒虚拟路径以 `SANDBOX_VIRTUAL_PATH_PREFIX` 为边界，用户可见路径与宿主机真实路径不要混用。
|
||||
- 面向用户或外部系统的输入在边界校验；内部服务之间优先信任已有类型、仓储和框架约束，避免为了假设场景堆叠防御代码。
|
||||
|
||||
## 跨切面关注点
|
||||
|
||||
- **配置**：环境变量来自 Compose 和 `.env`，用户持久化配置由 `yuxi.config.app.Config` 管理，运行时配置通过智能体 context 进入 LangGraph。
|
||||
- **权限**：前端路由守卫提供页面级跳转，后端认证与权限检查仍是最终边界。
|
||||
- **状态与存储**：Postgres 存业务与知识库元数据，LangGraph checkpoint 使用独立连接池或 SQLite fallback，Redis 承载运行事件和取消信号，MinIO/本地 `saves`/沙盒目录承载文件。
|
||||
- **文档处理**：上传文件先进入解析和分块边界，再进入知识库实现；解析插件和知识库实现应保持可替换。
|
||||
- **观测与调试**：开发阶段优先使用 `docker logs api-dev --tail 100`、worker 日志和现有测试分层定位问题；Langfuse 相关逻辑集中在服务层和智能体运行配置附近。
|
||||
@@ -0,0 +1,139 @@
|
||||
|
||||
# 项目目录结构 (Project Overview)
|
||||
|
||||
Yuxi 是一个基于大模型的智能知识库与知识图谱智能体开发平台，融合了 RAG 技术与知识图谱技术，基于 LangGraph v1 + Vue.js + FastAPI + LightRAG 架构构建。项目完全通过 Docker Compose 进行管理，支持热重载开发。
|
||||
|
||||
架构代码地图见 [ARCHITECTURE.md](ARCHITECTURE.md)。修改不熟悉的模块前，先阅读其中的后端、前端、运行链路和架构不变量说明，再用符号搜索定位具体实现；该文档只维护相对稳定的系统边界，不替代细节文档或源码注释。
|
||||
|
||||
## 开发准则
|
||||
|
||||
Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed.
|
||||
|
||||
**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment.
|
||||
|
||||
## 1. Think Before Coding
|
||||
|
||||
**Don't assume. Don't hide confusion. Surface tradeoffs.**
|
||||
|
||||
Before implementing:
|
||||
- Restate the request as the smallest acceptance criteria you are about to satisfy. If you cannot state it simply, you do not understand the request yet.
|
||||
- State your assumptions explicitly. If uncertain, ask.
|
||||
- If multiple interpretations exist, present them - don't pick silently.
|
||||
- If a simpler approach exists, say so. Push back when warranted.
|
||||
- If something is unclear, stop. Name what's confusing. Ask.
|
||||
- Treat phrases like "可以", "也可以", "类似这样", or "for example" as acceptable simple directions, not permission to design a larger mechanism.
|
||||
|
||||
## 2. Simplicity First
|
||||
|
||||
**Minimum code that solves the problem. Nothing speculative.**
|
||||
|
||||
- No features beyond what was asked.
|
||||
- No abstractions for single-use code.
|
||||
- No "flexibility" or "configurability" that wasn't requested.
|
||||
- No error handling for impossible scenarios.
|
||||
- Do not fill in imagined requirements. If you start adding aggregation, priority rules, fallback layers, protocol interpreters, or generic frameworks that were not explicitly asked for, stop and reduce the solution to the acceptance criteria.
|
||||
- For small status/progress/summary changes, prefer a direct projection: read the source data, select the needed items, return the smallest useful shape. Do not rebuild an event stream or debug view unless that is the request.
|
||||
- If you write 200 lines and it could be 50, rewrite it.
|
||||
|
||||
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify.
|
||||
|
||||
## 3. Surgical Changes
|
||||
|
||||
**Touch only what you must. Clean up only your own mess.**
|
||||
|
||||
When editing existing code:
|
||||
- Don't "improve" adjacent code, comments, or formatting.
|
||||
- Don't refactor things that aren't broken.
|
||||
- Match existing style, even if you'd do it differently.
|
||||
- If you notice unrelated dead code, mention it - don't delete it.
|
||||
|
||||
When your changes create orphans:
|
||||
- Remove imports/variables/functions that YOUR changes made unused.
|
||||
- Don't remove pre-existing dead code unless asked.
|
||||
|
||||
The test: Every changed line should trace directly to the user's request.
|
||||
|
||||
## 4. Goal-Driven Execution
|
||||
|
||||
**Define success criteria. Loop until verified.**
|
||||
|
||||
Transform tasks into verifiable goals:
|
||||
- "Add validation" → "Write tests for invalid inputs, then make them pass"
|
||||
- "Fix the bug" → "Write a test that reproduces it, then make it pass"
|
||||
- "Refactor X" → "Ensure tests pass before and after"
|
||||
|
||||
For multi-step tasks, state a brief plan:
|
||||
```
|
||||
1. [Step] → verify: [check]
|
||||
2. [Step] → verify: [check]
|
||||
3. [Step] → verify: [check]
|
||||
```
|
||||
|
||||
Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification.
|
||||
|
||||
---
|
||||
|
||||
**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes.
|
||||
|
||||
## 代码 Review 准则
|
||||
|
||||
进行代码 Review 时，按以下顺序审查：
|
||||
|
||||
1. 首先确认代码是否能够完成基本功能，并覆盖主要使用场景；如果主路径或关键场景没有验证清楚，应优先指出。
|
||||
2. 审查当前实施方案是否是上下文中的最优解，是否会增加用户或维护者的理解负担；如果存在更简洁、更容易理解但改动面更大的方案，不要直接重写，先向用户说明取舍并确认。
|
||||
3. 检查是否存在过度设计、过度防御或过度嵌套：过度设计通常表现为加入无关功能；过度防御通常表现为用非预期的回退或保底掩盖设计问题；过度嵌套通常表现为 helper 过多、调用链绕、没有遵循从上到下的阅读顺序。
|
||||
4. 认真评估测试脚本和测试用例的价值。对繁琐但只是在“给出靶子后评估靶子”的低价值测试，应建议清理或合并；保留能验证真实行为、关键路径和回归风险的测试。
|
||||
|
||||
## 开发与调试工作流 (Development & Debugging Workflow)
|
||||
|
||||
本项目完全通过 Docker Compose 进行管理。所有开发和调试都应在运行的容器环境中进行。使用 `docker compose up -d` 命令进行构建和启动。
|
||||
|
||||
**核心原则**:
|
||||
|
||||
1. 由于 Compose 服务 `api` / `web`（容器名 `api-dev` / `web-dev`）均配置了热重载 (hot-reloading)，本地修改代码后无需重启容器，服务会自动更新。应该先检查项目是否已经在后台启动（`docker ps`），查看日志（`docker logs api-dev --tail 100`）具体的可以阅读 [docker-compose.yml](docker-compose.yml).
|
||||
2. 开发完成之后必须按改动范围进行 检查 -> 测试 -> Lint：相关单元测试必跑；涉及接口时跑集成测试；涉及关键主链路时补跑端到端测试。测试脚本不完善时应完善脚本。
|
||||
3. 测试规范务必遵守 [testing-guidelines.md](docs/develop-guides/testing-guidelines.md) 中的规范，测试脚本务必放在 backend/test/unit、backend/test/integration 或 backend/test/e2e 对应目录下，并且在提交前确保测试通过。
|
||||
4. 非常重要！千万不要使用过度的防御/回退机制来掩盖设计上的缺陷，良好的软件应该在预设的条件下运行，其余情况均应该及时发现问题/错误并修复，而不是通过增加冗余代码来掩盖问题。
|
||||
|
||||
### 需求沟通规范
|
||||
|
||||
在沟通需求的时候，当需求不明确的时候，需要主动挖掘需求细节，对齐需求的验收标准，明确需求的优先级和范围，避免模糊需求导致的过度设计和不必要的工作。
|
||||
|
||||
- 需求/修改 明确之后，如果改动较大，则需要在 docs/vibe 目录下创建一个包含日期的文档，记录需求的细节和验收标准
|
||||
- 该需求文档中，还应该包括本次任务的目标以及 checklist（简要）
|
||||
|
||||
### 前端开发规范
|
||||
- 使用 pnpm 管理
|
||||
- API 接口规范：所有的 API 接口都应该定义在 web/src/apis 下面
|
||||
- Icon 应该优先从 lucide-vue-next （推荐，但是需要注意尺寸）
|
||||
- 样式使用 less，非特殊情况必须使用 [base.css](web/src/assets/css/base.css) 中的颜色变量
|
||||
- UI 设计规范详见 [design](docs/develop-guides/design.md)
|
||||
|
||||
|
||||
### 后端开发规范
|
||||
|
||||
```bash
|
||||
# 代码检查和格式化
|
||||
make format # 格式化代码
|
||||
|
||||
```
|
||||
注意：
|
||||
- Python 代码要符合 pythonic 风格
|
||||
- 尽量使用较新的语法，避免使用旧版本的语法（版本兼容到 3.12+）
|
||||
- 更新 [changelog.md](docs/develop-guides/changelog.md) 文档记录本次修改，多个类似的功能更新已经补充在一起
|
||||
- 开发完成后务必在 docker 中进行测试，可以读取 .env 获取管理员账户和密码；敏感值仅用于本地测试命令，不要输出到回复、日志摘录、测试文件或文档中
|
||||
- 不允许把代码写得稀碎：不要为简单线性逻辑拆出一堆细碎 helper；优先写成职责清晰、结构完整、可一眼读懂的实现。
|
||||
- 拆函数必须服务于明确的复用、隔离副作用或降低认知负担；如果拆分后调用链更绕、上下文更分散，就应合并回更直接的实现。
|
||||
- 遵循向下规则（The Stepdown Rule）：公开的、高层次的方法放在文件顶部，细节逐层下沉。读者从上往下阅读时，每一层只调用紧接着的下一层实现，像读报纸标题一样逐级展开细节，无需跳跃。
|
||||
|
||||
**其他**：
|
||||
|
||||
- 如果需要新建说明文档（仅开发者可见，非必要不创建），则保存在 `docs/vibe` 文件夹下面
|
||||
- 代码更新后要检查文档部分是否有需要更新的地方，文档的目录定义在 `docs/.vitepress/config.mts` 中
|
||||
- 如果新增面向用户的正式文档，除了补正文档内容外，还需要同步更新 `docs/.vitepress/config.mts` 的导航；Langfuse 集成说明归档在 `docs/agents` 分组下维护，并同步更新 `docs/develop-guides/changelog.md`
|
||||
|
||||
## 提交规范
|
||||
|
||||
1. 参考 [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) 规范编写提交信息。
|
||||
2. 使用中文提交信息，标题简洁明了，描述具体改动内容和原因。
|
||||
3. 创建 PR 必须参考 [contributing.md](docs/develop-guides/contributing.md) 以及 PR 模板[PULL_REQUEST_TEMPLATE.md](.github/PULL_REQUEST_TEMPLATE.md)，并在提交前完成其中的检查项。
|
||||
@@ -0,0 +1,94 @@
|
||||
# Contributing to Yuxi
|
||||
|
||||
感谢你关注 Yuxi。欢迎提交 Issue、改进文档、修复 Bug 或贡献新功能。
|
||||
|
||||
更完整的开发文档可参考 [docs/develop-guides/contributing.md](docs/develop-guides/contributing.md)。
|
||||
|
||||
## 开始之前
|
||||
|
||||
- 提交前请先搜索现有 [Issues](https://github.com/xerrors/Yuxi/issues)
|
||||
- 对于较大的功能改动，建议先开 Issue 讨论方案
|
||||
- 保持改动聚焦，避免在一次 PR 中混入无关重构
|
||||
|
||||
## 开发方式
|
||||
|
||||
本项目通过 Docker Compose 进行开发，推荐直接在容器环境中调试。
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
docker ps
|
||||
docker logs api-dev --tail 100
|
||||
```
|
||||
|
||||
项目中的 `api-dev` 和 `web-dev` 默认支持热重载，本地修改代码后通常无需重启容器。
|
||||
|
||||
## 提交流程
|
||||
|
||||
1. Fork 仓库并创建分支
|
||||
2. 在对应目录完成开发与测试
|
||||
3. 提交清晰的 Commit Message
|
||||
4. 发起 Pull Request，并说明修改内容、原因和验证方式
|
||||
5. PR 模板 [PULL_REQUEST_TEMPLATE.md](.github/PULL_REQUEST_TEMPLATE.md) 中的检查项需要在提交前完成
|
||||
|
||||
示例：
|
||||
|
||||
```bash
|
||||
git checkout -b feature/your-change
|
||||
git commit -m "feat: add knowledge graph import flow"
|
||||
git push origin feature/your-change
|
||||
```
|
||||
|
||||
## 代码要求
|
||||
|
||||
### 通用
|
||||
|
||||
- 保持实现简单直接，避免过度设计
|
||||
- 只修改当前任务所需内容，不顺手做额外重构
|
||||
- 更新相关文档
|
||||
- 如有必要，同步更新 [docs/develop-guides/changelog.md](docs/develop-guides/changelog.md)
|
||||
- 设计部分请参考 [docs/develop-guides/design.md](docs/develop-guides/design.md)
|
||||
|
||||
### 后端
|
||||
|
||||
- 使用 Python 3.12+ 风格
|
||||
- 提交前运行：
|
||||
|
||||
```bash
|
||||
make format
|
||||
make lint
|
||||
docker compose exec api uv run pytest
|
||||
```
|
||||
|
||||
- 测试脚本建议放在 `backend/test`
|
||||
|
||||
### 前端
|
||||
|
||||
- 使用 `pnpm`
|
||||
- API 接口统一放在 `web/src/apis`
|
||||
- 优先使用 `lucide-vue-next` 图标
|
||||
- 样式使用 `less`
|
||||
- 非特殊情况优先复用 [web/src/assets/css/base.css](web/src/assets/css/base.css) 中的颜色变量
|
||||
|
||||
## Pull Request 建议
|
||||
|
||||
- 标题清晰，能说明变更目标
|
||||
- 描述中包含改动内容、影响范围和验证结果
|
||||
- 如果涉及 UI，请附截图或录屏
|
||||
- 如果涉及接口或行为变化，请补充文档
|
||||
|
||||
## 提交信息建议
|
||||
|
||||
推荐使用以下前缀：
|
||||
|
||||
- `feat`
|
||||
- `fix`
|
||||
- `docs`
|
||||
- `refactor`
|
||||
- `test`
|
||||
- `chore`
|
||||
|
||||
## 问题反馈
|
||||
|
||||
- Bug 反馈/功能讨论：<https://github.com/xerrors/Yuxi/issues>
|
||||
|
||||
感谢你的贡献 ❤️。
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 Yuxi Project Contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,54 @@
|
||||
|
||||
.PHONY: up up-lite down logs lint format seed reset
|
||||
|
||||
PYTEST_ARGS ?=
|
||||
BACKEND_PYTHON ?= $(shell cat backend/.python-version)
|
||||
|
||||
up:
|
||||
@if [ ! -f .env ]; then \
|
||||
echo "Error: .env file not found. Please create it from .env.template"; \
|
||||
exit 1; \
|
||||
fi
|
||||
docker compose up -d
|
||||
|
||||
down:
|
||||
docker compose down
|
||||
|
||||
reset:
|
||||
@if [ ! -f .env ]; then \
|
||||
echo "Error: .env file not found. Please create it from .env.template"; \
|
||||
exit 1; \
|
||||
fi
|
||||
docker compose down
|
||||
rm -rf docker/volumes
|
||||
docker compose up -d
|
||||
@echo "Waiting for api to be ready..."
|
||||
@until docker compose exec -T api true >/dev/null 2>&1; do sleep 2; done
|
||||
$(MAKE) seed
|
||||
|
||||
up-lite:
|
||||
@if [ ! -f .env ]; then \
|
||||
echo "Error: .env file not found. Please create it from .env.template"; \
|
||||
exit 1; \
|
||||
fi
|
||||
LITE_MODE=true VITE_USE_RUNS_API=false docker compose up -d postgres redis minio api web
|
||||
|
||||
logs:
|
||||
@docker logs --tail=50 api-dev
|
||||
@echo "\n\nBranch: $$(git branch --show-current)"
|
||||
@echo "Commit ID: $$(git rev-parse HEAD)"
|
||||
@echo "System: $$(uname -a)"
|
||||
|
||||
seed:
|
||||
docker compose exec api uv run python scripts/seed_initial_users.py
|
||||
|
||||
######################
|
||||
# LINTING AND FORMATTING
|
||||
######################
|
||||
|
||||
format:
|
||||
cd backend && UV_PYTHON=$(BACKEND_PYTHON) uv run ruff format package
|
||||
cd backend && UV_PYTHON=$(BACKEND_PYTHON) uv run ruff check package --fix
|
||||
cd backend && UV_PYTHON=$(BACKEND_PYTHON) uv run ruff check --select I package --fix
|
||||
cd web && pnpm run format
|
||||
cd web && pnpm run lint
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
<div align="center">
|
||||
<h1>Yuxi</h1>
|
||||
|
||||
<p><strong>A multi-tenant agent platform combining RAG and knowledge graphs</strong><br/>Make enterprise knowledge retrievable, reasoned over, and deliverable by agents</p>
|
||||
|
||||
[](https://github.com/xerrors/Yuxi/blob/main/docker-compose.yml)
|
||||
[](https://github.com/xerrors/Yuxi/issues)
|
||||
[](https://github.com/xerrors/Yuxi/blob/main/LICENSE)
|
||||
[](https://deepwiki.com/xerrors/Yuxi)
|
||||
[](https://zread.ai/xerrors/Yuxi)
|
||||
[](https://www.bilibili.com/video/BV1TZEx6NEit/)
|
||||
|
||||
<a href="https://trendshift.io/repositories/24335" target="_blank"><img src="https://trendshift.io/api/badge/repositories/24335" alt="xerrors%2FYuxi | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
|
||||
[[Docs]](https://xerrors.github.io/Yuxi) · [[中文]](README.md)
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
**Image generated by GPT-Image-2.*
|
||||
|
||||
## Introduction
|
||||
|
||||
Yuxi is an LLM-powered platform for building knowledge-base and knowledge-graph agents. It unifies **RAG retrieval**, **Milvus-backed in-knowledge-base graphs**, and **LangGraph multi-agent orchestration** into a single multi-tenant workspace: administrators configure knowledge bases, models, and permissions, while users chat — in a ChatGPT-like interface — with agents that can mount Skills, MCPs, sub-agents, and sandbox tools, and receive answers with cited sources, graph-based reasoning, and deliverable artifacts.
|
||||
|
||||
Navigation: [Introduction](https://xerrors.github.io/Yuxi/) ｜ [Quick Start](https://xerrors.github.io/Yuxi/intro/quick-start) ｜ [Roadmap](https://xerrors.github.io/Yuxi/develop-guides/roadmap); for the latest updates, see the [changelog](https://xerrors.github.io/Yuxi/develop-guides/changelog).
|
||||
|
||||
## Core Features
|
||||
|
||||
- 🤖 **Agent development** — Built on LangGraph, with sub-agents (SubAgents), Skills, MCPs, Tools, and middleware; long-running tasks run asynchronously on a background worker, backed by a sandbox file system for persisting, previewing, and downloading tool artifacts.
|
||||
- 📚 **Knowledge base (RAG)** — Multi-format document parsing (MinerU / PaddleX / OCR), configurable Embedding and Rerank models, knowledge base evaluation, in-app PDF / image preview, and retrieval sources backfilled as chat citations.
|
||||
- 🕸️ **Knowledge graph** — Build, visualize, and retrieve entity-relation graphs inside Milvus knowledge bases, then fuse graph hits with chunk retrieval for agent reasoning.
|
||||
- 🏢 **Multi-tenancy & permissions** — User / department-level access control, unified model provider configuration, and API Key authentication for external system integration.
|
||||
- ⚙️ **Platform & engineering** — Vue + FastAPI architecture, ready-to-run Docker Compose deployment, dark mode, a lightweight LITE startup mode, and production-grade orchestration.
|
||||
|
||||
## Tech Stack
|
||||
|
||||
| Layer | Technologies |
|
||||
| --- | --- |
|
||||
| Frontend | Vue 3 · Vite · Pinia |
|
||||
| Backend | FastAPI · LangGraph · ARQ (async worker) |
|
||||
| Storage | PostgreSQL · Redis · MinIO · Milvus · Neo4j |
|
||||
| Doc parsing | MinerU · PaddleX · RapidOCR |
|
||||
| Deployment | Docker Compose |
|
||||
|
||||
|
||||
|
||||
|
||||
## Quick Start
|
||||
|
||||
**Prerequisites**: [Docker](https://docs.docker.com/get-docker/) and Docker Compose installed, plus at least one OpenAI-compatible LLM API.
|
||||
|
||||
**1. Clone and initialize**
|
||||
|
||||
```bash
|
||||
git clone --branch v0.7.1.beta1 --depth 1 https://github.com/xerrors/Yuxi.git
|
||||
cd Yuxi
|
||||
|
||||
# Linux/macOS
|
||||
./scripts/init.sh
|
||||
|
||||
# Windows PowerShell
|
||||
.\scripts\init.ps1
|
||||
```
|
||||
|
||||
**2. Start with Docker**
|
||||
|
||||
```bash
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
**3. Open the platform**
|
||||
|
||||
Once the services are ready, open `http://localhost:5173` in your browser and sign in with the admin account generated during initialization.
|
||||
|
||||
> 💡 If you don't need heavy dependencies like knowledge bases / graphs, run `make up-lite` for a lightweight LITE mode with faster cold starts. See the [docs](https://xerrors.github.io/Yuxi) for more deployment details.
|
||||
|
||||
## Examples and Demo
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<img src="https://xerrors.oss-cn-shanghai.aliyuncs.com/github/image-20260326125852369.png" width="100%" alt="Home"/>
|
||||
<br/>
|
||||
<strong>Home</strong>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img src="https://github.com/user-attachments/assets/d3e4fe09-fa48-4686-93ea-2c50300ade21" width="100%" alt="Dashboard statistics"/>
|
||||
<br/>
|
||||
<strong>Dashboard Statistics</strong>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<img src="https://xerrors.oss-cn-shanghai.aliyuncs.com/github/image-20260326130528866.png" width="100%" alt="Agent configuration"/>
|
||||
<br/>
|
||||
<strong>Agent Configuration</strong>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img src="https://github.com/user-attachments/assets/06d56525-69bf-463a-8360-286b2cf8796f" width="100%" alt="Knowledge base invocation"/>
|
||||
<br/>
|
||||
<strong>Knowledge Base Invocation</strong>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<img src="https://github.com/user-attachments/assets/0548d89c-15a3-47cf-ba87-1b544f7dd749" width="100%" alt="Create knowledge base"/>
|
||||
<br/>
|
||||
<strong>Create Knowledge Base</strong>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img src="https://github.com/user-attachments/assets/21396d04-376b-4e9a-8139-eec8c3cc915a" width="100%" alt="Knowledge base management"/>
|
||||
<br/>
|
||||
<strong>Knowledge Base Management</strong>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<img src="https://github.com/user-attachments/assets/fc46a14b-16fb-47ea-84a0-148a451f3012" width="100%" alt="Knowledge graph"/>
|
||||
<br/>
|
||||
<strong>Knowledge Graph Visualization</strong>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img src="https://github.com/user-attachments/assets/d8b3de51-2854-455b-956f-2ae2d8d5f677" width="100%" alt="Project docs"/>
|
||||
<br/>
|
||||
<strong>Project Documentation</strong>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<img src="https://xerrors.oss-cn-shanghai.aliyuncs.com/github/image-20260326130404306.png" width="100%" alt="Skills management"/>
|
||||
<br/>
|
||||
<strong>Extension Management (Skills)</strong>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img src="https://github.com/user-attachments/assets/9305d7a4-663b-4e5d-a252-211d6caa019b" width="100%" alt="MCPs management"/>
|
||||
<br/>
|
||||
<strong>Extension Management (MCPs)</strong>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<img src="https://github.com/user-attachments/assets/13bd22ea-ddde-4262-8c29-69fb948bce44" width="100%" alt="User and department permissions"/>
|
||||
<br/>
|
||||
<strong>User / Department Permission Management</strong>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img src="https://github.com/user-attachments/assets/cc886b04-719e-4abd-807d-e9955080003d" width="100%" alt="Model provider configuration"/>
|
||||
<br/>
|
||||
<strong>Model Provider Configuration</strong>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## Acknowledgements
|
||||
|
||||
Yuxi references and builds on the following excellent open-source projects:
|
||||
|
||||
- [LightRAG](https://github.com/HKUDS/LightRAG) - Used as the foundation for graph construction and retrieval.
|
||||
- [DeepAgents](https://github.com/langchain-ai/deepagents) - Used as the deep agent framework.
|
||||
- [DeerFlow](https://github.com/bytedance/deer-flow) - Referenced for Sandbox agent architecture ideas.
|
||||
- [RAGflow](https://github.com/infiniflow/ragflow) - Referenced for document text chunking strategies.
|
||||
- [LangGraph](https://github.com/langchain-ai/langgraph) - Multi-agent orchestration framework and the core architectural foundation of this project.
|
||||
- [QwenPaw](https://github.com/agentscope-ai/QwenPaw) - Referenced for model configuration and personal file area design.
|
||||
|
||||
## Contributing
|
||||
|
||||
Thanks to all contributors for supporting this project!
|
||||
|
||||
<a href="https://github.com/xerrors/Yuxi/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=xerrors/Yuxi&max=100&columns=10" />
|
||||
</a>
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://star-history.com/#xerrors/Yuxi)
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the MIT License. See [LICENSE](LICENSE) for details.
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
|
||||
**If this project helps you, please give us a ⭐️.**
|
||||
|
||||
</div>
|
||||
@@ -0,0 +1,126 @@
|
||||
<div align="center">
|
||||
<h1>语析 Yuxi</h1>
|
||||
|
||||
<p><strong>多租户 Harness + 企业知识库</strong><br/>让企业知识可被智能体检索、推理与交付</p>
|
||||
|
||||
[](https://github.com/xerrors/Yuxi/blob/main/docker-compose.yml)
|
||||
[](https://github.com/xerrors/Yuxi/issues)
|
||||
[](https://github.com/xerrors/Yuxi/blob/main/LICENSE)
|
||||
[](https://deepwiki.com/xerrors/Yuxi)
|
||||
[](https://www.bilibili.com/video/BV1erE26iEgv/?share_source=copy_web&vd_source=37b0bdbf95b72ea38b2dc959cfadc4d8)
|
||||
|
||||
|
||||
<a href="https://trendshift.io/repositories/24335" target="_blank"><img src="https://trendshift.io/api/badge/repositories/24335" alt="xerrors%2FYuxi | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
|
||||
[[项目文档]](https://xerrors.github.io/Yuxi) · [[版本特性]](http://xhslink.com/o/5Y6QWnmjF2d) · [[🇬🇧 English README]](README.en.md)
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
## 简介
|
||||
|
||||
语析（Yuxi）是一个基于大模型的智能知识库与知识图谱智能体开发平台。它把 **RAG 检索**、**Milvus 知识库内知识图谱** 与 **LangGraph 多智能体编排** 整合进统一的多租户工作台：管理员配置知识库、模型与权限，用户在类 ChatGPT 的界面中与可挂载 Skills、MCP、子智能体和沙盒工具的智能体对话，并获得带引用来源、知识图谱推理与可交付产物的回答。
|
||||
|
||||
导航：[项目介绍](https://xerrors.github.io/Yuxi/) ｜ [快速开始](https://xerrors.github.io/Yuxi/intro/quick-start) ｜ [开发路线图](https://xerrors.github.io/Yuxi/develop-guides/roadmap) | [0.7 版本特性](http://xhslink.com/o/5Y6QWnmjF2d)；最新开发动态，详见 [changelog](https://xerrors.github.io/Yuxi/develop-guides/changelog)。
|
||||
|
||||
> 📢 求职：作者为江南大学软件工程博士研究生，研究方向 AI Agent、知识图谱与大模型应用，预计 2027 年毕业，现寻求实习/全职机会，欢迎联系：wenjie.zhang@stu.jiangnan.edu.cn
|
||||
|
||||
---
|
||||
|
||||
🩷 赞助商
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td style="width: 220px; padding: 8px 12px 8px 8px; vertical-align: middle;">
|
||||
<img
|
||||
width="220"
|
||||
height="64"
|
||||
alt="7fb163d0fb02740948521dbcaf6191ea"
|
||||
src="https://xerrors.oss-cn-shanghai.aliyuncs.com/github/image-20260623195812766.png"
|
||||
/>
|
||||
</td>
|
||||
<td style="padding: 8px 8px 8px 0; vertical-align: middle;">
|
||||
<p style="margin: 0 0 4px 0;">
|
||||
感谢 <a href="https://sui-xiang.com/">随想AI中转站</a > 对本项目的赞助！
|
||||
随想AI中转站 是一家可靠高效的 API 中继服务提供商，提供 Claude、Codex、Gemini 等的中继服务。注重隐私的中转站·无数据倒卖·无模型掺水，隐私，透明，极速售后。新账户注册每日签到就送 0.5 元测试额度，充值额度 1:1，无需订阅，按量付费。
|
||||
</p >
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
|
||||
## 技术栈
|
||||
|
||||
| 层 | 技术 |
|
||||
| --- | --- |
|
||||
| 前端 | Vue 3 · Vite · Pinia |
|
||||
| 后端 | FastAPI · LangGraph · ARQ (异步 worker) |
|
||||
| 存储 | PostgreSQL · Redis · MinIO · Milvus · Neo4j |
|
||||
| 文档解析 | MinerU · PaddleX · RapidOCR |
|
||||
| 部署 | Docker Compose |
|
||||
## 快速开始
|
||||
|
||||
**前置要求**：已安装 [Docker](https://docs.docker.com/get-docker/) 与 Docker Compose，并准备至少一个兼容 OpenAI 接口的大模型 API。
|
||||
|
||||
**1. 克隆代码并初始化**
|
||||
|
||||
```bash
|
||||
git clone --branch v0.7.1.beta1 --depth 1 https://github.com/xerrors/Yuxi.git
|
||||
cd Yuxi
|
||||
|
||||
# Linux/macOS
|
||||
./scripts/init.sh
|
||||
|
||||
# Windows PowerShell
|
||||
.\scripts\init.ps1
|
||||
```
|
||||
|
||||
**2. 使用 Docker 启动**
|
||||
|
||||
```bash
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
**3. 访问平台**
|
||||
|
||||
等待启动完成后，浏览器打开 `http://localhost:5173`，使用初始化时生成的管理员账户登录即可。
|
||||
|
||||
> 💡 不需要知识库 / 知识图谱等重依赖时，可使用 `make up-lite` 以 LITE 轻量模式启动，加快冷启动速度。更多部署说明见 [项目文档](https://xerrors.github.io/Yuxi)。
|
||||
|
||||
## 致谢
|
||||
|
||||
本项目参考并引用了以下优秀开源项目，在此致以诚挚的感谢：
|
||||
|
||||
- [LightRAG](https://github.com/HKUDS/LightRAG) - 早期版本曾参考其图谱构建与检索思路；当前 Yuxi 已实现自研 Milvus 知识库/图谱链路以替换历史集成，降低兼容性问题
|
||||
- [DeepAgents](https://github.com/langchain-ai/deepagents) - 直接引入作为深度智能体框架
|
||||
- [DeerFlow](https://github.com/bytedance/deer-flow) - 参考了其 Sandbox 智能体架构的实现思路
|
||||
- [RAGflow](https://github.com/infiniflow/ragflow) - 参考了其文档 Text Chunking 的分块策略
|
||||
- [LangGraph](https://github.com/langchain-ai/langgraph) - 多智能体编排框架，本项目的核心架构基础
|
||||
- [QwenPaw](https://github.com/agentscope-ai/QwenPaw) - 参考模型配置与个人文件区域设计
|
||||
|
||||
## 参与贡献
|
||||
|
||||
感谢所有贡献者的支持！
|
||||
|
||||
<a href="https://github.com/xerrors/Yuxi/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=xerrors/Yuxi&max=100&columns=10" />
|
||||
</a>
|
||||
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://star-history.com/#xerrors/Yuxi)
|
||||
|
||||
## 📄 许可证
|
||||
|
||||
本项目采用 MIT 许可证 - 查看 [LICENSE](LICENSE) 文件了解详情。
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
|
||||
**如果这个项目对您有帮助，请不要忘记给我们一个 ⭐️**
|
||||
|
||||
</div>
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`xerrors/Yuxi`
|
||||
- 原始仓库：https://github.com/xerrors/Yuxi
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
@@ -0,0 +1 @@
|
||||
3.13
|
||||
@@ -0,0 +1,3 @@
|
||||
recursive-include yuxi/config/static *.txt *.yaml
|
||||
recursive-include yuxi/agents/skills *.md
|
||||
recursive-include yuxi/agents/skills/buildin/mysql-reporter/scripts *.py
|
||||
@@ -0,0 +1,5 @@
|
||||
# yuxi
|
||||
|
||||
Yuxi 核心后端 Python 包。
|
||||
|
||||
该目录用于将 `yuxi` 作为本地可构建、可安装的 Python package 提供给 `backend` 项目使用。
|
||||
@@ -0,0 +1,118 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=80", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "yuxi"
|
||||
version = "0.7.1.beta1"
|
||||
description = "Yuxi 智能知识库与知识图谱平台核心后端包"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12,<3.14"
|
||||
dependencies = [
|
||||
"agent-sandbox>=0.0.26",
|
||||
"aioboto3>=13.0.0",
|
||||
"aiofiles>=24.1.0",
|
||||
"aiohttp>=3.14.1",
|
||||
"aiosqlite>=0.20.0",
|
||||
"argon2-cffi>=25.1.0",
|
||||
"asyncpg>=0.30.0",
|
||||
"chardet>=5.0.0",
|
||||
"colorlog>=6.9.0",
|
||||
"dashscope>=1.23.2",
|
||||
"deepagents>=0.6.7",
|
||||
"docling>=2.68.0",
|
||||
"docx2txt>=0.9",
|
||||
"httpx>=0.27.0",
|
||||
"igraph>=0.11.8",
|
||||
"json-repair>=0.54.0",
|
||||
"langchain>=1.3.9",
|
||||
"langchain-community>=0.4",
|
||||
"langchain-core>=1.2.5",
|
||||
"langchain-deepseek>=1.0",
|
||||
"langchain-mcp-adapters>=0.1.9",
|
||||
"langchain-openai>=1.0.2",
|
||||
"langchain-tavily>=0.2.13",
|
||||
"langchain-text-splitters>=1.0",
|
||||
"langfuse>=4.0.0",
|
||||
"langgraph>=1.0.1",
|
||||
"langgraph-checkpoint-postgres>=2.0.0",
|
||||
"langgraph-checkpoint-sqlite>=3.0",
|
||||
"langgraph-cli[inmem]>=0.4",
|
||||
"langsmith>=0.4",
|
||||
"llama-index>=0.14",
|
||||
"llama-index-readers-file>=0.4.7",
|
||||
"loguru>=0.7.3",
|
||||
"markdownify>=1.1.0",
|
||||
"mcp>=1.20",
|
||||
"minio>=7.2.7",
|
||||
"neo4j>=5.28.1",
|
||||
"networkx>=3.5",
|
||||
"openai>=1.109",
|
||||
"opencv-python-headless>=4.11.0.86",
|
||||
"onnxruntime>=1.20.0",
|
||||
"Pillow>=10.5.0",
|
||||
"psycopg[binary,pool]>=3.3.3",
|
||||
"pyjwt>=2.13.0",
|
||||
"pymilvus>=2.5.8",
|
||||
"pymupdf>=1.25.5",
|
||||
"pymysql>=1.1.0",
|
||||
"pypinyin>=0.55.0",
|
||||
"python-dotenv>=1.1.0",
|
||||
"python-multipart>=0.0.31",
|
||||
"pyyaml>=6.0.2",
|
||||
"rapidocr>=3.0.0",
|
||||
"readability-lxml>=0.8.1",
|
||||
"redis>=5.2.0",
|
||||
"requests>=2.34.0",
|
||||
"rich>=13.7.1",
|
||||
"sqlalchemy[asyncio]>=2.0.0",
|
||||
"tabulate>=0.9.0",
|
||||
"tavily-python>=0.7.0",
|
||||
"tenacity>=8.0.0",
|
||||
"tomli",
|
||||
"tomli-w",
|
||||
"torch>=2.12.1",
|
||||
"torchvision==0.27.1",
|
||||
"tqdm>=4.66.4",
|
||||
"typer>=0.16.0",
|
||||
"unstructured>=0.17.2",
|
||||
"wcmatch>=8.0.0",
|
||||
# Markdown 解析语义切分相关依赖
|
||||
"markdown-it-py>=3.0.0",
|
||||
"mdit-py-plugins>=0.4.0",
|
||||
"scikit-learn>=1.3.0",
|
||||
"nltk>=3.8.1",
|
||||
"beautifulsoup4>=4.12.0",
|
||||
]
|
||||
|
||||
[tool.setuptools]
|
||||
include-package-data = true
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["yuxi*"]
|
||||
|
||||
|
||||
[tool.uv]
|
||||
constraint-dependencies = [
|
||||
"cryptography>=48.0.1",
|
||||
"langchain-anthropic>=1.4.6",
|
||||
"pypdf>=6.13.3",
|
||||
"starlette>=1.3.1",
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "tsinghua"
|
||||
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
default = true
|
||||
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cpu"
|
||||
url = "https://download.pytorch.org/whl/cpu"
|
||||
explicit = true
|
||||
|
||||
[tool.uv.sources]
|
||||
yuxi = { path = "package", editable = true }
|
||||
torch = { index = "pytorch-cpu" }
|
||||
torchvision = { index = "pytorch-cpu" }
|
||||
Generated
+5472
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,33 @@
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(".env", override=True)
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor # noqa: E402
|
||||
from importlib import import_module # noqa: E402
|
||||
|
||||
from yuxi.config import config as config # noqa: E402
|
||||
|
||||
try:
|
||||
from importlib.metadata import version
|
||||
|
||||
__version__ = version("yuxi")
|
||||
except Exception:
|
||||
__version__ = "unknown"
|
||||
|
||||
executor = ThreadPoolExecutor() # noqa: E402
|
||||
|
||||
|
||||
def get_version():
|
||||
"""Return the Yuxi version."""
|
||||
return __version__
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "knowledge_base":
|
||||
knowledge = import_module("yuxi.knowledge")
|
||||
return getattr(knowledge, name)
|
||||
raise AttributeError(f"module 'yuxi' has no attribute {name!r}")
|
||||
|
||||
|
||||
def __dir__():
|
||||
return sorted(set(globals()) | {"knowledge_base"})
|
||||
@@ -0,0 +1,29 @@
|
||||
# Base classes - 核心基类
|
||||
from yuxi.agents.base import BaseAgent
|
||||
|
||||
# 从 buildin 模块导入 agent_manager
|
||||
from yuxi.agents.context import BaseContext
|
||||
|
||||
# MCP - Agent 层统一入口（自动过滤 disabled_tools）
|
||||
from yuxi.agents.mcp.service import get_enabled_mcp_tools
|
||||
|
||||
# Model utilities - 模型加载
|
||||
from yuxi.agents.models import load_chat_model, resolve_chat_model_spec
|
||||
from yuxi.agents.state import BaseState
|
||||
|
||||
# Tools - 核心工具函数
|
||||
from yuxi.agents.toolkits.utils import get_tool_info
|
||||
|
||||
__all__ = [
|
||||
# Base classes
|
||||
"BaseAgent",
|
||||
"BaseContext",
|
||||
"BaseState",
|
||||
# Model utilities
|
||||
"load_chat_model",
|
||||
"resolve_chat_model_spec",
|
||||
# Core tools
|
||||
"get_tool_info",
|
||||
# Core MCP
|
||||
"get_enabled_mcp_tools",
|
||||
]
|
||||
@@ -0,0 +1,49 @@
|
||||
from deepagents.backends import CompositeBackend, StateBackend
|
||||
|
||||
from .composite import create_agent_composite_backend, create_agent_filesystem_middleware
|
||||
from .knowledge_base_backend import resolve_visible_knowledge_bases_for_context
|
||||
from .sandbox import (
|
||||
SKILLS_PATH,
|
||||
USER_DATA_PATH,
|
||||
VIRTUAL_PATH_PREFIX,
|
||||
ProvisionerSandboxBackend,
|
||||
ProvisionerSandboxProvider,
|
||||
SandboxConnection,
|
||||
get_sandbox_provider,
|
||||
init_sandbox_provider,
|
||||
resolve_virtual_path,
|
||||
sandbox_id_for_thread,
|
||||
sandbox_outputs_dir,
|
||||
sandbox_uploads_dir,
|
||||
sandbox_user_data_dir,
|
||||
sandbox_workspace_dir,
|
||||
shutdown_sandbox_provider,
|
||||
virtual_path_for_thread_file,
|
||||
)
|
||||
from .skills_backend import SelectedSkillsReadonlyBackend
|
||||
|
||||
__all__ = [
|
||||
"CompositeBackend",
|
||||
"StateBackend",
|
||||
"SelectedSkillsReadonlyBackend",
|
||||
"create_agent_composite_backend",
|
||||
"create_agent_filesystem_middleware",
|
||||
"ProvisionerSandboxBackend",
|
||||
"ProvisionerSandboxProvider",
|
||||
"SandboxConnection",
|
||||
"get_sandbox_provider",
|
||||
"init_sandbox_provider",
|
||||
"shutdown_sandbox_provider",
|
||||
"resolve_virtual_path",
|
||||
"resolve_visible_knowledge_bases_for_context",
|
||||
"virtual_path_for_thread_file",
|
||||
"sandbox_id_for_thread",
|
||||
"sandbox_user_data_dir",
|
||||
"sandbox_workspace_dir",
|
||||
"sandbox_uploads_dir",
|
||||
"sandbox_outputs_dir",
|
||||
# Config paths
|
||||
"VIRTUAL_PATH_PREFIX",
|
||||
"USER_DATA_PATH",
|
||||
"SKILLS_PATH",
|
||||
]
|
||||
@@ -0,0 +1,201 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from deepagents.backends.composite import (
|
||||
CompositeBackend,
|
||||
_remap_file_info_path,
|
||||
_route_for_path,
|
||||
_strip_route_from_pattern,
|
||||
)
|
||||
from deepagents.backends.protocol import FileInfo, GlobResult
|
||||
from deepagents.middleware.filesystem import FilesystemMiddleware
|
||||
|
||||
from yuxi.agents.skills.service import normalize_string_list
|
||||
from yuxi.utils.paths import VIRTUAL_PATH_CONVERSATION_HISTORY, VIRTUAL_PATH_LARGE_TOOL_RESULTS, VIRTUAL_PATH_OUTPUTS
|
||||
|
||||
from .sandbox import ProvisionerSandboxBackend
|
||||
from .skills_backend import SelectedSkillsReadonlyBackend
|
||||
|
||||
_TOOL_RESULT_EVICTION_EXEMPT_TOOLS = frozenset({"read_file", "open_kb_document"})
|
||||
|
||||
|
||||
def _coerce_glob_result(result) -> GlobResult:
|
||||
if isinstance(result, GlobResult):
|
||||
return result
|
||||
return GlobResult(matches=result or [])
|
||||
|
||||
|
||||
class CustomCompositeBackend(CompositeBackend):
|
||||
"""修复 glob 路由逻辑的 CompositeBackend。"""
|
||||
|
||||
def glob(self, pattern: str, path: str = "/") -> GlobResult:
|
||||
backend, backend_path, route_prefix = _route_for_path(
|
||||
default=self.default,
|
||||
sorted_routes=self.sorted_routes,
|
||||
path=path,
|
||||
)
|
||||
if route_prefix is not None:
|
||||
result = _coerce_glob_result(backend.glob(pattern, backend_path))
|
||||
if result.error:
|
||||
return result
|
||||
return GlobResult(matches=[_remap_file_info_path(fi, route_prefix) for fi in (result.matches or [])])
|
||||
|
||||
if path is None or path == "/":
|
||||
results: list[FileInfo] = []
|
||||
default_result = _coerce_glob_result(self.default.glob(pattern, path))
|
||||
if default_result.error:
|
||||
return default_result
|
||||
results.extend(default_result.matches or [])
|
||||
for route_prefix, backend in self.routes.items():
|
||||
route_pattern = _strip_route_from_pattern(pattern, route_prefix)
|
||||
result = _coerce_glob_result(backend.glob(route_pattern, "/"))
|
||||
if result.error:
|
||||
return result
|
||||
results.extend(_remap_file_info_path(fi, route_prefix) for fi in (result.matches or []))
|
||||
results.sort(key=lambda x: x.get("path", ""))
|
||||
return GlobResult(matches=results)
|
||||
|
||||
return _coerce_glob_result(self.default.glob(pattern, path))
|
||||
|
||||
async def aglob(self, pattern: str, path: str = "/") -> GlobResult:
|
||||
backend, backend_path, route_prefix = _route_for_path(
|
||||
default=self.default,
|
||||
sorted_routes=self.sorted_routes,
|
||||
path=path,
|
||||
)
|
||||
if route_prefix is not None:
|
||||
result = _coerce_glob_result(await backend.aglob(pattern, backend_path))
|
||||
if result.error:
|
||||
return result
|
||||
return GlobResult(matches=[_remap_file_info_path(fi, route_prefix) for fi in (result.matches or [])])
|
||||
|
||||
if path is None or path == "/":
|
||||
results: list[FileInfo] = []
|
||||
default_result = _coerce_glob_result(await self.default.aglob(pattern, path))
|
||||
if default_result.error:
|
||||
return default_result
|
||||
results.extend(default_result.matches or [])
|
||||
for route_prefix, backend in self.routes.items():
|
||||
route_pattern = _strip_route_from_pattern(pattern, route_prefix)
|
||||
result = _coerce_glob_result(await backend.aglob(route_pattern, "/"))
|
||||
if result.error:
|
||||
return result
|
||||
results.extend(_remap_file_info_path(fi, route_prefix) for fi in (result.matches or []))
|
||||
results.sort(key=lambda x: x.get("path", ""))
|
||||
return GlobResult(matches=results)
|
||||
|
||||
return _coerce_glob_result(await self.default.aglob(pattern, path))
|
||||
|
||||
|
||||
class YuxiFilesystemMiddleware(FilesystemMiddleware):
|
||||
"""Filesystem middleware that budgets large tool outputs before they hit model context."""
|
||||
|
||||
def wrap_tool_call(self, request, handler):
|
||||
tool_result = handler(request)
|
||||
|
||||
if self._tool_token_limit_before_evict is None:
|
||||
return tool_result
|
||||
if request.tool_call["name"] in _TOOL_RESULT_EVICTION_EXEMPT_TOOLS:
|
||||
return tool_result
|
||||
|
||||
return self._intercept_large_tool_result(tool_result, request.runtime)
|
||||
|
||||
async def awrap_tool_call(self, request, handler):
|
||||
tool_result = await handler(request)
|
||||
|
||||
if self._tool_token_limit_before_evict is None:
|
||||
return tool_result
|
||||
if request.tool_call["name"] in _TOOL_RESULT_EVICTION_EXEMPT_TOOLS:
|
||||
return tool_result
|
||||
|
||||
return await self._aintercept_large_tool_result(tool_result, request.runtime)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _BackendScope:
|
||||
thread_id: str
|
||||
uid: str
|
||||
readable_skills: list[str]
|
||||
file_thread_id: str
|
||||
skills_thread_id: str
|
||||
|
||||
@classmethod
|
||||
def from_runtime(cls, runtime) -> _BackendScope:
|
||||
config = getattr(runtime, "config", None)
|
||||
configurable = config.get("configurable", {}) if isinstance(config, dict) else {}
|
||||
context = getattr(runtime, "context", None)
|
||||
state = getattr(runtime, "state", None)
|
||||
return cls.from_sources(
|
||||
configurable if isinstance(configurable, dict) else {},
|
||||
context,
|
||||
state if isinstance(state, dict) else {},
|
||||
readable_skills_source=context,
|
||||
error_context="runtime configurable context",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_sources(cls, *sources, readable_skills_source, error_context: str) -> _BackendScope:
|
||||
def string_value(key: str) -> str | None:
|
||||
for source in sources:
|
||||
value = source.get(key) if isinstance(source, dict) else getattr(source, key, None)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
thread_id = string_value("thread_id")
|
||||
if not thread_id:
|
||||
raise ValueError(f"thread_id is required in {error_context}")
|
||||
|
||||
uid = string_value("uid")
|
||||
if not uid:
|
||||
raise ValueError(f"uid is required in {error_context}")
|
||||
|
||||
selected = getattr(readable_skills_source, "_readable_skills", [])
|
||||
return cls(
|
||||
thread_id=thread_id,
|
||||
uid=uid,
|
||||
readable_skills=normalize_string_list(selected if isinstance(selected, list) else []),
|
||||
file_thread_id=string_value("file_thread_id") or thread_id,
|
||||
skills_thread_id=string_value("skills_thread_id") or thread_id,
|
||||
)
|
||||
|
||||
def create_backend(self) -> CompositeBackend:
|
||||
return CustomCompositeBackend(
|
||||
default=ProvisionerSandboxBackend(
|
||||
thread_id=self.thread_id,
|
||||
uid=self.uid,
|
||||
readable_skills=self.readable_skills,
|
||||
file_thread_id=self.file_thread_id,
|
||||
skills_thread_id=self.skills_thread_id,
|
||||
),
|
||||
routes={
|
||||
"/skills/": SelectedSkillsReadonlyBackend(selected_slugs=self.readable_skills),
|
||||
},
|
||||
artifacts_root=VIRTUAL_PATH_OUTPUTS,
|
||||
)
|
||||
|
||||
|
||||
def create_agent_composite_backend(runtime) -> CompositeBackend:
|
||||
return _BackendScope.from_runtime(runtime).create_backend()
|
||||
|
||||
|
||||
def create_agent_filesystem_middleware(
|
||||
tool_token_limit_before_evict: int | None = None,
|
||||
*,
|
||||
context=None,
|
||||
) -> FilesystemMiddleware:
|
||||
backend = create_agent_composite_backend
|
||||
if context is not None:
|
||||
backend = _BackendScope.from_sources(
|
||||
context,
|
||||
readable_skills_source=context,
|
||||
error_context="runtime context",
|
||||
).create_backend()
|
||||
middleware = YuxiFilesystemMiddleware(
|
||||
backend=backend,
|
||||
tool_token_limit_before_evict=tool_token_limit_before_evict,
|
||||
)
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
middleware._conversation_history_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
return middleware
|
||||
@@ -0,0 +1,22 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
async def resolve_visible_knowledge_bases_for_context(context) -> list[dict[str, Any]]:
|
||||
from yuxi import knowledge_base
|
||||
|
||||
uid = getattr(context, "uid", None)
|
||||
if not uid:
|
||||
setattr(context, "_visible_knowledge_bases", [])
|
||||
return []
|
||||
|
||||
result = await knowledge_base.get_databases_by_uid(str(uid))
|
||||
databases = result.get("databases") or []
|
||||
enabled_knowledges = getattr(context, "knowledges", None)
|
||||
if enabled_knowledges is not None:
|
||||
enabled_ids = {str(value).strip() for value in enabled_knowledges if str(value).strip()}
|
||||
databases = [db for db in databases if str(db.get("kb_id") or "").strip() in enabled_ids]
|
||||
|
||||
setattr(context, "_visible_knowledge_bases", databases)
|
||||
return databases
|
||||
@@ -0,0 +1,47 @@
|
||||
from .backend import ProvisionerSandboxBackend
|
||||
from .paths import (
|
||||
VIRTUAL_PATH_PREFIX,
|
||||
ensure_thread_dirs,
|
||||
ensure_workspace_default_files,
|
||||
resolve_virtual_path,
|
||||
sandbox_outputs_dir,
|
||||
sandbox_uploads_dir,
|
||||
sandbox_user_data_dir,
|
||||
sandbox_workspace_agents_prompt_file,
|
||||
sandbox_workspace_dir,
|
||||
virtual_path_for_thread_file,
|
||||
)
|
||||
from .provider import (
|
||||
ProvisionerSandboxProvider,
|
||||
SandboxConnection,
|
||||
get_sandbox_provider,
|
||||
init_sandbox_provider,
|
||||
sandbox_id_for_thread,
|
||||
shutdown_sandbox_provider,
|
||||
)
|
||||
|
||||
# Sandbox-visible paths for viewer/filesystem services.
|
||||
USER_DATA_PATH = VIRTUAL_PATH_PREFIX
|
||||
SKILLS_PATH = "/home/gem/skills"
|
||||
|
||||
__all__ = [
|
||||
"ProvisionerSandboxBackend",
|
||||
"ProvisionerSandboxProvider",
|
||||
"SandboxConnection",
|
||||
"VIRTUAL_PATH_PREFIX",
|
||||
"ensure_thread_dirs",
|
||||
"ensure_workspace_default_files",
|
||||
"get_sandbox_provider",
|
||||
"init_sandbox_provider",
|
||||
"resolve_virtual_path",
|
||||
"sandbox_id_for_thread",
|
||||
"sandbox_outputs_dir",
|
||||
"sandbox_uploads_dir",
|
||||
"sandbox_user_data_dir",
|
||||
"sandbox_workspace_agents_prompt_file",
|
||||
"sandbox_workspace_dir",
|
||||
"shutdown_sandbox_provider",
|
||||
"virtual_path_for_thread_file",
|
||||
"USER_DATA_PATH",
|
||||
"SKILLS_PATH",
|
||||
]
|
||||
@@ -0,0 +1,629 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import uuid
|
||||
from contextlib import suppress
|
||||
from datetime import datetime
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from deepagents.backends.protocol import (
|
||||
EditResult,
|
||||
ExecuteResponse,
|
||||
FileDownloadResponse,
|
||||
FileInfo,
|
||||
FileUploadResponse,
|
||||
GlobResult,
|
||||
GrepMatch,
|
||||
GrepResult,
|
||||
LsResult,
|
||||
ReadResult,
|
||||
WriteResult,
|
||||
)
|
||||
from deepagents.backends.sandbox import MAX_BINARY_BYTES, BaseSandbox
|
||||
from deepagents.backends.utils import _get_file_type
|
||||
|
||||
from yuxi import config as conf
|
||||
from yuxi.agents.skills.service import sync_thread_readable_skills
|
||||
from yuxi.utils.logging_config import logger
|
||||
from yuxi.utils.paths import (
|
||||
OUTPUTS_DIR_NAME,
|
||||
UPLOADS_DIR_NAME,
|
||||
VIRTUAL_PATH_PREFIX,
|
||||
VIRTUAL_SKILLS_PATH,
|
||||
WORKSPACE_DIR_NAME,
|
||||
)
|
||||
|
||||
from .provider import get_sandbox_provider, sandbox_id_for_thread
|
||||
|
||||
_USER_DATA_ROOT = "/" + VIRTUAL_PATH_PREFIX.strip("/")
|
||||
_WORKSPACE_ROOT = f"{_USER_DATA_ROOT}/{WORKSPACE_DIR_NAME}"
|
||||
_UPLOADS_ROOT = f"{_USER_DATA_ROOT}/{UPLOADS_DIR_NAME}"
|
||||
_OUTPUTS_ROOT = f"{_USER_DATA_ROOT}/{OUTPUTS_DIR_NAME}"
|
||||
_SKILLS_ROOT = "/" + VIRTUAL_SKILLS_PATH.strip("/")
|
||||
_READABLE_ROOTS = (_USER_DATA_ROOT, _SKILLS_ROOT)
|
||||
_WRITABLE_ROOTS = (_WORKSPACE_ROOT, _OUTPUTS_ROOT)
|
||||
_BINARY_PREVIEW_TOO_LARGE_ERROR = f"Binary file exceeds maximum preview size of {MAX_BINARY_BYTES} bytes"
|
||||
|
||||
|
||||
def _normalize_path(path: str) -> str:
|
||||
raw = str(path or "").strip()
|
||||
if not raw:
|
||||
raise ValueError("path is required")
|
||||
if not raw.startswith("/"):
|
||||
raise ValueError("path must start with /")
|
||||
pure = PurePosixPath(raw)
|
||||
if ".." in pure.parts:
|
||||
raise ValueError("path traversal is not allowed")
|
||||
return str(pure)
|
||||
|
||||
|
||||
def _is_same_or_child(path: str, root: str) -> bool:
|
||||
root = root.rstrip("/") or "/"
|
||||
if root == "/":
|
||||
return path == "/" or path.startswith("/")
|
||||
return path == root or path.startswith(f"{root}/")
|
||||
|
||||
|
||||
def _path_overlaps_root(path: str, root: str) -> bool:
|
||||
return _is_same_or_child(path, root) or _is_same_or_child(root, path)
|
||||
|
||||
|
||||
def _can_read_path(path: str) -> bool:
|
||||
return any(_is_same_or_child(path, root) for root in _READABLE_ROOTS)
|
||||
|
||||
|
||||
def _can_list_path(path: str) -> bool:
|
||||
return any(_path_overlaps_root(path, root) for root in _READABLE_ROOTS)
|
||||
|
||||
|
||||
def _can_write_path(path: str) -> bool:
|
||||
return any(_is_same_or_child(path, root) for root in _WRITABLE_ROOTS)
|
||||
|
||||
|
||||
def _readable_search_paths(path: str) -> list[str]:
|
||||
if _can_read_path(path):
|
||||
return [path]
|
||||
return [root for root in _READABLE_ROOTS if _is_same_or_child(root, path)]
|
||||
|
||||
|
||||
def _glob_for_search_root(pattern: str, root: str) -> str:
|
||||
bare_pattern = str(pattern or "*").lstrip("/")
|
||||
bare_root = root.strip("/")
|
||||
if bare_pattern == bare_root:
|
||||
return "*"
|
||||
root_prefix = f"{bare_root}/"
|
||||
if bare_pattern.startswith(root_prefix):
|
||||
return bare_pattern[len(root_prefix) :] or "*"
|
||||
return pattern
|
||||
|
||||
|
||||
def _filter_readable_infos(infos: list[FileInfo]) -> list[FileInfo]:
|
||||
result: list[FileInfo] = []
|
||||
for info in infos:
|
||||
try:
|
||||
path = _normalize_path(info.get("path", ""))
|
||||
except ValueError:
|
||||
continue
|
||||
if _can_list_path(path):
|
||||
result.append(info)
|
||||
return result
|
||||
|
||||
|
||||
def _filter_readable_matches(matches: list[GrepMatch]) -> list[GrepMatch]:
|
||||
result: list[GrepMatch] = []
|
||||
for match in matches:
|
||||
try:
|
||||
path = _normalize_path(match.get("path", ""))
|
||||
except ValueError:
|
||||
continue
|
||||
if _can_read_path(path):
|
||||
result.append(match)
|
||||
return result
|
||||
|
||||
|
||||
def _permission_error(operation: str, path: str) -> str:
|
||||
return f"permission denied for {operation} on '{path}'"
|
||||
|
||||
|
||||
def _describe_read_error(file_path: str, exc: Exception) -> str:
|
||||
if isinstance(exc, FileNotFoundError):
|
||||
return f"Error: File '{file_path}' not found"
|
||||
if isinstance(exc, IsADirectoryError):
|
||||
return f"Error: Path '{file_path}' is a directory"
|
||||
if isinstance(exc, PermissionError):
|
||||
return f"Error: Access denied for '{file_path}'"
|
||||
if isinstance(exc, ValueError):
|
||||
return f"Error: Invalid path '{file_path}': {exc}"
|
||||
detail = str(exc).strip()
|
||||
if detail:
|
||||
return f"Error: Failed to read '{file_path}': {detail}"
|
||||
return f"Error: Failed to read '{file_path}'"
|
||||
|
||||
|
||||
def _is_missing_file_error(exc: Exception) -> bool:
|
||||
if isinstance(exc, FileNotFoundError):
|
||||
return True
|
||||
|
||||
status_code = getattr(exc, "status_code", None)
|
||||
response = getattr(exc, "response", None)
|
||||
if status_code == 404 or getattr(response, "status_code", None) == 404:
|
||||
return True
|
||||
|
||||
detail = str(exc).lower()
|
||||
return "status_code: 404" in detail or "file does not exist" in detail
|
||||
|
||||
|
||||
def _looks_like_binary(content: bytes) -> bool:
|
||||
if not content:
|
||||
return False
|
||||
if b"\x00" in content:
|
||||
return True
|
||||
try:
|
||||
content.decode("utf-8")
|
||||
return False
|
||||
except UnicodeDecodeError:
|
||||
return True
|
||||
|
||||
|
||||
def _is_utf8_decode_failure(exc: Exception) -> bool:
|
||||
detail = str(exc).lower()
|
||||
return "utf-8" in detail and "can't decode" in detail
|
||||
|
||||
|
||||
class ProvisionerSandboxBackend(BaseSandbox):
|
||||
def __init__(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
uid: str,
|
||||
readable_skills: list[str] | None = None,
|
||||
file_thread_id: str | None = None,
|
||||
skills_thread_id: str | None = None,
|
||||
):
|
||||
self._thread_id = str(thread_id or "").strip()
|
||||
if not self._thread_id:
|
||||
raise ValueError("thread_id is required for ProvisionerSandboxBackend")
|
||||
self._file_thread_id = str(file_thread_id or self._thread_id).strip()
|
||||
if not self._file_thread_id:
|
||||
raise ValueError("file_thread_id is required for ProvisionerSandboxBackend")
|
||||
self._skills_thread_id = str(skills_thread_id or self._thread_id).strip()
|
||||
if not self._skills_thread_id:
|
||||
raise ValueError("skills_thread_id is required for ProvisionerSandboxBackend")
|
||||
self._uid = str(uid or "").strip()
|
||||
if not self._uid:
|
||||
raise ValueError("uid is required for ProvisionerSandboxBackend")
|
||||
|
||||
self._readable_skills = list(readable_skills or [])
|
||||
self._provider = get_sandbox_provider()
|
||||
self._id = sandbox_id_for_thread(self._file_thread_id, self._skills_thread_id, uid=self._uid)
|
||||
self._client: Any | None = None
|
||||
self._client_url: str | None = None
|
||||
self._command_timeout_seconds = int(getattr(conf, "sandbox_exec_timeout_seconds", 180))
|
||||
self._max_output_bytes = int(getattr(conf, "sandbox_max_output_bytes", 262_144))
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return self._id
|
||||
|
||||
def _build_client(self, sandbox_url: str):
|
||||
try:
|
||||
from agent_sandbox import Sandbox as AgentSandboxClient
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise RuntimeError(
|
||||
"agent-sandbox is required. Install dependency `agent-sandbox` in the docker image."
|
||||
) from exc
|
||||
|
||||
return AgentSandboxClient(base_url=sandbox_url, timeout=self._command_timeout_seconds)
|
||||
|
||||
def _get_client(self) -> Any:
|
||||
sync_thread_readable_skills(self._skills_thread_id, self._readable_skills)
|
||||
connection = self._provider.get(
|
||||
self._thread_id,
|
||||
uid=self._uid,
|
||||
create_if_missing=True,
|
||||
file_thread_id=self._file_thread_id,
|
||||
skills_thread_id=self._skills_thread_id,
|
||||
)
|
||||
if connection is None:
|
||||
raise RuntimeError(f"sandbox is unavailable for thread {self._thread_id}")
|
||||
|
||||
if self._client is None or self._client_url != connection.sandbox_url:
|
||||
self._client = self._build_client(connection.sandbox_url)
|
||||
self._client_url = connection.sandbox_url
|
||||
|
||||
return self._client
|
||||
|
||||
def _read_binary(self, path: str, offset: int = 0, limit: int | None = None) -> bytes:
|
||||
"""Read file content from the sandbox file API and normalize it to bytes.
|
||||
|
||||
The underlying API returns plain text by default and may include an
|
||||
explicit `encoding="base64"` marker for binary payloads. This helper is
|
||||
the single normalization point used by read(), edit(), and download_files().
|
||||
"""
|
||||
start_line = max(0, int(offset))
|
||||
end_line = start_line + int(limit) if limit is not None else None
|
||||
|
||||
result = self._get_client().file.read_file(
|
||||
file=path,
|
||||
start_line=start_line,
|
||||
end_line=end_line,
|
||||
)
|
||||
|
||||
content = result.data.content
|
||||
if content is None:
|
||||
return b""
|
||||
if isinstance(content, bytes):
|
||||
return content
|
||||
if not isinstance(content, str):
|
||||
return str(content).encode("utf-8")
|
||||
|
||||
encoding = getattr(result.data, "encoding", None)
|
||||
if isinstance(encoding, str) and encoding.lower() == "base64":
|
||||
return base64.b64decode(content, validate=True)
|
||||
return content.encode("utf-8")
|
||||
|
||||
def _file_size_bytes(self, path: str) -> int:
|
||||
path_b64 = base64.b64encode(path.encode("utf-8")).decode("ascii")
|
||||
command = (
|
||||
'python3 -c "'
|
||||
"import base64, os, stat; "
|
||||
f"path = base64.b64decode('{path_b64}').decode('utf-8'); "
|
||||
"st = os.stat(path); "
|
||||
"print(st.st_size if stat.S_ISREG(st.st_mode) else -1)"
|
||||
'"'
|
||||
)
|
||||
result = self.execute(command)
|
||||
if result.exit_code not in (0, None):
|
||||
detail = (result.output or "").strip()
|
||||
raise RuntimeError(detail or f"failed to stat '{path}'")
|
||||
|
||||
output = (result.output or "").strip().splitlines()
|
||||
try:
|
||||
size = int(output[-1])
|
||||
except (IndexError, ValueError) as exc:
|
||||
raise RuntimeError(f"failed to stat '{path}'") from exc
|
||||
if size < 0:
|
||||
raise IsADirectoryError(path)
|
||||
return size
|
||||
|
||||
def _read_file_base64(self, path: str) -> str:
|
||||
path_b64 = base64.b64encode(path.encode("utf-8")).decode("ascii")
|
||||
output_path = f"/tmp/yuxi-read-file-{uuid.uuid4().hex}.b64"
|
||||
output_path_b64 = base64.b64encode(output_path.encode("utf-8")).decode("ascii")
|
||||
command = (
|
||||
'python3 -c "'
|
||||
"import base64; "
|
||||
f"path = base64.b64decode('{path_b64}').decode('utf-8'); "
|
||||
f"output_path = base64.b64decode('{output_path_b64}').decode('utf-8'); "
|
||||
"open(output_path, 'w').write(base64.b64encode(open(path, 'rb').read()).decode('ascii'))"
|
||||
'"'
|
||||
)
|
||||
client = self._get_client()
|
||||
try:
|
||||
result = client.shell.exec_command(
|
||||
command=command,
|
||||
timeout=self._command_timeout_seconds,
|
||||
truncate=False,
|
||||
)
|
||||
output = result.data.output or ""
|
||||
if result.data.exit_code not in (0, None):
|
||||
raise RuntimeError(output.strip() or f"failed to read '{path}'")
|
||||
|
||||
content = self._read_binary(output_path).decode("ascii").strip()
|
||||
base64.b64decode(content, validate=True)
|
||||
return content
|
||||
finally:
|
||||
with suppress(Exception):
|
||||
client.shell.exec_command(command=f"rm -f {output_path}", timeout=10)
|
||||
|
||||
def _read_base64_file(self, path: str) -> ReadResult:
|
||||
if self._file_size_bytes(path) > MAX_BINARY_BYTES:
|
||||
return ReadResult(error=_BINARY_PREVIEW_TOO_LARGE_ERROR)
|
||||
return ReadResult(file_data={"content": self._read_file_base64(path), "encoding": "base64"})
|
||||
|
||||
def read(
|
||||
self,
|
||||
file_path: str,
|
||||
offset: int = 0,
|
||||
limit: int = 2000,
|
||||
) -> ReadResult:
|
||||
"""Read allowed file content via the sandbox file API."""
|
||||
try:
|
||||
normalized_path = _normalize_path(file_path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return ReadResult(error=f"Invalid path '{file_path}': {exc}")
|
||||
if not _can_read_path(normalized_path):
|
||||
return ReadResult(error=_permission_error("read", normalized_path))
|
||||
|
||||
try:
|
||||
if _get_file_type(normalized_path) != "text":
|
||||
return self._read_base64_file(normalized_path)
|
||||
|
||||
try:
|
||||
content = self._read_binary(normalized_path, offset=offset, limit=limit)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if not _is_utf8_decode_failure(exc):
|
||||
raise
|
||||
return self._read_base64_file(normalized_path)
|
||||
|
||||
if not _looks_like_binary(content):
|
||||
return ReadResult(file_data={"content": content.decode("utf-8"), "encoding": "utf-8"})
|
||||
|
||||
return self._read_base64_file(normalized_path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
error = _describe_read_error(file_path, exc)
|
||||
return ReadResult(error=error.removeprefix("Error: "))
|
||||
|
||||
def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse:
|
||||
"""Execute a shell command in the sandbox.
|
||||
|
||||
Output is normalized to text and truncated to the configured maximum
|
||||
payload size before being returned.
|
||||
"""
|
||||
try:
|
||||
kwargs: dict[str, Any] = {"command": command}
|
||||
if timeout is not None:
|
||||
kwargs["timeout"] = timeout
|
||||
result = self._get_client().shell.exec_command(**kwargs)
|
||||
|
||||
output = result.data.output or ""
|
||||
exit_code = result.data.exit_code
|
||||
|
||||
truncated = False
|
||||
encoded = output.encode("utf-8", errors="ignore")
|
||||
if len(encoded) > self._max_output_bytes:
|
||||
output = encoded[: self._max_output_bytes].decode("utf-8", errors="ignore")
|
||||
truncated = True
|
||||
|
||||
return ExecuteResponse(
|
||||
output=output,
|
||||
exit_code=exit_code if isinstance(exit_code, int) else None,
|
||||
truncated=truncated,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error(f"Sandbox execute failed for thread {self._thread_id}: {exc}")
|
||||
return ExecuteResponse(output=f"Error: {exc}", exit_code=1, truncated=False)
|
||||
|
||||
def ls(self, path: str) -> LsResult:
|
||||
"""List direct children of an allowed sandbox path with lightweight metadata."""
|
||||
try:
|
||||
normalized_path = _normalize_path(path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return LsResult(error=f"Invalid path '{path}': {exc}")
|
||||
if not _can_list_path(normalized_path):
|
||||
return LsResult(error=_permission_error("read", normalized_path))
|
||||
|
||||
try:
|
||||
result = self._get_client().file.list_path(path=normalized_path, recursive=False, include_size=True)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return LsResult(error=str(exc) or f"Failed to list '{path}'")
|
||||
|
||||
entries = result.data.files or []
|
||||
infos: list[FileInfo] = []
|
||||
for entry in entries:
|
||||
info: FileInfo = {"path": entry.path, "is_dir": entry.is_directory}
|
||||
size = entry.size
|
||||
if isinstance(size, int):
|
||||
info["size"] = size
|
||||
modified_time = entry.modified_time
|
||||
if modified_time:
|
||||
if isinstance(modified_time, str) and modified_time.isdigit():
|
||||
info["modified_at"] = datetime.fromtimestamp(int(modified_time)).isoformat()
|
||||
elif isinstance(modified_time, str):
|
||||
try:
|
||||
info["modified_at"] = datetime.fromisoformat(modified_time).isoformat()
|
||||
except ValueError:
|
||||
info["modified_at"] = modified_time
|
||||
elif isinstance(modified_time, (int, float)):
|
||||
info["modified_at"] = datetime.fromtimestamp(modified_time).isoformat()
|
||||
infos.append(info)
|
||||
return LsResult(entries=_filter_readable_infos(infos))
|
||||
|
||||
def write(self, file_path: str, content: str) -> WriteResult:
|
||||
"""Write a new text file.
|
||||
|
||||
This method is intentionally text-only. Binary payloads should go through
|
||||
upload_files(), which uses base64 encoding for the sandbox file API.
|
||||
"""
|
||||
try:
|
||||
normalized_path = _normalize_path(file_path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return WriteResult(error=f"Error: Invalid path '{file_path}': {exc}")
|
||||
if not _can_write_path(normalized_path):
|
||||
return WriteResult(error=f"Error: {_permission_error('write', normalized_path)}")
|
||||
if not isinstance(content, str):
|
||||
return WriteResult(error="Error: write() only supports text content; use upload_files() for binary data")
|
||||
try:
|
||||
self._read_binary(normalized_path)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
else:
|
||||
return WriteResult(error=f"Error: File '{file_path}' already exists")
|
||||
|
||||
try:
|
||||
result = self._get_client().file.write_file(file=normalized_path, content=content)
|
||||
if not result.success:
|
||||
return WriteResult(error=result.message or f"Failed to write file '{file_path}'")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return WriteResult(error=str(exc) or f"Failed to write file '{file_path}'")
|
||||
|
||||
return WriteResult(path=normalized_path)
|
||||
|
||||
def edit(
|
||||
self,
|
||||
file_path: str,
|
||||
old_string: str,
|
||||
new_string: str,
|
||||
replace_all: bool = False, # noqa: FBT001, FBT002
|
||||
) -> EditResult:
|
||||
"""Edit an existing text file by replacing string content.
|
||||
|
||||
This method operates on UTF-8-decoded text content only. Binary files
|
||||
are not supported here and should be handled via download/upload flows.
|
||||
"""
|
||||
try:
|
||||
normalized_path = _normalize_path(file_path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return EditResult(error=f"Error: Invalid path '{file_path}': {exc}")
|
||||
if not _can_write_path(normalized_path):
|
||||
return EditResult(error=f"Error: {_permission_error('write', normalized_path)}")
|
||||
|
||||
# Check if old_string exists
|
||||
try:
|
||||
text = self._read_binary(normalized_path).decode("utf-8", errors="replace")
|
||||
except Exception: # noqa: BLE001
|
||||
return EditResult(error=f"Error: File '{file_path}' not found")
|
||||
|
||||
count = text.count(old_string)
|
||||
if count == 0:
|
||||
return EditResult(error=f"Error: String not found in file: '{old_string}'")
|
||||
if count > 1 and not replace_all:
|
||||
return EditResult(
|
||||
error=(
|
||||
f"Error: String '{old_string}' appears multiple times. "
|
||||
"Use replace_all=True to replace all occurrences."
|
||||
)
|
||||
)
|
||||
|
||||
# Use str_replace_editor API
|
||||
replace_mode = "ALL" if replace_all else "FIRST"
|
||||
try:
|
||||
result = self._get_client().file.str_replace_editor(
|
||||
command="str_replace",
|
||||
path=normalized_path,
|
||||
old_str=old_string,
|
||||
new_str=new_string,
|
||||
replace_mode=replace_mode,
|
||||
)
|
||||
if not result.success:
|
||||
return EditResult(error=result.message or f"Error editing file '{file_path}'")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return EditResult(error=f"Error editing file: {exc}")
|
||||
|
||||
return EditResult(path=normalized_path, occurrences=count if replace_all else 1)
|
||||
|
||||
def grep(
|
||||
self,
|
||||
pattern: str,
|
||||
path: str | None = None,
|
||||
glob: str | None = None,
|
||||
) -> GrepResult:
|
||||
"""Search allowed sandbox paths for literal text."""
|
||||
try:
|
||||
normalized_path = _normalize_path(path or "/")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return GrepResult(error=f"Invalid path '{path or '/'}': {exc}")
|
||||
|
||||
search_paths = _readable_search_paths(normalized_path)
|
||||
if not search_paths:
|
||||
return GrepResult(error=_permission_error("read", normalized_path))
|
||||
|
||||
matches: list[GrepMatch] = []
|
||||
for search_path in search_paths:
|
||||
result = super().grep(pattern=pattern, path=search_path, glob=glob)
|
||||
if result.error:
|
||||
return result
|
||||
matches.extend(result.matches or [])
|
||||
return GrepResult(matches=_filter_readable_matches(matches))
|
||||
|
||||
def glob(self, pattern: str, path: str = "/") -> GlobResult:
|
||||
"""Return files matching a glob pattern under allowed sandbox paths."""
|
||||
try:
|
||||
normalized_path = _normalize_path(path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return GlobResult(error=f"Invalid path '{path}': {exc}")
|
||||
if ".." in PurePosixPath(str(pattern or "")).parts:
|
||||
return GlobResult(error="Invalid glob pattern: path traversal is not allowed")
|
||||
|
||||
search_paths = _readable_search_paths(normalized_path)
|
||||
if not search_paths:
|
||||
return GlobResult(error=_permission_error("read", normalized_path))
|
||||
|
||||
infos: list[FileInfo] = []
|
||||
for search_path in search_paths:
|
||||
try:
|
||||
result = self._get_client().file.find_files(
|
||||
path=search_path,
|
||||
glob=_glob_for_search_root(pattern, search_path),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return GlobResult(error=str(exc) or f"Failed to glob '{path}'")
|
||||
for file_path in result.data.files or []:
|
||||
infos.append({"path": file_path})
|
||||
infos = _filter_readable_infos(infos)
|
||||
infos.sort(key=lambda item: item.get("path", ""))
|
||||
return GlobResult(matches=infos)
|
||||
|
||||
def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
|
||||
"""Upload binary or text file payloads via the sandbox file API.
|
||||
|
||||
Contents are base64-encoded before calling the remote write_file API so
|
||||
arbitrary bytes can be transferred safely.
|
||||
"""
|
||||
responses: list[FileUploadResponse] = []
|
||||
for path, content in files:
|
||||
try:
|
||||
normalized_path = _normalize_path(path)
|
||||
if not _can_write_path(normalized_path):
|
||||
responses.append(FileUploadResponse(path=normalized_path, error="permission_denied"))
|
||||
continue
|
||||
result = self._get_client().file.write_file(
|
||||
file=normalized_path,
|
||||
content=base64.b64encode(content).decode("ascii"),
|
||||
encoding="base64",
|
||||
)
|
||||
if not result.success:
|
||||
raise Exception(result.message or "Upload failed")
|
||||
responses.append(FileUploadResponse(path=normalized_path, error=None))
|
||||
except PermissionError:
|
||||
normalized_path = str(path)
|
||||
responses.append(FileUploadResponse(path=normalized_path, error="permission_denied"))
|
||||
except IsADirectoryError:
|
||||
normalized_path = str(path)
|
||||
responses.append(FileUploadResponse(path=normalized_path, error="is_directory"))
|
||||
except FileNotFoundError:
|
||||
normalized_path = str(path)
|
||||
responses.append(FileUploadResponse(path=normalized_path, error="file_not_found"))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
normalized_path = str(path)
|
||||
logger.warning(f"Upload to sandbox failed for {normalized_path}: {exc}")
|
||||
responses.append(FileUploadResponse(path=normalized_path, error="invalid_path"))
|
||||
return responses
|
||||
|
||||
def download_files(self, paths: list[str]) -> list[FileDownloadResponse]:
|
||||
"""Download file payloads as raw bytes from the sandbox file API.
|
||||
|
||||
_read_binary() normalizes the sandbox file API response to bytes.
|
||||
"""
|
||||
responses: list[FileDownloadResponse] = []
|
||||
for path in paths:
|
||||
try:
|
||||
normalized_path = _normalize_path(path)
|
||||
if not _can_read_path(normalized_path):
|
||||
responses.append(
|
||||
FileDownloadResponse(path=normalized_path, content=None, error="permission_denied")
|
||||
)
|
||||
continue
|
||||
content = self._read_binary(normalized_path)
|
||||
responses.append(FileDownloadResponse(path=normalized_path, content=content, error=None))
|
||||
except PermissionError:
|
||||
normalized_path = str(path)
|
||||
responses.append(FileDownloadResponse(path=normalized_path, content=None, error="permission_denied"))
|
||||
except IsADirectoryError:
|
||||
normalized_path = str(path)
|
||||
responses.append(FileDownloadResponse(path=normalized_path, content=None, error="is_directory"))
|
||||
except FileNotFoundError:
|
||||
normalized_path = str(path)
|
||||
responses.append(FileDownloadResponse(path=normalized_path, content=None, error="file_not_found"))
|
||||
except ValueError:
|
||||
normalized_path = str(path)
|
||||
responses.append(FileDownloadResponse(path=normalized_path, content=None, error="invalid_path"))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
normalized_path = str(path)
|
||||
if _is_missing_file_error(exc):
|
||||
responses.append(FileDownloadResponse(path=normalized_path, content=None, error="file_not_found"))
|
||||
continue
|
||||
logger.warning(f"Download from sandbox failed for {normalized_path}: {exc}")
|
||||
responses.append(FileDownloadResponse(path=normalized_path, content=None, error=f"read_failed: {exc}"))
|
||||
return responses
|
||||
@@ -0,0 +1,196 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from yuxi import config as conf
|
||||
from yuxi.utils.logging_config import logger
|
||||
from yuxi.utils.paths import (
|
||||
OUTPUTS_DIR_NAME,
|
||||
UPLOADS_DIR_NAME,
|
||||
VIRTUAL_PATH_PREFIX,
|
||||
WORKSPACE_AGENTS_DIR_NAME,
|
||||
WORKSPACE_AGENTS_PROMPT_FILE_NAME,
|
||||
WORKSPACE_DIR_NAME,
|
||||
)
|
||||
|
||||
_SAFE_ID_RE = re.compile(r"^[A-Za-z0-9_-]+$")
|
||||
|
||||
|
||||
def get_virtual_path_prefix() -> str:
|
||||
return "/" + VIRTUAL_PATH_PREFIX.strip("/")
|
||||
|
||||
|
||||
def validate_thread_id(thread_id: str) -> str:
|
||||
value = str(thread_id or "").strip()
|
||||
if not value:
|
||||
raise ValueError("thread_id is required")
|
||||
if not _SAFE_ID_RE.match(value):
|
||||
raise ValueError("thread_id contains invalid characters")
|
||||
return value
|
||||
|
||||
|
||||
def _thread_root_dir(thread_id: str) -> Path:
|
||||
safe_thread_id = validate_thread_id(thread_id)
|
||||
return Path(conf.save_dir) / "threads" / safe_thread_id / "user-data"
|
||||
|
||||
|
||||
def _validate_uid(uid: str) -> str:
|
||||
value = str(uid or "").strip()
|
||||
if not value:
|
||||
raise ValueError("uid is required")
|
||||
if not _SAFE_ID_RE.match(value):
|
||||
raise ValueError("uid contains invalid characters")
|
||||
return value
|
||||
|
||||
|
||||
def _global_user_data_dir(uid: str) -> Path:
|
||||
"""Return the shared host-side directory used for one user's workspace files."""
|
||||
safe_uid = _validate_uid(uid)
|
||||
return Path(conf.save_dir) / "threads" / "shared" / safe_uid
|
||||
|
||||
|
||||
def sandbox_user_data_dir(thread_id: str) -> Path:
|
||||
return _thread_root_dir(thread_id)
|
||||
|
||||
|
||||
def sandbox_workspace_dir(thread_id: str, uid: str) -> Path:
|
||||
validate_thread_id(thread_id)
|
||||
return _global_user_data_dir(uid) / WORKSPACE_DIR_NAME
|
||||
|
||||
|
||||
def sandbox_workspace_agents_prompt_file(thread_id: str, uid: str) -> Path:
|
||||
return sandbox_workspace_dir(thread_id, uid) / WORKSPACE_AGENTS_DIR_NAME / WORKSPACE_AGENTS_PROMPT_FILE_NAME
|
||||
|
||||
|
||||
def _threads_root_dir() -> Path:
|
||||
return (Path(conf.save_dir) / "threads").resolve(strict=False)
|
||||
|
||||
|
||||
def _resolve_threads_child_path(path: Path) -> Path:
|
||||
root = _threads_root_dir()
|
||||
resolved = path.resolve(strict=False)
|
||||
if not resolved.is_relative_to(root):
|
||||
raise ValueError("path resolved outside threads root")
|
||||
return resolved
|
||||
|
||||
|
||||
def _chmod_writable(path: Path, *, dir: bool = False) -> None:
|
||||
safe_path = _resolve_threads_child_path(path)
|
||||
mode = 0o777 if dir else 0o666
|
||||
try:
|
||||
safe_path.chmod(mode)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def ensure_workspace_default_files(workspace_dir: Path) -> None:
|
||||
workspace_dir = _resolve_threads_child_path(workspace_dir)
|
||||
agents_dir = workspace_dir / WORKSPACE_AGENTS_DIR_NAME
|
||||
agents_file = agents_dir / WORKSPACE_AGENTS_PROMPT_FILE_NAME
|
||||
|
||||
try:
|
||||
agents_dir.mkdir(parents=True, exist_ok=True)
|
||||
_chmod_writable(agents_dir, dir=True)
|
||||
except FileExistsError:
|
||||
logger.warning("工作区默认 Agents 目录创建失败：路径已被文件占用")
|
||||
return
|
||||
except OSError as exc:
|
||||
logger.warning(f"工作区默认 Agents 目录初始化失败: {exc}")
|
||||
return
|
||||
|
||||
try:
|
||||
with agents_file.open("xb"):
|
||||
pass
|
||||
_chmod_writable(agents_file)
|
||||
except FileExistsError:
|
||||
if agents_file.is_dir():
|
||||
logger.warning("工作区默认 AGENTS.md 创建失败：路径已被目录占用")
|
||||
except OSError as exc:
|
||||
logger.warning(f"工作区默认 Agents 文件初始化失败: {exc}")
|
||||
|
||||
|
||||
def sandbox_uploads_dir(thread_id: str) -> Path:
|
||||
return _thread_root_dir(thread_id) / UPLOADS_DIR_NAME
|
||||
|
||||
|
||||
def sandbox_outputs_dir(thread_id: str) -> Path:
|
||||
return _thread_root_dir(thread_id) / OUTPUTS_DIR_NAME
|
||||
|
||||
|
||||
def ensure_thread_dirs(thread_id: str, uid: str) -> None:
|
||||
_resolve_threads_child_path(_global_user_data_dir(uid)).mkdir(parents=True, exist_ok=True)
|
||||
workspace_dir = _resolve_threads_child_path(sandbox_workspace_dir(thread_id, uid))
|
||||
workspace_dir.mkdir(parents=True, exist_ok=True)
|
||||
ensure_workspace_default_files(workspace_dir)
|
||||
_resolve_threads_child_path(sandbox_uploads_dir(thread_id)).mkdir(parents=True, exist_ok=True)
|
||||
_resolve_threads_child_path(sandbox_outputs_dir(thread_id)).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _resolve_user_data_base_dir(thread_id: str, uid: str, relative_path: str) -> tuple[Path, Path]:
|
||||
"""Map a virtual user-data path to the correct host-side base directory."""
|
||||
parts = Path(relative_path).parts
|
||||
if not parts:
|
||||
base_dir = sandbox_user_data_dir(thread_id)
|
||||
return base_dir.resolve(), base_dir.resolve()
|
||||
|
||||
namespace = parts[0]
|
||||
if namespace == WORKSPACE_DIR_NAME:
|
||||
# Workspace is shared across one user's threads, so it lives outside the per-thread root.
|
||||
base_dir = sandbox_workspace_dir(thread_id, uid)
|
||||
target_path = base_dir.joinpath(*parts[1:]) if len(parts) > 1 else base_dir
|
||||
return base_dir.resolve(), target_path.resolve()
|
||||
if namespace == UPLOADS_DIR_NAME:
|
||||
base_dir = sandbox_uploads_dir(thread_id)
|
||||
target_path = base_dir.joinpath(*parts[1:]) if len(parts) > 1 else base_dir
|
||||
return base_dir.resolve(), target_path.resolve()
|
||||
if namespace == OUTPUTS_DIR_NAME:
|
||||
base_dir = sandbox_outputs_dir(thread_id)
|
||||
target_path = base_dir.joinpath(*parts[1:]) if len(parts) > 1 else base_dir
|
||||
return base_dir.resolve(), target_path.resolve()
|
||||
|
||||
base_dir = sandbox_user_data_dir(thread_id)
|
||||
return base_dir.resolve(), (base_dir / relative_path).resolve()
|
||||
|
||||
|
||||
def resolve_virtual_path(thread_id: str, virtual_path: str, *, uid: str) -> Path:
|
||||
clean_virtual_path = "/" + str(virtual_path or "").strip().lstrip("/")
|
||||
virtual_prefix = get_virtual_path_prefix()
|
||||
|
||||
if clean_virtual_path != virtual_prefix and not clean_virtual_path.startswith(f"{virtual_prefix}/"):
|
||||
raise ValueError(f"path must start with {virtual_prefix}")
|
||||
|
||||
relative_path = clean_virtual_path[len(virtual_prefix) :].lstrip("/")
|
||||
base_dir, target_path = _resolve_user_data_base_dir(thread_id, uid, relative_path)
|
||||
|
||||
try:
|
||||
target_path.relative_to(base_dir)
|
||||
except ValueError as exc:
|
||||
raise ValueError("path traversal detected") from exc
|
||||
|
||||
return target_path
|
||||
|
||||
|
||||
def virtual_path_for_thread_file(thread_id: str, path: str | Path, *, uid: str) -> str:
|
||||
target_path = Path(path).resolve()
|
||||
thread_root = sandbox_user_data_dir(thread_id).resolve()
|
||||
global_workspace_root = sandbox_workspace_dir(thread_id, uid).resolve()
|
||||
|
||||
try:
|
||||
relative_path = target_path.relative_to(global_workspace_root)
|
||||
except ValueError:
|
||||
try:
|
||||
relative_path = target_path.relative_to(thread_root)
|
||||
except ValueError as exc:
|
||||
raise ValueError("file is outside allowed user-data directories") from exc
|
||||
relative_path_str = relative_path.as_posix()
|
||||
else:
|
||||
workspace_relative = relative_path.as_posix()
|
||||
relative_path_str = (
|
||||
WORKSPACE_DIR_NAME if workspace_relative in {"", "."} else f"{WORKSPACE_DIR_NAME}/{workspace_relative}"
|
||||
)
|
||||
|
||||
prefix = get_virtual_path_prefix().rstrip("/")
|
||||
if not relative_path_str:
|
||||
return prefix
|
||||
return f"{prefix}/{relative_path_str}"
|
||||
@@ -0,0 +1,278 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from yuxi import config as conf
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
from .provisioner_client import ProvisionerClient, SandboxRecord
|
||||
|
||||
|
||||
def sandbox_id_for_thread(thread_id: str, skills_thread_id: str | None = None, *, uid: str | None = None) -> str:
|
||||
file_thread_id = str(thread_id or "").strip()
|
||||
skills_id = str(skills_thread_id or file_thread_id).strip()
|
||||
uid_id = str(uid or "").strip()
|
||||
scope = file_thread_id if skills_id == file_thread_id else f"{file_thread_id}:{skills_id}"
|
||||
identity = f"{uid_id}:{scope}" if uid_id else scope
|
||||
digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
||||
return digest[:12]
|
||||
|
||||
|
||||
def _sandbox_key(uid: str, file_thread_id: str, skills_thread_id: str) -> str:
|
||||
return f"{uid}::{file_thread_id}::{skills_thread_id}"
|
||||
|
||||
|
||||
def normalize_env(env: dict | None) -> dict[str, str]:
|
||||
if not isinstance(env, dict):
|
||||
return {}
|
||||
return {str(key): "" if value is None else str(value) for key, value in env.items() if str(key)}
|
||||
|
||||
|
||||
def postgres_conninfo() -> str:
|
||||
db_url = os.getenv("POSTGRES_URL", "").strip()
|
||||
return db_url.replace("+asyncpg", "").replace("+psycopg", "")
|
||||
|
||||
|
||||
def load_user_agent_env(uid: str) -> dict[str, str]:
|
||||
conninfo = postgres_conninfo()
|
||||
if not conninfo:
|
||||
return {}
|
||||
|
||||
try:
|
||||
import psycopg
|
||||
|
||||
with psycopg.connect(conninfo, connect_timeout=3) as conn:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute("SELECT env FROM agent_envs WHERE uid = %s", (uid,))
|
||||
row = cursor.fetchone()
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"failed to load agent env for uid {uid}: {exc}") from exc
|
||||
|
||||
if not row:
|
||||
return {}
|
||||
|
||||
value = row[0]
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"stored agent env for uid {uid} is not valid JSON") from exc
|
||||
return normalize_env(value)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SandboxConnection:
|
||||
cache_key: str
|
||||
thread_id: str
|
||||
file_thread_id: str
|
||||
skills_thread_id: str
|
||||
uid: str
|
||||
sandbox_id: str
|
||||
sandbox_url: str
|
||||
|
||||
|
||||
class ProvisionerSandboxProvider:
|
||||
def __init__(self):
|
||||
provider_name = str(getattr(conf, "sandbox_provider", "provisioner")).strip().lower()
|
||||
if provider_name != "provisioner":
|
||||
raise RuntimeError("only sandbox_provider=provisioner is supported")
|
||||
|
||||
provisioner_url = str(getattr(conf, "sandbox_provisioner_url", "") or "").strip()
|
||||
if not provisioner_url:
|
||||
raise RuntimeError("sandbox_provisioner_url is required")
|
||||
|
||||
self._client = ProvisionerClient(provisioner_url)
|
||||
self._lock = threading.Lock()
|
||||
self._thread_locks: dict[str, threading.Lock] = {}
|
||||
self._connections: dict[str, SandboxConnection] = {}
|
||||
self._last_touch_at: dict[str, float] = {}
|
||||
self._touch_interval_seconds = int(getattr(conf, "sandbox_keepalive_interval_seconds", 30))
|
||||
|
||||
def _thread_lock(self, cache_key: str) -> threading.Lock:
|
||||
with self._lock:
|
||||
lock = self._thread_locks.get(cache_key)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
self._thread_locks[cache_key] = lock
|
||||
return lock
|
||||
|
||||
def _record_to_connection(
|
||||
self,
|
||||
*,
|
||||
cache_key: str,
|
||||
thread_id: str,
|
||||
file_thread_id: str,
|
||||
skills_thread_id: str,
|
||||
uid: str,
|
||||
record: SandboxRecord,
|
||||
) -> SandboxConnection:
|
||||
connection = SandboxConnection(
|
||||
cache_key=cache_key,
|
||||
thread_id=thread_id,
|
||||
file_thread_id=file_thread_id,
|
||||
skills_thread_id=skills_thread_id,
|
||||
uid=uid,
|
||||
sandbox_id=record.sandbox_id,
|
||||
sandbox_url=record.sandbox_url,
|
||||
)
|
||||
self._connections[cache_key] = connection
|
||||
self._last_touch_at[cache_key] = time.time()
|
||||
return connection
|
||||
|
||||
def _should_touch(self, cache_key: str) -> bool:
|
||||
if self._touch_interval_seconds <= 0:
|
||||
return False
|
||||
last_touch = self._last_touch_at.get(cache_key)
|
||||
if last_touch is None:
|
||||
return True
|
||||
return (time.time() - last_touch) >= self._touch_interval_seconds
|
||||
|
||||
def _touch_if_needed(self, connection: SandboxConnection) -> bool:
|
||||
if not self._should_touch(connection.cache_key):
|
||||
return True
|
||||
is_alive = self._client.touch(connection.sandbox_id)
|
||||
self._last_touch_at[connection.cache_key] = time.time()
|
||||
return is_alive
|
||||
|
||||
def acquire(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
uid: str,
|
||||
file_thread_id: str | None = None,
|
||||
skills_thread_id: str | None = None,
|
||||
) -> str:
|
||||
file_id = str(file_thread_id or thread_id).strip()
|
||||
skills_id = str(skills_thread_id or thread_id).strip()
|
||||
cache_key = _sandbox_key(uid, file_id, skills_id)
|
||||
lock = self._thread_lock(cache_key)
|
||||
with lock:
|
||||
current = self._connections.get(cache_key)
|
||||
if current:
|
||||
if current.uid != uid:
|
||||
raise RuntimeError(f"sandbox scope {cache_key} belongs to uid {current.uid}, not {uid}")
|
||||
try:
|
||||
if self._touch_if_needed(current):
|
||||
return current.sandbox_id
|
||||
self._connections.pop(cache_key, None)
|
||||
self._last_touch_at.pop(cache_key, None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(f"Failed to touch sandbox {current.sandbox_id} for {cache_key}: {exc}")
|
||||
return current.sandbox_id
|
||||
|
||||
sandbox_id = sandbox_id_for_thread(file_id, skills_id, uid=uid)
|
||||
logger.info(f"Ensuring sandbox {sandbox_id} for file thread {file_id} and skills thread {skills_id}")
|
||||
record = self._client.create(
|
||||
sandbox_id,
|
||||
thread_id,
|
||||
uid,
|
||||
load_user_agent_env(uid),
|
||||
file_thread_id=file_id,
|
||||
skills_thread_id=skills_id,
|
||||
)
|
||||
|
||||
connection = self._record_to_connection(
|
||||
cache_key=cache_key,
|
||||
thread_id=thread_id,
|
||||
file_thread_id=file_id,
|
||||
skills_thread_id=skills_id,
|
||||
uid=uid,
|
||||
record=record,
|
||||
)
|
||||
return connection.sandbox_id
|
||||
|
||||
def get(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
uid: str,
|
||||
create_if_missing: bool = False,
|
||||
file_thread_id: str | None = None,
|
||||
skills_thread_id: str | None = None,
|
||||
) -> SandboxConnection | None:
|
||||
file_id = str(file_thread_id or thread_id).strip()
|
||||
skills_id = str(skills_thread_id or thread_id).strip()
|
||||
cache_key = _sandbox_key(uid, file_id, skills_id)
|
||||
lock = self._thread_lock(cache_key)
|
||||
with lock:
|
||||
current = self._connections.get(cache_key)
|
||||
if current:
|
||||
if current.uid != uid:
|
||||
raise RuntimeError(f"sandbox scope {cache_key} belongs to uid {current.uid}, not {uid}")
|
||||
try:
|
||||
if self._touch_if_needed(current):
|
||||
return current
|
||||
self._connections.pop(cache_key, None)
|
||||
self._last_touch_at.pop(cache_key, None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(f"Failed to touch sandbox {current.sandbox_id} for {cache_key}: {exc}")
|
||||
return current
|
||||
|
||||
sandbox_id = sandbox_id_for_thread(file_id, skills_id, uid=uid)
|
||||
if create_if_missing:
|
||||
record = self._client.create(
|
||||
sandbox_id,
|
||||
thread_id,
|
||||
uid,
|
||||
load_user_agent_env(uid),
|
||||
file_thread_id=file_id,
|
||||
skills_thread_id=skills_id,
|
||||
)
|
||||
else:
|
||||
record = self._client.discover(sandbox_id)
|
||||
if record is None:
|
||||
return None
|
||||
|
||||
return self._record_to_connection(
|
||||
cache_key=cache_key,
|
||||
thread_id=thread_id,
|
||||
file_thread_id=file_id,
|
||||
skills_thread_id=skills_id,
|
||||
uid=uid,
|
||||
record=record,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
with self._lock:
|
||||
connections = list(self._connections.values())
|
||||
self._connections.clear()
|
||||
self._last_touch_at.clear()
|
||||
|
||||
for connection in connections:
|
||||
try:
|
||||
self._client.delete(connection.sandbox_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(f"Failed to release sandbox {connection.sandbox_id} for {connection.cache_key}: {exc}")
|
||||
|
||||
|
||||
_sandbox_provider: ProvisionerSandboxProvider | None = None
|
||||
_sandbox_provider_lock = threading.Lock()
|
||||
|
||||
|
||||
def init_sandbox_provider() -> ProvisionerSandboxProvider:
|
||||
global _sandbox_provider
|
||||
with _sandbox_provider_lock:
|
||||
if _sandbox_provider is None:
|
||||
_sandbox_provider = ProvisionerSandboxProvider()
|
||||
return _sandbox_provider
|
||||
|
||||
|
||||
def get_sandbox_provider() -> ProvisionerSandboxProvider:
|
||||
provider = _sandbox_provider
|
||||
if provider is not None:
|
||||
return provider
|
||||
return init_sandbox_provider()
|
||||
|
||||
|
||||
def shutdown_sandbox_provider() -> None:
|
||||
global _sandbox_provider
|
||||
with _sandbox_provider_lock:
|
||||
provider = _sandbox_provider
|
||||
_sandbox_provider = None
|
||||
if provider is not None:
|
||||
provider.shutdown()
|
||||
@@ -0,0 +1,88 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SandboxRecord:
|
||||
sandbox_id: str
|
||||
sandbox_url: str
|
||||
status: str | None = None
|
||||
|
||||
|
||||
class ProvisionerClient:
|
||||
def __init__(self, base_url: str, *, timeout_seconds: int = 20):
|
||||
self._base_url = base_url.rstrip("/")
|
||||
self._timeout = httpx.Timeout(timeout_seconds)
|
||||
|
||||
def _request(self, method: str, path: str, **kwargs) -> httpx.Response:
|
||||
return httpx.request(
|
||||
method=method,
|
||||
url=f"{self._base_url}{path}",
|
||||
timeout=self._timeout,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def health(self) -> bool:
|
||||
response = self._request("GET", "/health")
|
||||
return response.status_code == 200
|
||||
|
||||
def create(
|
||||
self,
|
||||
sandbox_id: str,
|
||||
thread_id: str,
|
||||
uid: str,
|
||||
env: dict[str, str] | None = None,
|
||||
*,
|
||||
file_thread_id: str | None = None,
|
||||
skills_thread_id: str | None = None,
|
||||
) -> SandboxRecord:
|
||||
response = self._request(
|
||||
"POST",
|
||||
"/api/sandboxes",
|
||||
json={
|
||||
"sandbox_id": sandbox_id,
|
||||
"thread_id": thread_id,
|
||||
"file_thread_id": file_thread_id or thread_id,
|
||||
"skills_thread_id": skills_thread_id or thread_id,
|
||||
"uid": uid,
|
||||
"env": env or {},
|
||||
},
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"failed to create sandbox {sandbox_id}: {response.status_code} {response.text}")
|
||||
payload = response.json()
|
||||
return SandboxRecord(
|
||||
sandbox_id=payload["sandbox_id"],
|
||||
sandbox_url=payload["sandbox_url"],
|
||||
status=payload.get("status"),
|
||||
)
|
||||
|
||||
def discover(self, sandbox_id: str) -> SandboxRecord | None:
|
||||
response = self._request("GET", f"/api/sandboxes/{sandbox_id}")
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"failed to discover sandbox {sandbox_id}: {response.status_code} {response.text}")
|
||||
payload = response.json()
|
||||
return SandboxRecord(
|
||||
sandbox_id=payload["sandbox_id"],
|
||||
sandbox_url=payload["sandbox_url"],
|
||||
status=payload.get("status"),
|
||||
)
|
||||
|
||||
def touch(self, sandbox_id: str) -> bool:
|
||||
response = self._request("POST", f"/api/sandboxes/{sandbox_id}/touch")
|
||||
if response.status_code == 404:
|
||||
return False
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"failed to touch sandbox {sandbox_id}: {response.status_code} {response.text}")
|
||||
return True
|
||||
|
||||
def delete(self, sandbox_id: str) -> None:
|
||||
response = self._request("DELETE", f"/api/sandboxes/{sandbox_id}")
|
||||
if response.status_code in {200, 404}:
|
||||
return
|
||||
raise RuntimeError(f"failed to delete sandbox {sandbox_id}: {response.status_code} {response.text}")
|
||||
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
from deepagents.backends import FilesystemBackend
|
||||
from deepagents.backends.protocol import (
|
||||
EditResult,
|
||||
FileDownloadResponse,
|
||||
FileInfo,
|
||||
FileUploadResponse,
|
||||
GlobResult,
|
||||
GrepMatch,
|
||||
GrepResult,
|
||||
LsResult,
|
||||
ReadResult,
|
||||
WriteResult,
|
||||
)
|
||||
|
||||
from yuxi.agents.skills.service import get_skills_root_dir, is_valid_skill_slug
|
||||
|
||||
|
||||
class SelectedSkillsReadonlyBackend(FilesystemBackend):
|
||||
"""只读 skills backend，仅暴露选中的技能目录。"""
|
||||
|
||||
def __init__(self, *, selected_slugs: list[str] | None):
|
||||
super().__init__(root_dir=get_skills_root_dir(), virtual_mode=True)
|
||||
self._selected_slugs = {
|
||||
str(slug).strip()
|
||||
for slug in (selected_slugs or [])
|
||||
if isinstance(slug, str) and is_valid_skill_slug(str(slug))
|
||||
}
|
||||
|
||||
def _extract_slug(self, path: str | None) -> str | None:
|
||||
if not path:
|
||||
return None
|
||||
normalized = (path or "").strip()
|
||||
if not normalized or normalized == "/":
|
||||
return None
|
||||
pure = PurePosixPath(normalized if normalized.startswith("/") else f"/{normalized}")
|
||||
parts = [p for p in pure.parts if p not in ("/", "")]
|
||||
return parts[0] if parts else None
|
||||
|
||||
def _is_allowed_path(self, path: str | None) -> bool:
|
||||
slug = self._extract_slug(path)
|
||||
if slug is None:
|
||||
return True
|
||||
return slug in self._selected_slugs
|
||||
|
||||
def _is_allowed_file(self, file_path: str) -> bool:
|
||||
slug = self._extract_slug(file_path)
|
||||
return slug is not None and slug in self._selected_slugs
|
||||
|
||||
def _filter_infos(self, infos: list[FileInfo]) -> list[FileInfo]:
|
||||
return [item for item in infos if self._extract_slug(item.get("path", "")) in self._selected_slugs]
|
||||
|
||||
def _filter_matches(self, matches: list[GrepMatch]) -> list[GrepMatch]:
|
||||
return [item for item in matches if self._extract_slug(item.get("path", "")) in self._selected_slugs]
|
||||
|
||||
def ls(self, path: str) -> LsResult:
|
||||
if not self._selected_slugs:
|
||||
return LsResult(entries=[])
|
||||
|
||||
normalized = (path or "/").strip() or "/"
|
||||
if not self._is_allowed_path(normalized):
|
||||
return LsResult(error="Access denied: path is outside selected skills.")
|
||||
|
||||
result = super().ls(normalized)
|
||||
if result.error:
|
||||
return result
|
||||
infos = result.entries or []
|
||||
if normalized == "/":
|
||||
infos = self._filter_infos(infos)
|
||||
return LsResult(entries=infos)
|
||||
|
||||
def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> ReadResult:
|
||||
if not self._is_allowed_file(file_path):
|
||||
return ReadResult(error="Access denied: file is outside selected skills.")
|
||||
return super().read(file_path, offset=offset, limit=limit)
|
||||
|
||||
def grep(self, pattern: str, path: str | None = None, glob: str | None = None) -> GrepResult:
|
||||
if not self._selected_slugs:
|
||||
return GrepResult(matches=[])
|
||||
|
||||
if path is not None:
|
||||
if not self._is_allowed_path(path):
|
||||
return GrepResult(error="Access denied: path is outside selected skills.")
|
||||
result = super().grep(pattern=pattern, path=path, glob=glob)
|
||||
if result.error:
|
||||
return result
|
||||
return GrepResult(matches=self._filter_matches(result.matches or []))
|
||||
|
||||
matches: list[GrepMatch] = []
|
||||
for slug in sorted(self._selected_slugs):
|
||||
result = super().grep(pattern=pattern, path=f"/{slug}", glob=glob)
|
||||
if result.error:
|
||||
continue
|
||||
matches.extend(result.matches or [])
|
||||
return GrepResult(matches=matches)
|
||||
|
||||
def glob(self, pattern: str, path: str = "/") -> GlobResult:
|
||||
if not self._selected_slugs:
|
||||
return GlobResult(matches=[])
|
||||
if not self._is_allowed_path(path):
|
||||
return GlobResult(error="Access denied: path is outside selected skills.")
|
||||
result = super().glob(pattern=pattern, path=path)
|
||||
if result.error:
|
||||
return result
|
||||
return GlobResult(matches=self._filter_infos(result.matches or []))
|
||||
|
||||
def write(self, file_path: str, content: str) -> WriteResult:
|
||||
return WriteResult(error="Skills path is read-only.")
|
||||
|
||||
def edit(self, file_path: str, old_string: str, new_string: str, replace_all: bool = False) -> EditResult:
|
||||
return EditResult(error="Skills path is read-only.")
|
||||
|
||||
def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
|
||||
return [FileUploadResponse(path=p, error="permission_denied") for p, _ in files]
|
||||
|
||||
def download_files(self, paths: list[str]) -> list[FileDownloadResponse]:
|
||||
responses: list[FileDownloadResponse] = []
|
||||
for path in paths:
|
||||
if not self._is_allowed_file(path):
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="invalid_path"))
|
||||
continue
|
||||
target = self._resolve_path(path)
|
||||
if not target.exists():
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found"))
|
||||
continue
|
||||
if target.is_dir():
|
||||
responses.append(FileDownloadResponse(path=path, content=None, error="is_directory"))
|
||||
continue
|
||||
responses.append(FileDownloadResponse(path=path, content=target.read_bytes(), error=None))
|
||||
return responses
|
||||
@@ -0,0 +1,421 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from abc import abstractmethod
|
||||
from contextlib import suppress
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver, aiosqlite
|
||||
from langgraph.graph.state import CompiledStateGraph
|
||||
from langgraph.stream.transformers import CustomTransformer
|
||||
from langgraph.types import Command
|
||||
|
||||
from yuxi import config as sys_config
|
||||
from yuxi.agents.context import DEFAULT_MAX_EXECUTION_STEPS, BaseContext, resolve_agent_resource_options
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
from yuxi.utils import logger
|
||||
from yuxi.utils.hash_utils import subagent_child_thread_id
|
||||
from yuxi.utils.thread_utils import extract_thread_id as _metadata_thread_id
|
||||
|
||||
|
||||
def _json_safe(value: Any) -> Any:
|
||||
if value is None or isinstance(value, str | int | float | bool):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return {str(key): _json_safe(child) for key, child in value.items()}
|
||||
if isinstance(value, list | tuple):
|
||||
return [_json_safe(child) for child in value]
|
||||
if hasattr(value, "model_dump"):
|
||||
return _json_safe(value.model_dump())
|
||||
return str(value)
|
||||
|
||||
|
||||
def _normalize_tool_event_data(data: Any) -> Any:
|
||||
"""规整 tools 流事件：write_todos / task 等返回 Command 的工具，其 tool-finished
|
||||
output 是 Command 对象，_json_safe 只能退化成 repr 字符串，前端无法关联结果。
|
||||
这里从 Command.update["messages"] 取出真正的 ToolMessage，使其与普通工具一致。"""
|
||||
if not isinstance(data, dict) or data.get("event") != "tool-finished":
|
||||
return data
|
||||
output = data.get("output")
|
||||
if not isinstance(output, Command):
|
||||
return data
|
||||
update = output.update if isinstance(output.update, dict) else {}
|
||||
messages = update.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
return data
|
||||
tool_call_id = data.get("tool_call_id")
|
||||
tool_message = next(
|
||||
(m for m in messages if isinstance(m, ToolMessage) and m.tool_call_id == tool_call_id),
|
||||
next((m for m in messages if isinstance(m, ToolMessage)), None),
|
||||
)
|
||||
if tool_message is None:
|
||||
return data
|
||||
return {**data, "output": tool_message}
|
||||
|
||||
|
||||
def _subagent_route_for_namespace(
|
||||
routes: dict[tuple[str, ...], dict[str, str]], namespace: list[str]
|
||||
) -> dict[str, str] | None:
|
||||
ns = tuple(namespace)
|
||||
for path, route in sorted(routes.items(), key=lambda item: len(item[0]), reverse=True):
|
||||
if ns[: len(path)] == path:
|
||||
return route
|
||||
return None
|
||||
|
||||
|
||||
async def _collect_subagent_routes(run, parent_thread_id: str, routes: dict[tuple[str, ...], dict[str, str]]) -> None:
|
||||
subagents = getattr(run, "yuxi_subagents", None)
|
||||
if subagents is None:
|
||||
subagents = getattr(run, "subagents", None)
|
||||
if subagents is None:
|
||||
return
|
||||
|
||||
try:
|
||||
async for subagent in subagents:
|
||||
path = tuple(getattr(subagent, "path", ()) or ())
|
||||
subagent_slug = getattr(subagent, "name", None) or getattr(subagent, "graph_name", None)
|
||||
cause = getattr(subagent, "cause", None)
|
||||
tool_call_id = (
|
||||
cause.get("tool_call_id") if isinstance(cause, dict) else getattr(subagent, "trigger_call_id", None)
|
||||
)
|
||||
state = getattr(subagent, "state", None)
|
||||
metadata = getattr(subagent, "metadata", None)
|
||||
thread_id = _metadata_thread_id(metadata) or _metadata_thread_id(state)
|
||||
if not thread_id and isinstance(subagent_slug, str) and isinstance(tool_call_id, str) and tool_call_id:
|
||||
thread_id = subagent_child_thread_id(parent_thread_id, subagent_slug, tool_call_id)
|
||||
if path and isinstance(subagent_slug, str) and isinstance(tool_call_id, str) and tool_call_id and thread_id:
|
||||
routes[path] = {
|
||||
"thread_id": thread_id,
|
||||
"parent_thread_id": parent_thread_id,
|
||||
"subagent_slug": subagent_slug,
|
||||
"tool_call_id": tool_call_id,
|
||||
}
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.debug(f"collect subagent stream routes failed: {exc}")
|
||||
|
||||
|
||||
def _recursion_limit_from_context(context: BaseContext, default: int) -> int:
|
||||
value = getattr(context, "max_execution_steps", default)
|
||||
return int(value) if isinstance(value, int) and value > 0 else default
|
||||
|
||||
|
||||
class BaseAgent:
|
||||
"""
|
||||
定义一个基础 Agent 供 各类 graph 继承
|
||||
"""
|
||||
|
||||
name = "base_agent"
|
||||
description = "base_agent"
|
||||
capabilities: list[str] = [] # 智能体能力列表，如 ["file_upload", "web_search"] 等
|
||||
context_schema: type[BaseContext] = BaseContext # 智能体上下文 schema
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.graph = None # will be covered by get_graph
|
||||
self.checkpointer = None
|
||||
self._async_conn = None
|
||||
self.workdir = Path(sys_config.save_dir) / "agents" / self.module_name
|
||||
self.workdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@property
|
||||
def module_name(self) -> str:
|
||||
"""Get the module name of the agent class."""
|
||||
return self.__class__.__module__.split(".")[-2]
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
"""Get the agent's class name."""
|
||||
return self.__class__.__name__
|
||||
|
||||
async def get_info(
|
||||
self,
|
||||
include_configurable_items: bool = True,
|
||||
user_role: str | None = None,
|
||||
db=None,
|
||||
user=None,
|
||||
):
|
||||
# metadata 固定在代码中，由各 Agent 的类属性提供
|
||||
metadata = self.load_metadata()
|
||||
configurable_items = {}
|
||||
if include_configurable_items:
|
||||
configurable_items = self.context_schema.get_configurable_items(user_role=user_role)
|
||||
if db is not None and user is not None:
|
||||
resource_fields = {
|
||||
item["kind"]
|
||||
for item in configurable_items.values()
|
||||
if item.get("kind") in {"tools", "knowledges", "mcps", "skills", "subagents"}
|
||||
}
|
||||
resource_options = await resolve_agent_resource_options(resource_fields, db=db, user=user)
|
||||
for item in configurable_items.values():
|
||||
if item.get("kind") in resource_options:
|
||||
item["options"] = resource_options[item["kind"]]
|
||||
|
||||
# Merge metadata with class attributes, metadata takes precedence
|
||||
return {
|
||||
"id": self.id,
|
||||
"name": getattr(self, "name", "Unknown"),
|
||||
"description": getattr(self, "description", "Unknown"),
|
||||
"metadata": metadata,
|
||||
"configurable_items": configurable_items,
|
||||
"capabilities": getattr(self, "capabilities", []), # 智能体能力列表
|
||||
}
|
||||
|
||||
async def get_config(self):
|
||||
return self.context_schema()
|
||||
|
||||
async def stream_values(self, messages: list[str], input_context=None, **kwargs):
|
||||
context = self.context_schema()
|
||||
context.update_from_dict(input_context or {})
|
||||
graph = await self.get_graph(context=context)
|
||||
for event in graph.astream({"messages": messages}, stream_mode="values", context=context):
|
||||
yield event["messages"]
|
||||
|
||||
async def stream_messages(self, messages: list[str], input_context=None, **kwargs):
|
||||
context = self.context_schema()
|
||||
context.update_from_dict(input_context or {})
|
||||
graph = await self.get_graph(context=context)
|
||||
logger.debug(f"stream_messages: {context=}")
|
||||
|
||||
# 构建配置：LangGraph 会自动从 checkpointer 恢复 state
|
||||
input_config = {
|
||||
"configurable": {"thread_id": context.thread_id, "uid": context.uid},
|
||||
"recursion_limit": _recursion_limit_from_context(context, DEFAULT_MAX_EXECUTION_STEPS),
|
||||
}
|
||||
|
||||
# langfuse metadata and callbacks integration
|
||||
if callbacks := kwargs.get("callbacks"):
|
||||
input_config["callbacks"] = list(callbacks)
|
||||
if metadata := kwargs.get("metadata"):
|
||||
input_config["metadata"] = dict(metadata)
|
||||
if tags := kwargs.get("tags"):
|
||||
input_config["tags"] = list(tags)
|
||||
|
||||
async for msg, metadata in graph.astream(
|
||||
{"messages": messages},
|
||||
stream_mode="messages",
|
||||
context=context,
|
||||
config=input_config,
|
||||
):
|
||||
yield msg, metadata
|
||||
|
||||
async def _stream_input_with_state(self, graph_input, input_context=None, **kwargs):
|
||||
context = self.context_schema()
|
||||
context.update_from_dict(input_context or {})
|
||||
graph = await self.get_graph(context=context)
|
||||
logger.debug(f"stream_with_state: {context=}")
|
||||
|
||||
input_config = {
|
||||
"configurable": {"thread_id": context.thread_id, "uid": context.uid},
|
||||
"recursion_limit": _recursion_limit_from_context(context, DEFAULT_MAX_EXECUTION_STEPS),
|
||||
}
|
||||
|
||||
if callbacks := kwargs.get("callbacks"):
|
||||
input_config["callbacks"] = list(callbacks)
|
||||
if metadata := kwargs.get("metadata"):
|
||||
input_config["metadata"] = dict(metadata)
|
||||
if tags := kwargs.get("tags"):
|
||||
input_config["tags"] = list(tags)
|
||||
|
||||
run = await graph.astream_events(
|
||||
graph_input,
|
||||
context=context,
|
||||
config=input_config,
|
||||
version="v3",
|
||||
transformers=[CustomTransformer],
|
||||
)
|
||||
subagent_routes: dict[tuple[str, ...], dict[str, str]] = {}
|
||||
route_task = asyncio.create_task(_collect_subagent_routes(run, context.thread_id, subagent_routes))
|
||||
try:
|
||||
async for event in run:
|
||||
params = event.get("params") or {}
|
||||
namespace = list(params.get("namespace") or [])
|
||||
method = event.get("method")
|
||||
data = params.get("data")
|
||||
subagent_route = _subagent_route_for_namespace(subagent_routes, namespace)
|
||||
|
||||
if method == "custom":
|
||||
yield "custom", data
|
||||
continue
|
||||
if method == "messages":
|
||||
msg, metadata = data
|
||||
metadata = dict(metadata or {})
|
||||
actual_thread_id = (subagent_route or {}).get("thread_id") or _metadata_thread_id(metadata)
|
||||
metadata["namespace"] = namespace
|
||||
metadata["stream_event"] = {"method": method, "namespace": namespace}
|
||||
if subagent_route:
|
||||
metadata.update(subagent_route)
|
||||
if actual_thread_id:
|
||||
metadata["thread_id"] = actual_thread_id
|
||||
yield "messages", (msg, metadata)
|
||||
elif method == "values" and not namespace:
|
||||
yield "values", data
|
||||
elif method in {"tasks", "tools", "lifecycle"}:
|
||||
if method == "tools":
|
||||
data = _normalize_tool_event_data(data)
|
||||
event_payload = {
|
||||
"method": method,
|
||||
"namespace": namespace,
|
||||
"data": _json_safe(data),
|
||||
}
|
||||
actual_thread_id = (subagent_route or {}).get("thread_id") or _metadata_thread_id(params)
|
||||
if subagent_route:
|
||||
event_payload.update(subagent_route)
|
||||
if actual_thread_id:
|
||||
event_payload["thread_id"] = actual_thread_id
|
||||
yield "stream_event", event_payload
|
||||
finally:
|
||||
route_task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await route_task
|
||||
|
||||
async def stream_messages_with_state(self, messages: list[str], input_context=None, **kwargs):
|
||||
async for event in self._stream_input_with_state({"messages": messages}, input_context, **kwargs):
|
||||
yield event
|
||||
|
||||
async def stream_resume_with_state(self, resume_input, input_context=None, **kwargs):
|
||||
async for event in self._stream_input_with_state(resume_input, input_context, **kwargs):
|
||||
yield event
|
||||
|
||||
async def invoke_messages(self, messages: list[str], input_context=None, **kwargs):
|
||||
context = self.context_schema()
|
||||
context.update_from_dict(input_context or {})
|
||||
graph = await self.get_graph(context=context)
|
||||
logger.debug(f"invoke_messages: {context}")
|
||||
|
||||
# 构建配置
|
||||
input_config = {
|
||||
"configurable": {"thread_id": context.thread_id, "uid": context.uid},
|
||||
"recursion_limit": _recursion_limit_from_context(context, DEFAULT_MAX_EXECUTION_STEPS),
|
||||
}
|
||||
|
||||
# langfuse metadata and callbacks integration
|
||||
if callbacks := kwargs.get("callbacks"):
|
||||
input_config["callbacks"] = list(callbacks)
|
||||
if metadata := kwargs.get("metadata"):
|
||||
input_config["metadata"] = dict(metadata)
|
||||
if tags := kwargs.get("tags"):
|
||||
input_config["tags"] = list(tags)
|
||||
|
||||
msg = await graph.ainvoke(
|
||||
{"messages": messages},
|
||||
context=context,
|
||||
config=input_config,
|
||||
)
|
||||
return msg
|
||||
|
||||
async def check_checkpointer(self):
|
||||
app = await self.get_graph()
|
||||
if not hasattr(app, "checkpointer") or app.checkpointer is None:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def get_history(self, uid, thread_id) -> list[dict]:
|
||||
"""获取历史消息"""
|
||||
try:
|
||||
app = await self.get_graph()
|
||||
|
||||
if not await self.check_checkpointer():
|
||||
return []
|
||||
|
||||
config = {"configurable": {"thread_id": thread_id, "uid": uid}}
|
||||
state = await app.aget_state(config)
|
||||
|
||||
result = []
|
||||
if state:
|
||||
messages = state.values.get("messages", [])
|
||||
for msg in messages:
|
||||
if hasattr(msg, "model_dump"):
|
||||
msg_dict = msg.model_dump() # 转换成字典
|
||||
else:
|
||||
msg_dict = dict(msg) if hasattr(msg, "__dict__") else {"content": str(msg)}
|
||||
result.append(msg_dict)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取智能体 {self.name} 历史消息出错: {e}")
|
||||
return []
|
||||
|
||||
def reload_graph(self):
|
||||
"""重置 graph 缓存，强制下次调用 get_graph 时重新构建"""
|
||||
self.graph = None
|
||||
logger.info(f"{self.name} graph 缓存已清空，将在下次调用时重新构建")
|
||||
|
||||
@abstractmethod
|
||||
async def get_graph(self, **kwargs) -> CompiledStateGraph:
|
||||
"""
|
||||
获取并编译对话图实例。
|
||||
必须确保在编译时设置 checkpointer，否则将无法获取历史记录。
|
||||
例如: graph = workflow.compile(checkpointer=sqlite_checkpointer)
|
||||
"""
|
||||
pass
|
||||
|
||||
async def _get_checkpointer(self):
|
||||
if self.checkpointer is not None:
|
||||
return self.checkpointer
|
||||
|
||||
checkpointer = None
|
||||
backend = os.getenv("LANGGRAPH_CHECKPOINTER_BACKEND", "sqlite").strip().lower()
|
||||
|
||||
if backend == "postgres":
|
||||
checkpointer = await self._create_postgres_checkpointer()
|
||||
|
||||
if checkpointer is None:
|
||||
try:
|
||||
checkpointer = AsyncSqliteSaver(await self.get_async_conn())
|
||||
except Exception as e:
|
||||
logger.error(f"构建 sqlite checkpointer 失败: {e}, 尝试使用内存存储")
|
||||
checkpointer = InMemorySaver()
|
||||
|
||||
self.checkpointer = checkpointer
|
||||
return self.checkpointer
|
||||
|
||||
async def _create_postgres_checkpointer(self):
|
||||
postgres_url = os.getenv("POSTGRES_URL")
|
||||
if not postgres_url:
|
||||
logger.warning("POSTGRES_URL 未配置，无法启用 postgres checkpointer，回退 sqlite")
|
||||
return None
|
||||
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver # type: ignore
|
||||
except Exception as e:
|
||||
logger.warning(f"langgraph postgres checkpointer 不可用，回退 sqlite: {e}")
|
||||
return None
|
||||
|
||||
try:
|
||||
saver = AsyncPostgresSaver(pg_manager.langgraph_pool)
|
||||
|
||||
logger.info(f"{self.name} 使用 postgres checkpointer")
|
||||
return saver
|
||||
except Exception as e:
|
||||
logger.warning(f"初始化 postgres checkpointer 失败，回退 sqlite: {e}")
|
||||
return None
|
||||
|
||||
async def get_async_conn(self) -> aiosqlite.Connection:
|
||||
"""获取异步数据库连接"""
|
||||
if self._async_conn is not None:
|
||||
return self._async_conn
|
||||
|
||||
conn = await aiosqlite.connect(os.path.join(self.workdir, "aio_history.db"))
|
||||
# Patch: langgraph's AsyncSqliteSaver expects is_alive() method which aiosqlite may not have
|
||||
if not hasattr(conn, "is_alive"):
|
||||
conn.is_alive = lambda: True
|
||||
self._async_conn = conn
|
||||
return self._async_conn
|
||||
|
||||
async def get_aio_memory(self) -> AsyncSqliteSaver:
|
||||
"""获取异步存储实例"""
|
||||
return AsyncSqliteSaver(await self.get_async_conn())
|
||||
|
||||
def load_metadata(self) -> dict:
|
||||
"""Load metadata from agent class attribute."""
|
||||
metadata = getattr(self, "metadata", {})
|
||||
if isinstance(metadata, dict):
|
||||
return metadata
|
||||
logger.warning(f"Agent {self.module_name} metadata is not a dict, fallback to empty metadata")
|
||||
return {}
|
||||
@@ -0,0 +1,99 @@
|
||||
import asyncio
|
||||
import importlib
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
|
||||
from yuxi.agents.base import BaseAgent
|
||||
from yuxi.utils import logger
|
||||
from yuxi.utils.singleton import SingletonMeta
|
||||
|
||||
|
||||
class AgentManager(metaclass=SingletonMeta):
|
||||
def __init__(self):
|
||||
self._classes = {}
|
||||
self._instances = {} # 存储已创建的 agent 实例
|
||||
|
||||
def register_agent(self, agent_class):
|
||||
self._classes[agent_class.__name__] = agent_class
|
||||
|
||||
def init_all_agents(self):
|
||||
for agent_id in self._classes.keys():
|
||||
self.get_agent(agent_id)
|
||||
|
||||
def get_agent(self, agent_id, reload=False, reload_graph=False, **kwargs):
|
||||
# 检查是否已经创建了该 agent 的实例
|
||||
if reload or agent_id not in self._instances:
|
||||
agent_class = self._classes[agent_id]
|
||||
self._instances[agent_id] = agent_class()
|
||||
|
||||
# 如果仅需要重新加载 graph，则清空 graph 缓存
|
||||
if reload_graph and agent_id in self._instances:
|
||||
self._instances[agent_id].reload_graph()
|
||||
|
||||
return self._instances[agent_id]
|
||||
|
||||
def get_agents(self):
|
||||
return list(self._instances.values())
|
||||
|
||||
async def reload_all(self):
|
||||
for agent_id in self._classes.keys():
|
||||
self.get_agent(agent_id, reload=True)
|
||||
|
||||
async def get_agents_info(self, include_configurable_items: bool = True):
|
||||
agents = self.get_agents()
|
||||
return await asyncio.gather(
|
||||
*[a.get_info(include_configurable_items=include_configurable_items) for a in agents]
|
||||
)
|
||||
|
||||
def auto_discover_agents(self):
|
||||
"""自动发现并注册 yuxi/agents/buildin/ 下的所有智能体。
|
||||
|
||||
遍历 yuxi/agents/buildin/ 目录下的所有子文件夹，如果子文件夹包含 __init__.py，
|
||||
则尝试从中导入 BaseAgent 的子类并注册。(使用自动导入的方式，支持私有agent)
|
||||
"""
|
||||
# 获取 agents 目录的路径
|
||||
agents_dir = Path(__file__).parent
|
||||
|
||||
# 遍历所有子目录
|
||||
for item in agents_dir.iterdir():
|
||||
# logger.info(f"尝试导入模块：{item}")
|
||||
# 跳过非目录、common 目录、__pycache__ 等
|
||||
if not item.is_dir() or item.name.startswith("_"):
|
||||
continue
|
||||
|
||||
# 检查是否有 __init__.py 文件
|
||||
init_file = item / "__init__.py"
|
||||
if not init_file.exists():
|
||||
logger.warning(f"{item} 不是一个有效的模块")
|
||||
continue
|
||||
|
||||
# 尝试导入模块
|
||||
try:
|
||||
module_name = f"yuxi.agents.buildin.{item.name}"
|
||||
module = importlib.import_module(module_name)
|
||||
|
||||
# 查找模块中所有 BaseAgent 的子类
|
||||
for name, obj in inspect.getmembers(module):
|
||||
if (
|
||||
inspect.isclass(obj)
|
||||
and issubclass(obj, BaseAgent)
|
||||
and obj is not BaseAgent
|
||||
and obj.__module__.startswith(module_name)
|
||||
):
|
||||
logger.info(f"自动发现智能体: {obj.__name__} 来自 {item.name}")
|
||||
self.register_agent(obj)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"无法从 {item.name} 加载智能体: {e}")
|
||||
|
||||
|
||||
agent_manager = AgentManager()
|
||||
# 自动发现并注册所有智能体
|
||||
agent_manager.auto_discover_agents()
|
||||
agent_manager.init_all_agents()
|
||||
|
||||
__all__ = ["agent_manager"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pass
|
||||
@@ -0,0 +1,5 @@
|
||||
from .context import ChatBotContext
|
||||
from .graph import ChatbotAgent
|
||||
from .state import ChatBotState, SubAgentRunState, merge_subagent_runs
|
||||
|
||||
__all__ = ["ChatBotContext", "ChatBotState", "ChatbotAgent", "SubAgentRunState", "merge_subagent_runs"]
|
||||
@@ -0,0 +1,17 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from yuxi.agents.context import BaseContext
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class ChatBotContext(BaseContext):
|
||||
subagents: list[str] | None = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"name": "子智能体",
|
||||
"options": [],
|
||||
"description": "可选子智能体列表，为空表示启用当前用户可见的全部子智能体。",
|
||||
"type": "list",
|
||||
"kind": "subagents",
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,113 @@
|
||||
from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware
|
||||
from langchain.agents import create_agent
|
||||
from langchain.agents.middleware import ModelRetryMiddleware, TodoListMiddleware
|
||||
|
||||
from yuxi.agents import BaseAgent, load_chat_model, resolve_chat_model_spec
|
||||
from yuxi.agents.backends import create_agent_filesystem_middleware
|
||||
from yuxi.agents.context import (
|
||||
DEFAULT_SUMMARY_KEEP_MESSAGES,
|
||||
DEFAULT_SUMMARY_L2_TRIGGER_RATIO,
|
||||
DEFAULT_SUMMARY_THRESHOLD_K,
|
||||
DEFAULT_SUMMARY_TOOL_RESULT_TOKEN_LIMIT,
|
||||
DEFAULT_TOOL_RESULT_EVICTION_K_TOKENS,
|
||||
DEFAULT_YUXI_SUMMARY_PROMPT,
|
||||
prepare_agent_runtime_context,
|
||||
)
|
||||
from yuxi.agents.middlewares import (
|
||||
TokenUsageMiddleware,
|
||||
create_summary_middleware,
|
||||
save_attachments_to_fs,
|
||||
)
|
||||
from yuxi.agents.middlewares.skills import SkillsMiddleware
|
||||
from yuxi.agents.middlewares.subagent_task import create_subagent_task_middleware
|
||||
from yuxi.agents.toolkits.service import resolve_configured_runtime_tools
|
||||
|
||||
from .context import ChatBotContext
|
||||
from .prompt import TODO_MID_PROMPT, build_prompt_with_context
|
||||
from .state import ChatBotState
|
||||
|
||||
|
||||
async def _build_middlewares(context):
|
||||
"""构建中间件列表"""
|
||||
# summary middleware
|
||||
# 主 Agent 上下文优化：默认 100k tokens 触发压缩，压缩后保留最近 10 条消息
|
||||
summary_trigger_tokens = getattr(context, "summary_threshold", DEFAULT_SUMMARY_THRESHOLD_K) * 1024
|
||||
summary_keep_messages = getattr(context, "summary_keep_messages", DEFAULT_SUMMARY_KEEP_MESSAGES)
|
||||
summary_prompt = getattr(context, "summary_prompt", None) or DEFAULT_YUXI_SUMMARY_PROMPT
|
||||
summary_tool_result_token_limit = getattr(
|
||||
context,
|
||||
"summary_tool_result_token_limit",
|
||||
DEFAULT_SUMMARY_TOOL_RESULT_TOKEN_LIMIT,
|
||||
)
|
||||
summary_l2_trigger_ratio = getattr(context, "summary_l2_trigger_ratio", DEFAULT_SUMMARY_L2_TRIGGER_RATIO)
|
||||
model_spec = resolve_chat_model_spec(context.model)
|
||||
summary_middleware = create_summary_middleware(
|
||||
model=load_chat_model(fully_specified_name=model_spec),
|
||||
trigger=("tokens", summary_trigger_tokens),
|
||||
keep=("messages", summary_keep_messages),
|
||||
summary_prompt=summary_prompt,
|
||||
trim_tokens_to_summarize=summary_trigger_tokens,
|
||||
tool_result_offload_token_limit=summary_tool_result_token_limit,
|
||||
l1_l2_trigger_ratio=summary_l2_trigger_ratio,
|
||||
)
|
||||
|
||||
middlewares = [
|
||||
create_agent_filesystem_middleware(
|
||||
getattr(context, "tool_token_limit", DEFAULT_TOOL_RESULT_EVICTION_K_TOKENS) * 1024,
|
||||
context=context,
|
||||
),
|
||||
save_attachments_to_fs,
|
||||
SkillsMiddleware(),
|
||||
]
|
||||
subagent_middleware = await create_subagent_task_middleware(context)
|
||||
if subagent_middleware:
|
||||
middlewares.append(subagent_middleware)
|
||||
middlewares.extend(
|
||||
[
|
||||
summary_middleware,
|
||||
TodoListMiddleware(system_prompt=TODO_MID_PROMPT),
|
||||
PatchToolCallsMiddleware(),
|
||||
ModelRetryMiddleware(max_retries=getattr(context, "model_retry_times", 2)),
|
||||
TokenUsageMiddleware(),
|
||||
]
|
||||
)
|
||||
return middlewares
|
||||
|
||||
|
||||
class ChatbotAgent(BaseAgent):
|
||||
name = "智能助手"
|
||||
description = "基础的对话机器人，可以回答问题，可在配置中启用需要的工具。"
|
||||
capabilities = ["file_upload", "files"] # 支持文件上传功能
|
||||
context_schema = ChatBotContext
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
async def get_graph(self, context=None, **kwargs):
|
||||
|
||||
context = await prepare_agent_runtime_context(
|
||||
context or self.context_schema(),
|
||||
context_schema=self.context_schema,
|
||||
)
|
||||
|
||||
# 使用 create_agent 创建智能体
|
||||
model_spec = resolve_chat_model_spec(context.model)
|
||||
graph = create_agent(
|
||||
model=load_chat_model(fully_specified_name=model_spec),
|
||||
tools=await resolve_configured_runtime_tools(context),
|
||||
system_prompt=build_prompt_with_context(context),
|
||||
middleware=await _build_middlewares(context),
|
||||
state_schema=ChatBotState,
|
||||
checkpointer=await self._get_checkpointer(),
|
||||
)
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
def main():
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
# asyncio.run(main())
|
||||
@@ -0,0 +1,91 @@
|
||||
from yuxi.utils.datetime_utils import shanghai_now
|
||||
from yuxi.utils.paths import (
|
||||
VIRTUAL_PATH_OUTPUTS,
|
||||
VIRTUAL_PATH_PREFIX,
|
||||
VIRTUAL_PATH_UPLOADS,
|
||||
VIRTUAL_PATH_WORKSPACE,
|
||||
)
|
||||
|
||||
PROMPT = f"""
|
||||
你是一个交互式智能体“语析“。
|
||||
|
||||
专门用来回答用户的问题。请根据用户提供的信息，尽可能详细地回答问题。
|
||||
如果你不确定答案，可以说你不知道，但请尽量提供相关的信息或建议。请保持礼貌和专业。
|
||||
|
||||
<| 内部执行约束:重要 |>
|
||||
以下内容仅用于指导你的内部执行过程，不属于面向用户的基本设定。除非用户明确询问系统如何工作，
|
||||
否则不要主动向用户说明工作区、文件系统、知识库路径、工具调用方式等内部实现细节。
|
||||
|
||||
<| 文件系统约束 |>
|
||||
系统主要工作路径为 {VIRTUAL_PATH_PREFIX}，但必须遵守规范：
|
||||
- {VIRTUAL_PATH_OUTPUTS}：用于写入的文件夹
|
||||
- {VIRTUAL_PATH_OUTPUTS}/tmp/：用于存放中间结果或备份内容
|
||||
- {VIRTUAL_PATH_UPLOADS}：用于存放用户上传的附件（只读，除非用户要求，否则不得写入）
|
||||
- {VIRTUAL_PATH_WORKSPACE}：用于存放用户文件（用户私人目录，除非用户要求，否则不得写入）
|
||||
- 其他路径：非必要不写入其他路径
|
||||
|
||||
<| 风格规范 |>
|
||||
保持专业严谨，减少使用 Emoji
|
||||
|
||||
<| 可视化 HTML 辅助组件规范 |>
|
||||
回答的主要表达载体始终是 Markdown。只有当普通 Markdown 难以清晰表达数值对比、层级关系、流程结构、
|
||||
时间线、关键指标或布局示意时，才可以额外使用 Markdown 围栏代码块语言标记 `html:preview`
|
||||
输出一个轻量静态 HTML 辅助组件：
|
||||
```html:preview
|
||||
自包含的静态 HTML/CSS 内容
|
||||
```
|
||||
使用要求：
|
||||
- `html:preview` 只用于补齐 Markdown 的短板，不能替代正文回答；核心解释、推理、背景、风险、
|
||||
结论展开和完整明细必须放在普通 Markdown 中。
|
||||
- 如果 Markdown 的标题、列表、表格、引用或代码块已经足够清楚，不要使用 `html:preview`。
|
||||
- 预览内容应优先使用静态 HTML/CSS；可以引用方便访问、稳定、无需登录鉴权的 HTTPS 外链资源
|
||||
（如公开图片或字体），但必须保证没有外链时核心信息仍可读，不要依赖跨域受限、内网、
|
||||
临时链接或不稳定资源，不要编写 JavaScript。
|
||||
- 这是嵌入在回答中的辅助可视化组件，不是完整网页、不是正文容器、不是自带外壳的信息卡片；
|
||||
不要设计导航栏、页脚、登录态、表单、复杂按钮、营销页 Hero 或多屏网页结构。
|
||||
- 外层预览容器已经提供 12px 圆角、边框和裁切；HTML 内容本身不要再套卡片壳、面板壳或页面壳，
|
||||
不要给最外层内容添加大圆角、阴影、厚边框、额外外边距或整页背景。
|
||||
- 内容组织必须以“快速看懂”为中心：优先呈现少量关键指标、对比关系、趋势/阶段、状态和极短备注，
|
||||
避免为了视觉效果牺牲可读性。
|
||||
- 默认按 800px * 360px 的展示尺寸设计；前端最大可能支持到 700px 高度，真实宽高也会随容器变化，因此布局必须响应式。
|
||||
- HTML 内部不要写死整体画布高度；优先使用 `max-width: 100%`、`box-sizing: border-box`、
|
||||
弹性网格、换行和适度压缩间距来适配不同宽高。
|
||||
- 必须保证核心内容在 800px * 360px 内可读且不依赖滚动；如果预计放不下，必须减少内容，而不是缩小到难以阅读或继续堆叠。
|
||||
- 可视化组件最多呈现 1 个短标题、3-5 个关键指标或一组简短对比；不要在组件里放完整明细、长表格、长列表或多段说明。
|
||||
- 当数据超过 6 项时，不要逐项做卡片网格；应汇总为趋势、最大/最小值、异常点、Top 3、分布或区间。
|
||||
完整列表、明细表或逐日解释放在 `html:preview` 之后用普通 Markdown 展示。
|
||||
- 可视化组件内禁止放成段文字、长句解释、新闻正文、报告段落、多行预警说明或叙事性文案；
|
||||
组件内文字应以短标签、短结论、数字、单位、状态词和极短备注为主。
|
||||
- 单个说明文本建议不超过 20 个中文字符；超过一句话的解释、背景、风险说明、数据来源详情必须放在
|
||||
`html:preview` 后面的普通 Markdown 中。
|
||||
- 设计应克制、清晰、信息密度适中；优先使用紧凑指标组、摘要表、对比条、状态标签、时间轴和简单关系图，
|
||||
不要做复杂装饰、大图标、密集网格或过重视觉效果。
|
||||
- 如果用户是在询问 HTML 源码、教程示例或需要复制代码，必须使用普通 `html` 代码块，不要使用 `html:preview`。
|
||||
"""
|
||||
|
||||
# 效果不好，暂时不启用
|
||||
SOURCE_CITE_PROMPT = """
|
||||
|
||||
<| 引用来源 |>
|
||||
当你提供的信息来自于用户上传的文件或者知识库中的内容时，请务必在回答中注明信息来源，以增加答案的可信度和透明度。
|
||||
|
||||
对于论断内容，需要添加参考文献信息，将对应段落的末尾添加 cite 信息。使用
|
||||
<cite source="$SOURCE" type="$TYPE">$INDEX</cite>
|
||||
|
||||
- $SOURCE：信息来源，可以是文件名，可以是url
|
||||
- $TYPE：引用类型，可以是 "file"、"url"，对于网络搜索应该使用 "url"，对于用户上传的文件或者知识库中的内容应该使用 "file"
|
||||
- $INDEX：引用索引，应该从 1 开始
|
||||
|
||||
比如 <cite source="食品工艺学.pdf" type="file">1</cite>
|
||||
"""
|
||||
|
||||
TODO_MID_PROMPT = """
|
||||
你需要根据任务的复杂程度来使用 write_todos 来记录规划和待办事项，确保任务的每个步骤都被记录和跟踪。
|
||||
每个待办任务名称必须简短，控制在 20 个中文汉字以内。
|
||||
"""
|
||||
|
||||
|
||||
def build_prompt_with_context(context):
|
||||
current_date = f"当前日期：{shanghai_now().strftime('%Y-%m-%d')}"
|
||||
system_prompt = f"{current_date}\n\n{PROMPT.strip()}\n\n{context.system_prompt or ''}"
|
||||
return system_prompt.strip()
|
||||
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Literal, TypedDict
|
||||
|
||||
from yuxi.agents.state import BaseState
|
||||
|
||||
|
||||
class SubAgentRunState(TypedDict, total=False):
|
||||
id: str
|
||||
run_id: str
|
||||
subagent_slug: str
|
||||
subagent_name: str
|
||||
child_thread_id: str
|
||||
description: str
|
||||
status: Literal["pending", "running", "completed", "failed", "cancel_requested", "cancelled", "interrupted"]
|
||||
created_at: str
|
||||
completed_at: str
|
||||
error: str | None
|
||||
artifacts: list[str]
|
||||
events_url: str
|
||||
result_url: str
|
||||
|
||||
|
||||
def merge_subagent_runs(
|
||||
existing: list[SubAgentRunState] | None,
|
||||
new: list[SubAgentRunState] | None,
|
||||
) -> list[SubAgentRunState]:
|
||||
"""LangGraph state reducer：增量合并父 Agent 记录的子智能体运行摘要。
|
||||
|
||||
`run_id` 是一次真实子智能体执行的身份。只有相同 `run_id` 才会更新同一条记录；
|
||||
没有 `run_id` 的增量记录直接追加，不用工具调用 ID 或子线程 ID 做旧状态兼容匹配。
|
||||
"""
|
||||
if existing is None:
|
||||
return list(new or [])
|
||||
if new is None:
|
||||
return existing
|
||||
|
||||
merged = [dict(item) for item in existing]
|
||||
run_id_index = {item.get("run_id"): position for position, item in enumerate(merged) if item.get("run_id")}
|
||||
for item in new:
|
||||
run = dict(item)
|
||||
run_id = run.get("run_id")
|
||||
position = None
|
||||
if run_id and run_id in run_id_index:
|
||||
position = run_id_index[run_id]
|
||||
|
||||
if position is None:
|
||||
position = len(merged)
|
||||
merged.append(run)
|
||||
else:
|
||||
merged[position] = {**merged[position], **run}
|
||||
|
||||
if run_id:
|
||||
run_id_index[run_id] = position
|
||||
return merged
|
||||
|
||||
|
||||
class ChatBotState(BaseState):
|
||||
subagent_runs: Annotated[list[SubAgentRunState], merge_subagent_runs]
|
||||
@@ -0,0 +1,4 @@
|
||||
from .context import SubAgentContext
|
||||
from .graph import SubAgentBackend
|
||||
|
||||
__all__ = ["SubAgentBackend", "SubAgentContext"]
|
||||
@@ -0,0 +1,23 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from yuxi.agents.context import BaseContext
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class SubAgentContext(BaseContext):
|
||||
parent_thread_id: str | None = field(
|
||||
default=None,
|
||||
metadata={"name": "父线程ID", "configurable": False, "hide": True},
|
||||
)
|
||||
file_thread_id: str | None = field(
|
||||
default=None,
|
||||
metadata={"name": "文件线程ID", "configurable": False, "hide": True},
|
||||
)
|
||||
skills_thread_id: str | None = field(
|
||||
default=None,
|
||||
metadata={"name": "Skills线程ID", "configurable": False, "hide": True},
|
||||
)
|
||||
is_subagent_runtime: bool = field(
|
||||
default=False,
|
||||
metadata={"name": "子智能体运行态", "configurable": False, "hide": True},
|
||||
)
|
||||
@@ -0,0 +1,127 @@
|
||||
from typing import Any
|
||||
|
||||
from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware
|
||||
from langchain.agents import create_agent
|
||||
from langchain.agents.middleware import ModelRetryMiddleware, TodoListMiddleware
|
||||
from langchain.agents.middleware.types import AgentMiddleware
|
||||
|
||||
from yuxi.agents import BaseAgent, BaseState, load_chat_model, resolve_chat_model_spec
|
||||
from yuxi.agents.backends import create_agent_filesystem_middleware
|
||||
from yuxi.agents.buildin.chatbot.prompt import TODO_MID_PROMPT, build_prompt_with_context
|
||||
from yuxi.agents.buildin.subagent.context import SubAgentContext
|
||||
from yuxi.agents.context import (
|
||||
DEFAULT_SUMMARY_KEEP_MESSAGES,
|
||||
DEFAULT_SUMMARY_L2_TRIGGER_RATIO,
|
||||
DEFAULT_SUMMARY_THRESHOLD_K,
|
||||
DEFAULT_SUMMARY_TOOL_RESULT_TOKEN_LIMIT,
|
||||
DEFAULT_TOOL_RESULT_EVICTION_K_TOKENS,
|
||||
DEFAULT_YUXI_SUMMARY_PROMPT,
|
||||
prepare_agent_runtime_context,
|
||||
)
|
||||
from yuxi.agents.middlewares import TokenUsageMiddleware, create_summary_middleware, save_attachments_to_fs
|
||||
from yuxi.agents.middlewares.skills import SkillsMiddleware
|
||||
from yuxi.agents.toolkits.service import resolve_configured_runtime_tools
|
||||
|
||||
_SUBAGENT_DISABLED_TOOLS = frozenset({"present_artifacts", "ask_user_question", "install_skill"})
|
||||
|
||||
|
||||
def _tool_name(tool) -> str | None:
|
||||
if isinstance(tool, dict):
|
||||
name = tool.get("name")
|
||||
else:
|
||||
name = getattr(tool, "name", None)
|
||||
return name if isinstance(name, str) else None
|
||||
|
||||
|
||||
def _filter_disabled_tools(tools):
|
||||
return [tool for tool in tools if _tool_name(tool) not in _SUBAGENT_DISABLED_TOOLS]
|
||||
|
||||
|
||||
class _SubAgentToolFilterMiddleware(AgentMiddleware[Any, Any, Any]):
|
||||
def wrap_model_call(self, request, handler):
|
||||
return handler(request.override(tools=_filter_disabled_tools(request.tools or [])))
|
||||
|
||||
async def awrap_model_call(self, request, handler):
|
||||
return await handler(request.override(tools=_filter_disabled_tools(request.tools or [])))
|
||||
|
||||
|
||||
async def _build_middlewares(context):
|
||||
summary_trigger_tokens = getattr(context, "summary_threshold", DEFAULT_SUMMARY_THRESHOLD_K) * 1024
|
||||
summary_keep_messages = getattr(context, "summary_keep_messages", DEFAULT_SUMMARY_KEEP_MESSAGES)
|
||||
summary_prompt = getattr(context, "summary_prompt", None) or DEFAULT_YUXI_SUMMARY_PROMPT
|
||||
summary_tool_result_token_limit = getattr(
|
||||
context,
|
||||
"summary_tool_result_token_limit",
|
||||
DEFAULT_SUMMARY_TOOL_RESULT_TOKEN_LIMIT,
|
||||
)
|
||||
summary_l2_trigger_ratio = getattr(context, "summary_l2_trigger_ratio", DEFAULT_SUMMARY_L2_TRIGGER_RATIO)
|
||||
model_spec = resolve_chat_model_spec(context.model)
|
||||
summary_middleware = create_summary_middleware(
|
||||
model=load_chat_model(fully_specified_name=model_spec),
|
||||
trigger=("tokens", summary_trigger_tokens),
|
||||
keep=("messages", summary_keep_messages),
|
||||
summary_prompt=summary_prompt,
|
||||
trim_tokens_to_summarize=summary_trigger_tokens,
|
||||
tool_result_offload_token_limit=summary_tool_result_token_limit,
|
||||
l1_l2_trigger_ratio=summary_l2_trigger_ratio,
|
||||
)
|
||||
|
||||
return [
|
||||
create_agent_filesystem_middleware(
|
||||
getattr(context, "tool_token_limit", DEFAULT_TOOL_RESULT_EVICTION_K_TOKENS) * 1024,
|
||||
context=context,
|
||||
),
|
||||
save_attachments_to_fs,
|
||||
SkillsMiddleware(),
|
||||
summary_middleware,
|
||||
TodoListMiddleware(system_prompt=TODO_MID_PROMPT),
|
||||
PatchToolCallsMiddleware(),
|
||||
_SubAgentToolFilterMiddleware(),
|
||||
ModelRetryMiddleware(),
|
||||
TokenUsageMiddleware(),
|
||||
]
|
||||
|
||||
|
||||
class SubAgentBackend(BaseAgent):
|
||||
name = "子智能体"
|
||||
description = "用于被主智能体通过 task 工具调用的专用智能体后端。"
|
||||
capabilities = ["file_upload", "files"]
|
||||
context_schema = SubAgentContext
|
||||
|
||||
async def get_info(
|
||||
self,
|
||||
include_configurable_items: bool = True,
|
||||
user_role: str | None = None,
|
||||
db=None,
|
||||
user=None,
|
||||
):
|
||||
info = await super().get_info(
|
||||
include_configurable_items=include_configurable_items,
|
||||
user_role=user_role,
|
||||
db=db,
|
||||
user=user,
|
||||
)
|
||||
tools_item = (info.get("configurable_items") or {}).get("tools")
|
||||
if isinstance(tools_item, dict):
|
||||
tools_item["options"] = [
|
||||
option
|
||||
for option in tools_item.get("options") or []
|
||||
if option.get("key") not in _SUBAGENT_DISABLED_TOOLS
|
||||
]
|
||||
return info
|
||||
|
||||
async def get_graph(self, context=None, **kwargs):
|
||||
context = await prepare_agent_runtime_context(
|
||||
context or self.context_schema(),
|
||||
context_schema=self.context_schema,
|
||||
)
|
||||
model_spec = resolve_chat_model_spec(context.model)
|
||||
|
||||
return create_agent(
|
||||
model=load_chat_model(fully_specified_name=model_spec),
|
||||
tools=_filter_disabled_tools(await resolve_configured_runtime_tools(context)),
|
||||
system_prompt=build_prompt_with_context(context),
|
||||
middleware=await _build_middlewares(context),
|
||||
state_schema=BaseState,
|
||||
checkpointer=await self._get_checkpointer(),
|
||||
)
|
||||
@@ -0,0 +1,558 @@
|
||||
"""Define the configurable parameters for the agent."""
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from dataclasses import MISSING, dataclass, field, fields
|
||||
from typing import Any, get_origin
|
||||
|
||||
from yuxi.agents.backends.sandbox.paths import sandbox_workspace_agents_prompt_file
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
WORKSPACE_AGENTS_PROMPT_MAX_BYTES = 64 * 1024
|
||||
DEFAULT_SUMMARY_THRESHOLD_K = 100 # 100K tokens
|
||||
DEFAULT_SUMMARY_KEEP_MESSAGES = 10
|
||||
DEFAULT_SUMMARY_TOOL_RESULT_TOKEN_LIMIT = 300
|
||||
DEFAULT_SUMMARY_L2_TRIGGER_RATIO = 0.4
|
||||
DEFAULT_MAX_EXECUTION_STEPS = 300
|
||||
DEFAULT_TOOL_RESULT_EVICTION_K_TOKENS = 3
|
||||
DEFAULT_YUXI_SUMMARY_PROMPT = """你是对话上下文压缩助手。
|
||||
你的任务是把下面的对话历史压缩成后续智能体继续工作所需的高价值上下文。
|
||||
|
||||
请特别保留并清晰记录：
|
||||
|
||||
## SESSION INTENT
|
||||
用户当前的主要目标、任务范围和最终交付物。
|
||||
|
||||
## USER REQUIREMENTS AND PREFERENCES
|
||||
用户明确提出的要求、偏好、禁忌、输出格式、语言风格、技术约束、验收标准，以及对实现方式的取舍意见。只记录仍然可能影响后续回答或执行的内容。
|
||||
|
||||
## PROGRESS AND DECISIONS
|
||||
已经完成的步骤、关键结论、已确认的方案、被否定的方案及原因。
|
||||
|
||||
## ARTIFACTS AND REFERENCES
|
||||
已经创建、修改、读取或需要继续关注的文件、路径、工具输出路径、线程或运行标识。保留具体路径和关键标识符。
|
||||
|
||||
## NEXT STEPS
|
||||
为了完成用户目标，后续最应该继续做的具体步骤。没有待办时写 None。
|
||||
|
||||
要求：
|
||||
- 不要逐字复述冗长工具输出；保留结论、路径和必要证据。
|
||||
- 不要编造没有出现在对话中的事实。
|
||||
- 如果存在未解决的问题或风险，明确记录。
|
||||
- 使用与用户主要对话一致的语言。
|
||||
|
||||
<messages>
|
||||
{messages}
|
||||
</messages>
|
||||
|
||||
只输出压缩后的上下文，不要添加额外说明。"""
|
||||
|
||||
|
||||
def _role_can_access(auth: str | None, role: str | None) -> bool:
|
||||
if not auth:
|
||||
return True
|
||||
if auth == "admin":
|
||||
return role in {"admin", "superadmin"}
|
||||
if auth == "superadmin":
|
||||
return role == "superadmin"
|
||||
return False
|
||||
|
||||
|
||||
def _load_workspace_agents_prompt(thread_id: str, uid: str) -> str:
|
||||
prompt_file = sandbox_workspace_agents_prompt_file(thread_id, uid)
|
||||
try:
|
||||
with prompt_file.open("rb") as buffer:
|
||||
content = buffer.read(WORKSPACE_AGENTS_PROMPT_MAX_BYTES + 1)
|
||||
except FileNotFoundError:
|
||||
return ""
|
||||
except IsADirectoryError:
|
||||
logger.warning("读取工作区 AGENTS.md 失败: 路径是目录")
|
||||
return ""
|
||||
except OSError as exc:
|
||||
logger.warning(f"读取工作区 AGENTS.md 失败: {exc}")
|
||||
return ""
|
||||
|
||||
prompt = content[:WORKSPACE_AGENTS_PROMPT_MAX_BYTES].decode("utf-8", errors="replace").strip()
|
||||
if not prompt:
|
||||
return ""
|
||||
if len(content) > WORKSPACE_AGENTS_PROMPT_MAX_BYTES:
|
||||
return f"{prompt}\n\n[AGENTS.md 内容已截断]"
|
||||
return prompt
|
||||
|
||||
|
||||
async def build_agent_input_context(
|
||||
agent_config: dict | None,
|
||||
*,
|
||||
thread_id: str,
|
||||
uid: str,
|
||||
run_id: str | None = None,
|
||||
request_id: str | None = None,
|
||||
) -> dict:
|
||||
input_context = dict(agent_config or {})
|
||||
agents_prompt = await asyncio.to_thread(_load_workspace_agents_prompt, thread_id, uid)
|
||||
|
||||
if agents_prompt:
|
||||
agents_section = f"用户工作区 agents/AGENTS.md 内容：\n{agents_prompt}"
|
||||
base_prompt = str(input_context.get("system_prompt") or "").rstrip()
|
||||
input_context["system_prompt"] = f"{base_prompt}\n\n{agents_section}" if base_prompt else agents_section
|
||||
|
||||
input_context.update({"uid": uid, "thread_id": thread_id, "run_id": run_id, "request_id": request_id})
|
||||
return input_context
|
||||
|
||||
|
||||
def filter_config_by_role(
|
||||
config_json: dict,
|
||||
role: str | None,
|
||||
context_schema: type["BaseContext"] | None = None,
|
||||
) -> dict:
|
||||
"""按 Context 字段 metadata.auth 过滤 config_json.context。"""
|
||||
if not isinstance(config_json, dict):
|
||||
return {}
|
||||
|
||||
schema = context_schema or BaseContext
|
||||
restricted_fields = {
|
||||
f.name
|
||||
for f in fields(schema)
|
||||
if f.metadata.get("auth") and not _role_can_access(str(f.metadata.get("auth")), role)
|
||||
}
|
||||
if not restricted_fields:
|
||||
return dict(config_json)
|
||||
|
||||
filtered = dict(config_json)
|
||||
context = filtered.get("context")
|
||||
if isinstance(context, dict):
|
||||
filtered["context"] = {key: value for key, value in context.items() if key not in restricted_fields}
|
||||
return filtered
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class BaseContext:
|
||||
"""
|
||||
定义一个基础 Context 供 各类 graph 继承
|
||||
|
||||
配置优先级:
|
||||
1. 运行时配置(RunnableConfig)：最高优先级，直接从函数参数传入
|
||||
2. 类默认配置：最低优先级，类中定义的默认值
|
||||
"""
|
||||
|
||||
def update(self, data: dict):
|
||||
"""更新配置字段"""
|
||||
for key, value in data.items():
|
||||
if hasattr(self, key):
|
||||
setattr(self, key, value)
|
||||
|
||||
thread_id: str = field(
|
||||
default_factory=lambda: str(uuid.uuid4()),
|
||||
metadata={"name": "线程ID", "configurable": False, "description": "用来唯一标识一个对话线程"},
|
||||
)
|
||||
|
||||
uid: str = field(
|
||||
default_factory=lambda: str(uuid.uuid4()),
|
||||
metadata={"name": "UID", "configurable": False, "description": "用来唯一标识一个用户"},
|
||||
)
|
||||
|
||||
run_id: str | None = field(
|
||||
default=None,
|
||||
metadata={"name": "运行 ID", "configurable": False, "hide": True},
|
||||
)
|
||||
|
||||
request_id: str | None = field(
|
||||
default=None,
|
||||
metadata={"name": "请求 ID", "configurable": False, "hide": True},
|
||||
)
|
||||
|
||||
system_prompt: str = field(
|
||||
default="You are a helpful assistant.",
|
||||
metadata={"name": "系统提示词", "description": "用来描述智能体的角色和行为", "kind": "prompt"},
|
||||
)
|
||||
|
||||
model: str = field(
|
||||
default="",
|
||||
metadata={
|
||||
"name": "智能体模型",
|
||||
"options": [],
|
||||
"description": "智能体的驱动模型，留空时使用系统默认模型。",
|
||||
"kind": "llm",
|
||||
},
|
||||
)
|
||||
|
||||
tools: list[str] | None = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"name": "工具",
|
||||
"description": "内置的工具。默认选择当前用户可用的全部工具。",
|
||||
"type": "list",
|
||||
"kind": "tools",
|
||||
},
|
||||
)
|
||||
|
||||
knowledges: list[str] | None = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"name": "知识库",
|
||||
"description": "知识库列表，可以在左侧知识库页面中创建知识库。默认选择当前用户可访问的全部知识库。",
|
||||
"type": "list",
|
||||
"kind": "knowledges",
|
||||
},
|
||||
)
|
||||
|
||||
mcps: list[str] | None = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"name": "MCP服务器",
|
||||
"options": [],
|
||||
"description": (
|
||||
"MCP服务器列表，默认选择当前用户可用的全部 MCP 服务器。建议使用支持 SSE 的 MCP 服务器，"
|
||||
"如果需要使用 uvx 或 npx 运行的服务器，也请在项目外部启动 MCP 服务器，并在项目中配置 MCP 服务器。"
|
||||
),
|
||||
"type": "list",
|
||||
"kind": "mcps",
|
||||
},
|
||||
)
|
||||
|
||||
skills: list[str] | None = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"name": "Skills",
|
||||
"options": [],
|
||||
"description": "可选 Skill 拓展列表，默认选择当前用户可用的全部 Skill 拓展。"
|
||||
"Skill 拓展依赖的工具和 MCP 服务器也会被自动挂载。",
|
||||
"type": "list",
|
||||
"kind": "skills",
|
||||
},
|
||||
)
|
||||
|
||||
summary_threshold: int = field(
|
||||
default=DEFAULT_SUMMARY_THRESHOLD_K,
|
||||
metadata={
|
||||
"name": "上下文摘要触发阈值 (K)",
|
||||
"description": (
|
||||
f"当上下文大小超过该值时，启用摘要功能以优化上下文使用。单位为 K，默认值为 "
|
||||
f"{DEFAULT_SUMMARY_THRESHOLD_K}K。"
|
||||
),
|
||||
"type": "number",
|
||||
"auth": "admin",
|
||||
},
|
||||
)
|
||||
|
||||
summary_keep_messages: int = field(
|
||||
default=DEFAULT_SUMMARY_KEEP_MESSAGES,
|
||||
metadata={
|
||||
"name": "摘要后保留消息数",
|
||||
"description": (
|
||||
f"上下文摘要触发后，除摘要消息外保留最近的消息数量，默认 {DEFAULT_SUMMARY_KEEP_MESSAGES} 条。"
|
||||
),
|
||||
"type": "number",
|
||||
"auth": "admin",
|
||||
},
|
||||
)
|
||||
|
||||
summary_prompt: str = field(
|
||||
default=DEFAULT_YUXI_SUMMARY_PROMPT,
|
||||
metadata={
|
||||
"name": "上下文摘要提示词",
|
||||
"description": "触发上下文摘要时使用的提示词，必须能接收 {messages} 作为待摘要消息占位符。",
|
||||
"type": "string",
|
||||
"kind": "prompt",
|
||||
"auth": "admin",
|
||||
},
|
||||
)
|
||||
|
||||
summary_tool_result_token_limit: int = field(
|
||||
default=DEFAULT_SUMMARY_TOOL_RESULT_TOKEN_LIMIT,
|
||||
metadata={
|
||||
"name": "摘要工具结果 token 上限",
|
||||
"description": (
|
||||
"上下文摘要 L1 清洗历史工具结果时，超过该 token 数的 ToolMessage 会写入 outputs，"
|
||||
"并在上下文中保留不超过该 token 数的预览；未超过则保持原样。默认 "
|
||||
f"{DEFAULT_SUMMARY_TOOL_RESULT_TOKEN_LIMIT}。"
|
||||
),
|
||||
"type": "number",
|
||||
"auth": "admin",
|
||||
},
|
||||
)
|
||||
|
||||
summary_l2_trigger_ratio: float = field(
|
||||
default=DEFAULT_SUMMARY_L2_TRIGGER_RATIO,
|
||||
metadata={
|
||||
"name": "L2 摘要触发比例",
|
||||
"description": (
|
||||
"L1 结构精简后，剩余上下文超过 摘要触发阈值 * 该比例 时才进入 L2 summary。"
|
||||
"建议范围 0.1 到 1.0，值越小越容易触发 L2，默认 "
|
||||
f"{DEFAULT_SUMMARY_L2_TRIGGER_RATIO}。"
|
||||
),
|
||||
"type": "number",
|
||||
"auth": "admin",
|
||||
},
|
||||
)
|
||||
|
||||
max_execution_steps: int = field(
|
||||
default=DEFAULT_MAX_EXECUTION_STEPS,
|
||||
metadata={
|
||||
"name": "最大执行步数",
|
||||
"description": (
|
||||
"单次 Agent 运行允许的最大 LangGraph 执行步数，对应 recursion_limit，默认 "
|
||||
f"{DEFAULT_MAX_EXECUTION_STEPS}。"
|
||||
),
|
||||
"type": "number",
|
||||
"auth": "admin",
|
||||
},
|
||||
)
|
||||
|
||||
model_retry_times: int = field(
|
||||
default=2,
|
||||
metadata={
|
||||
"name": "模型重试次数",
|
||||
"description": "模型调用失败时的最大重试次数，默认值为 2。",
|
||||
"type": "number",
|
||||
"auth": "admin",
|
||||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_configurable_items(cls, user_role: str | None = None):
|
||||
"""实现一个可配置的参数列表，在 UI 上配置时使用"""
|
||||
configurable_items = {}
|
||||
for f in fields(cls):
|
||||
if f.init and not f.metadata.get("hide", False):
|
||||
if user_role is not None and not _role_can_access(f.metadata.get("auth"), user_role):
|
||||
continue
|
||||
if f.metadata.get("configurable", True):
|
||||
type_name = cls._get_type_name(f.type)
|
||||
|
||||
options = f.metadata.get("options", [])
|
||||
if callable(options):
|
||||
options = options()
|
||||
|
||||
configurable_items[f.name] = {
|
||||
"type": f.metadata.get("type", type_name),
|
||||
"name": f.metadata.get("name", f.name),
|
||||
"options": options,
|
||||
"default": f.default
|
||||
if f.default is not MISSING
|
||||
else f.default_factory()
|
||||
if f.default_factory is not MISSING
|
||||
else None,
|
||||
"description": f.metadata.get("description", ""),
|
||||
"kind": f.metadata.get("kind", ""),
|
||||
}
|
||||
|
||||
return configurable_items
|
||||
|
||||
@classmethod
|
||||
def _get_type_name(cls, field_type) -> str:
|
||||
"""获取类型名称"""
|
||||
origin = get_origin(field_type)
|
||||
if origin is not None:
|
||||
if hasattr(origin, "__name__"):
|
||||
return origin.__name__
|
||||
return str(origin)
|
||||
elif hasattr(field_type, "__name__"):
|
||||
return field_type.__name__
|
||||
else:
|
||||
return str(field_type)
|
||||
|
||||
def update_from_dict(self, data: dict):
|
||||
"""从字典更新配置字段"""
|
||||
for key, value in data.items():
|
||||
if hasattr(self, key):
|
||||
setattr(self, key, value)
|
||||
|
||||
|
||||
_DEFAULT_ALL_CONTEXT_FIELDS = frozenset({"tools", "knowledges", "mcps", "skills"})
|
||||
_EMPTY_ALL_CONTEXT_FIELDS = frozenset({"subagents"})
|
||||
_AGENT_RESOURCE_FIELDS = _DEFAULT_ALL_CONTEXT_FIELDS | _EMPTY_ALL_CONTEXT_FIELDS
|
||||
|
||||
|
||||
def _normalize_selected_resource_keys(value: Any, available: list[str]) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
|
||||
allowed = set(available)
|
||||
normalized: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in value:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
key = item.strip()
|
||||
if not key or key in seen or key not in allowed:
|
||||
continue
|
||||
seen.add(key)
|
||||
normalized.append(key)
|
||||
return normalized
|
||||
|
||||
|
||||
def _resource_fields_requiring_available_keys(normalized: dict, resource_fields: set[str]) -> set[str]:
|
||||
fields_to_load: set[str] = set()
|
||||
for field_name in resource_fields:
|
||||
current = normalized.get(field_name)
|
||||
if current is None:
|
||||
if field_name in _DEFAULT_ALL_CONTEXT_FIELDS | _EMPTY_ALL_CONTEXT_FIELDS:
|
||||
fields_to_load.add(field_name)
|
||||
else:
|
||||
normalized[field_name] = []
|
||||
elif field_name in _EMPTY_ALL_CONTEXT_FIELDS and current == []:
|
||||
normalized[field_name] = None
|
||||
fields_to_load.add(field_name)
|
||||
elif isinstance(current, list) and current:
|
||||
fields_to_load.add(field_name)
|
||||
else:
|
||||
normalized[field_name] = []
|
||||
return fields_to_load
|
||||
|
||||
|
||||
def _resource_option(key: Any, name: Any = None, description: Any = None) -> dict[str, str]:
|
||||
key_value = str(key)
|
||||
return {
|
||||
"key": key_value,
|
||||
"name": str(name or key_value),
|
||||
"description": str(description or ""),
|
||||
}
|
||||
|
||||
|
||||
async def resolve_agent_resource_options(
|
||||
resource_fields: set[str] | None = None,
|
||||
*,
|
||||
db,
|
||||
user,
|
||||
) -> dict[str, list[dict[str, str]]]:
|
||||
fields_to_load = _AGENT_RESOURCE_FIELDS if resource_fields is None else resource_fields
|
||||
if not fields_to_load:
|
||||
return {}
|
||||
|
||||
options: dict[str, list[dict[str, str]]] = {}
|
||||
|
||||
if "tools" in fields_to_load:
|
||||
from yuxi.agents.toolkits.service import get_tool_metadata
|
||||
|
||||
options["tools"] = [
|
||||
_resource_option(tool["slug"], tool.get("name"), tool.get("description"))
|
||||
for tool in get_tool_metadata(category="buildin")
|
||||
if tool.get("slug")
|
||||
]
|
||||
if "knowledges" in fields_to_load:
|
||||
from yuxi.knowledge import knowledge_base
|
||||
|
||||
databases = (await knowledge_base.get_databases_by_user(user)).get("databases", [])
|
||||
options["knowledges"] = [
|
||||
_resource_option(item.get("kb_id"), item.get("name"), item.get("description"))
|
||||
for item in databases
|
||||
if isinstance(item, dict) and item.get("kb_id")
|
||||
]
|
||||
if "mcps" in fields_to_load:
|
||||
from yuxi.agents.mcp.service import get_all_mcp_servers
|
||||
|
||||
servers = await get_all_mcp_servers(db)
|
||||
options["mcps"] = [
|
||||
_resource_option(server.slug, server.name, server.description)
|
||||
for server in servers
|
||||
if server.enabled and server.slug
|
||||
]
|
||||
if "skills" in fields_to_load:
|
||||
from yuxi.agents.skills.service import list_accessible_skills
|
||||
|
||||
skills = await list_accessible_skills(db, user)
|
||||
options["skills"] = [
|
||||
_resource_option(skill.slug, skill.name, skill.description) for skill in skills if skill.slug
|
||||
]
|
||||
if "subagents" in fields_to_load:
|
||||
from yuxi.repositories.agent_repository import AgentRepository
|
||||
|
||||
subagents = await AgentRepository(db).list_visible_subagents(user=user)
|
||||
options["subagents"] = [
|
||||
_resource_option(agent.slug, agent.name, agent.description) for agent in subagents if agent.slug
|
||||
]
|
||||
|
||||
return options
|
||||
|
||||
|
||||
async def normalize_agent_context_config(
|
||||
context: dict | None,
|
||||
*,
|
||||
db,
|
||||
user,
|
||||
context_schema: type[BaseContext] | None = None,
|
||||
) -> dict:
|
||||
schema = context_schema or BaseContext
|
||||
raw_context = dict(context) if isinstance(context, dict) else {}
|
||||
filtered = filter_config_by_role({"context": raw_context}, getattr(user, "role", None), schema)
|
||||
normalized = dict(filtered.get("context") or {})
|
||||
field_names = {item.name for item in fields(schema)}
|
||||
resource_fields = _AGENT_RESOURCE_FIELDS & field_names
|
||||
if not resource_fields:
|
||||
return normalized
|
||||
|
||||
fields_to_load = _resource_fields_requiring_available_keys(normalized, resource_fields)
|
||||
if not fields_to_load:
|
||||
return normalized
|
||||
|
||||
resource_options = await resolve_agent_resource_options(fields_to_load, db=db, user=user)
|
||||
available = {
|
||||
field_name: [option["key"] for option in field_options]
|
||||
for field_name, field_options in resource_options.items()
|
||||
}
|
||||
|
||||
for field_name, available_keys in available.items():
|
||||
current = normalized.get(field_name)
|
||||
if current is None:
|
||||
normalized[field_name] = available_keys
|
||||
else:
|
||||
normalized[field_name] = _normalize_selected_resource_keys(current, available_keys)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
async def prepare_agent_runtime_context(
|
||||
context: BaseContext,
|
||||
*,
|
||||
context_schema: type[BaseContext] | None = None,
|
||||
) -> BaseContext:
|
||||
"""准备 Agent 运行时上下文，主要是根据 context 中的 uid 加载用户可访问的资源列表，并进行规范化处理。"""
|
||||
schema = context_schema or type(context)
|
||||
uid = str(getattr(context, "uid", "") or "").strip()
|
||||
if not uid:
|
||||
return context
|
||||
|
||||
from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context
|
||||
from yuxi.agents.middlewares.skills import resolve_runtime_skills_for_context
|
||||
from yuxi.repositories.user_repository import UserRepository
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
|
||||
resource_fields = _AGENT_RESOURCE_FIELDS
|
||||
async with pg_manager.get_async_session_context() as db:
|
||||
user = await UserRepository().get_by_uid_with_db(db, uid)
|
||||
if user is None:
|
||||
for field_name in resource_fields:
|
||||
if hasattr(context, field_name):
|
||||
setattr(context, field_name, [])
|
||||
setattr(context, "_visible_knowledge_bases", [])
|
||||
setattr(context, "_prompt_skills", [])
|
||||
setattr(context, "_readable_skills", [])
|
||||
setattr(context, "_runtime_skill_metadata", {})
|
||||
setattr(context, "_runtime_skill_dependency_map", {})
|
||||
return context
|
||||
|
||||
raw_resources = {
|
||||
field_name: getattr(context, field_name, None)
|
||||
for field_name in resource_fields
|
||||
if hasattr(context, field_name)
|
||||
}
|
||||
normalized = await normalize_agent_context_config(
|
||||
raw_resources,
|
||||
db=db,
|
||||
user=user,
|
||||
context_schema=schema,
|
||||
)
|
||||
for field_name in resource_fields:
|
||||
if hasattr(context, field_name):
|
||||
setattr(context, field_name, normalized.get(field_name, []))
|
||||
|
||||
await resolve_visible_knowledge_bases_for_context(context)
|
||||
skill_scope = await resolve_runtime_skills_for_context(context, db=db, user=user)
|
||||
context.skills = skill_scope["context_skills"]
|
||||
setattr(context, "_prompt_skills", skill_scope["prompt_skills"])
|
||||
setattr(context, "_readable_skills", skill_scope["readable_skills"])
|
||||
setattr(context, "_runtime_skill_metadata", skill_scope["runtime_skill_metadata"])
|
||||
setattr(context, "_runtime_skill_dependency_map", skill_scope["runtime_skill_dependency_map"])
|
||||
|
||||
return context
|
||||
@@ -0,0 +1,83 @@
|
||||
"""MCP 服务器数据访问层 - Repository"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
from yuxi.storage.postgres.models_business import MCPServer
|
||||
|
||||
|
||||
class MCPServerRepository:
|
||||
"""MCP 服务器数据访问层"""
|
||||
|
||||
async def get_by_slug(self, slug: str) -> MCPServer | None:
|
||||
"""根据名称获取 MCP 服务器"""
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(MCPServer).where(MCPServer.slug == slug))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def list(self) -> list[MCPServer]:
|
||||
"""获取所有 MCP 服务器"""
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(MCPServer))
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def list_enabled(self) -> list[MCPServer]:
|
||||
"""获取所有启用的 MCP 服务器"""
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(MCPServer).where(MCPServer.enabled == 1))
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def create(self, data: dict[str, Any]) -> MCPServer:
|
||||
"""创建 MCP 服务器"""
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
server = MCPServer(**data)
|
||||
session.add(server)
|
||||
return server
|
||||
|
||||
async def update(self, slug: str, data: dict[str, Any]) -> MCPServer | None:
|
||||
"""更新 MCP 服务器"""
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(MCPServer).where(MCPServer.slug == slug))
|
||||
server = result.scalar_one_or_none()
|
||||
if server is None:
|
||||
return None
|
||||
for key, value in data.items():
|
||||
if key != "slug":
|
||||
setattr(server, key, value)
|
||||
return server
|
||||
|
||||
async def delete(self, slug: str) -> bool:
|
||||
"""删除 MCP 服务器"""
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(MCPServer).where(MCPServer.slug == slug))
|
||||
server = result.scalar_one_or_none()
|
||||
if server is None:
|
||||
return False
|
||||
await session.delete(server)
|
||||
return True
|
||||
|
||||
async def upsert(self, data: dict[str, Any]) -> MCPServer:
|
||||
"""插入或更新 MCP 服务器"""
|
||||
slug = data.get("slug")
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(MCPServer).where(MCPServer.slug == slug))
|
||||
existing = result.scalar_one_or_none()
|
||||
if existing is None:
|
||||
server = MCPServer(**data)
|
||||
session.add(server)
|
||||
else:
|
||||
for key, value in data.items():
|
||||
if key != "slug":
|
||||
setattr(existing, key, value)
|
||||
server = existing
|
||||
return server
|
||||
|
||||
async def exists_by_slug(self, slug: str) -> bool:
|
||||
"""检查 MCP 服务器是否存在"""
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(MCPServer.id).where(MCPServer.slug == slug))
|
||||
return result.scalar_one_or_none() is not None
|
||||
@@ -0,0 +1,609 @@
|
||||
"""MCP Service - Unified business logic and state management for MCP.
|
||||
|
||||
Responsibilities:
|
||||
- Server configuration CRUD operations
|
||||
- Built-in configuration synchronization (Code <-> Database)
|
||||
- Unified entry point for Agent tool retrieval (auto-filtering disabled_tools)
|
||||
- MCP Client and Tools management (formerly in agents/common/mcp.py)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any, cast
|
||||
|
||||
from langchain_mcp_adapters.client import MultiServerMCPClient
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from yuxi.storage.postgres.models_business import MCPServer
|
||||
from yuxi.utils import logger
|
||||
|
||||
# =============================================================================
|
||||
# === Global Cache & State ===
|
||||
# =============================================================================
|
||||
|
||||
# Global Lock for MCP state
|
||||
_mcp_lock = asyncio.Lock()
|
||||
|
||||
# 本地仅缓存工具对象。配置始终以数据库为准，每次按 server_slug 现查。
|
||||
# cache key 使用 server_slug:config_hash，当配置变化时会自然失效。
|
||||
_mcp_tools_cache: dict[str, list[Callable[..., Any]]] = {}
|
||||
|
||||
# MCP tools statistics (for reporting enabled/disabled counts)
|
||||
_mcp_tools_stats: dict[str, dict[str, int]] = {}
|
||||
_UNSET = object()
|
||||
|
||||
# Default MCP Server configurations (Imported to DB on first run)
|
||||
_DEFAULT_MCP_SERVERS = {
|
||||
"mcp-server-chart": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@antv/mcp-server-chart"],
|
||||
"transport": "stdio",
|
||||
"description": "图表生成工具，支持生成各类图表（柱状图、折线图、饼图等）",
|
||||
"icon": "📊",
|
||||
"tags": ["内置", "图表"],
|
||||
},
|
||||
}
|
||||
|
||||
_RETIRED_BUILTIN_MCP_SERVER_SLUGS = ("sequentialthinking",)
|
||||
|
||||
_SYNCED_MCP_FIELDS = (
|
||||
"description",
|
||||
"transport",
|
||||
"url",
|
||||
"command",
|
||||
"args",
|
||||
"env",
|
||||
"headers",
|
||||
"timeout",
|
||||
"sse_read_timeout",
|
||||
"tags",
|
||||
"icon",
|
||||
)
|
||||
|
||||
# =============================================================================
|
||||
# === Core Logic (Moved from agents/common/mcp.py) ===
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def ensure_builtin_mcp_servers_in_db() -> None:
|
||||
"""Ensure built-in MCP server definitions exist in the database."""
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
|
||||
try:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
any_changed = False
|
||||
for slug in _RETIRED_BUILTIN_MCP_SERVER_SLUGS:
|
||||
result = await session.execute(
|
||||
select(MCPServer).filter(MCPServer.slug == slug, MCPServer.created_by == "system")
|
||||
)
|
||||
retired = result.scalar_one_or_none()
|
||||
if retired:
|
||||
await session.delete(retired)
|
||||
clear_mcp_server_tools_cache(slug)
|
||||
any_changed = True
|
||||
logger.info(f"Removed retired built-in MCP server '{slug}' from database")
|
||||
|
||||
for slug, config in _DEFAULT_MCP_SERVERS.items():
|
||||
result = await session.execute(select(MCPServer).filter(MCPServer.slug == slug))
|
||||
existing = result.scalar_one_or_none()
|
||||
if not existing:
|
||||
session.add(
|
||||
MCPServer(
|
||||
slug=slug,
|
||||
name=config.get("name", slug),
|
||||
description=config.get("description"),
|
||||
transport=config["transport"],
|
||||
url=config.get("url"),
|
||||
command=config.get("command"),
|
||||
args=config.get("args"),
|
||||
env=config.get("env"),
|
||||
headers=config.get("headers"),
|
||||
timeout=config.get("timeout"),
|
||||
sse_read_timeout=config.get("sse_read_timeout"),
|
||||
tags=config.get("tags"),
|
||||
icon=config.get("icon"),
|
||||
enabled=0,
|
||||
created_by="system",
|
||||
updated_by="system",
|
||||
)
|
||||
)
|
||||
any_changed = True
|
||||
logger.info(f"Added built-in MCP server '{slug}' to database")
|
||||
continue
|
||||
|
||||
server_changed = False
|
||||
for field in _SYNCED_MCP_FIELDS:
|
||||
next_value = config.get(field)
|
||||
if getattr(existing, field) != next_value:
|
||||
setattr(existing, field, next_value)
|
||||
server_changed = True
|
||||
if server_changed:
|
||||
existing.updated_by = "system"
|
||||
any_changed = True
|
||||
|
||||
if any_changed:
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to ensure builtin MCP servers in database: {e}")
|
||||
|
||||
|
||||
async def get_mcp_client(
|
||||
server_configs: dict[str, Any] | None = None,
|
||||
) -> MultiServerMCPClient | None:
|
||||
"""Initializes an MCP client with the given server configurations."""
|
||||
try:
|
||||
client = MultiServerMCPClient(server_configs) # pyright: ignore[reportArgumentType]
|
||||
logger.info(f"Initialized MCP client with servers: {list(server_configs.keys())}")
|
||||
return client
|
||||
except Exception as e:
|
||||
logger.error("Failed to initialize MCP client: {}", e)
|
||||
return None
|
||||
|
||||
|
||||
def to_camel_case(s: str) -> str:
|
||||
"""Convert string to lowerCamelCase."""
|
||||
|
||||
# Handle - and _
|
||||
s = re.sub(r"[-_]+(.)", lambda m: m.group(1).upper(), s)
|
||||
# Lowercase first letter
|
||||
if len(s) > 0:
|
||||
s = s[0].lower() + s[1:]
|
||||
return s
|
||||
|
||||
|
||||
async def _load_enabled_mcp_server_configs(
|
||||
*,
|
||||
names: list[str] | None = None,
|
||||
db: AsyncSession | None = None,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Load enabled MCP server configs directly from the database."""
|
||||
if db is not None:
|
||||
stmt = select(MCPServer).where(MCPServer.enabled == 1)
|
||||
if names:
|
||||
stmt = stmt.where(MCPServer.slug.in_(names))
|
||||
result = await db.execute(stmt)
|
||||
servers = result.scalars().all()
|
||||
return {server.slug: server.to_mcp_config() for server in servers}
|
||||
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
return await _load_enabled_mcp_server_configs(names=names, db=session)
|
||||
|
||||
|
||||
async def get_enabled_mcp_server_config(server_slug: str, *, db: AsyncSession | None = None) -> dict[str, Any] | None:
|
||||
"""Get the latest enabled MCP server config from the database."""
|
||||
configs = await _load_enabled_mcp_server_configs(names=[server_slug], db=db)
|
||||
return configs.get(server_slug)
|
||||
|
||||
|
||||
async def get_enabled_mcp_server_slugs(*, db: AsyncSession | None = None) -> list[str]:
|
||||
"""Get enabled MCP server slugs from the database."""
|
||||
if db is not None:
|
||||
result = await db.execute(select(MCPServer.slug).where(MCPServer.enabled == 1))
|
||||
return [name for name in result.scalars().all() if isinstance(name, str)]
|
||||
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
return await get_enabled_mcp_server_slugs(db=session)
|
||||
|
||||
|
||||
async def get_mcp_tools(
|
||||
server_slug: str,
|
||||
additional_servers: dict[str, dict[str, Any]] | None = None,
|
||||
disabled_tools: list[str] = None,
|
||||
cache: bool = True,
|
||||
force_refresh: bool = False,
|
||||
) -> list[Callable[..., Any]]:
|
||||
"""Get MCP tools for a specific server.
|
||||
|
||||
Architecture:
|
||||
1. Fetching: Connects to MCP server to get ALL tools.
|
||||
2. Caching: Stores the FULL, UNFILTERED list of tools in `_mcp_tools_cache`.
|
||||
3. Filtering: Filters the return value based on `disabled_tools` argument.
|
||||
|
||||
Args:
|
||||
server_slug: Server slug
|
||||
additional_servers: Additional server configurations
|
||||
disabled_tools: List of tool names to filter out from the RETURN value (does not affect cache)
|
||||
cache: Whether to use/update the cache (default: True)
|
||||
force_refresh: Whether to force a refresh from the server (default: False)
|
||||
"""
|
||||
if additional_servers and server_slug in additional_servers:
|
||||
server_config = additional_servers[server_slug]
|
||||
else:
|
||||
server_config = await get_enabled_mcp_server_config(server_slug)
|
||||
|
||||
if server_config is None:
|
||||
logger.warning(f"MCP server '{server_slug}' not found in database or disabled")
|
||||
return []
|
||||
|
||||
# 配置 hash 直接基于完整配置生成。只要数据库中的配置发生变化，
|
||||
# 本地工具缓存 key 就会变化，从而自然触发重建。
|
||||
config_payload = json.dumps(server_config, sort_keys=True, ensure_ascii=True, separators=(",", ":"))
|
||||
config_hash = hashlib.sha256(config_payload.encode("utf-8")).hexdigest()[:16]
|
||||
cache_key = f"{server_slug}:{config_hash}"
|
||||
|
||||
all_processed_tools: list[Callable[..., Any]] = []
|
||||
|
||||
async with _mcp_lock:
|
||||
if not force_refresh and cache and cache_key in _mcp_tools_cache:
|
||||
all_processed_tools = _mcp_tools_cache[cache_key]
|
||||
|
||||
if not all_processed_tools:
|
||||
try:
|
||||
# disabled_tools 只影响返回值过滤，不参与 MCP client 建连参数。
|
||||
client_config = {k: v for k, v in server_config.items() if k not in ("disabled_tools",)}
|
||||
|
||||
client = await get_mcp_client({server_slug: client_config})
|
||||
if client is None:
|
||||
return []
|
||||
|
||||
raw_tools = cast(list[Any], await client.get_tools())
|
||||
|
||||
server_cc = to_camel_case(server_slug)
|
||||
for tool in raw_tools:
|
||||
original_name = tool.name
|
||||
tool_cc = to_camel_case(original_name)
|
||||
unique_id = f"mcp__{server_cc}__{tool_cc}"
|
||||
|
||||
if tool.metadata is None:
|
||||
tool.metadata = {}
|
||||
tool.metadata["id"] = unique_id
|
||||
# 开启错误处理，防止工具调用抛出 ToolException 时击穿服务
|
||||
tool.handle_tool_error = True
|
||||
all_processed_tools.append(tool)
|
||||
|
||||
if cache:
|
||||
async with _mcp_lock:
|
||||
stale_keys = [
|
||||
key for key in _mcp_tools_cache if key.startswith(f"{server_slug}:") and key != cache_key
|
||||
]
|
||||
for stale_key in stale_keys:
|
||||
_mcp_tools_cache.pop(stale_key, None)
|
||||
_mcp_tools_cache[cache_key] = all_processed_tools
|
||||
|
||||
global_config_disabled = server_config.get("disabled_tools") or []
|
||||
enabled_count = len([t for t in all_processed_tools if t.name not in global_config_disabled])
|
||||
_mcp_tools_stats[server_slug] = {
|
||||
"total": len(all_processed_tools),
|
||||
"enabled": enabled_count,
|
||||
"disabled": len(all_processed_tools) - enabled_count,
|
||||
}
|
||||
|
||||
logger.info(
|
||||
f"Refreshed MCP tools cache for '{server_slug}' with key '{cache_key}': "
|
||||
f"{len(all_processed_tools)} tools loaded."
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to load tools from MCP server '{server_slug}': {e}")
|
||||
return []
|
||||
|
||||
# 3. Filtering (Apply to Return Value Only)
|
||||
if disabled_tools:
|
||||
filtered_tools = [t for t in all_processed_tools if t.name not in disabled_tools]
|
||||
logger.debug(
|
||||
f"Returning {len(filtered_tools)}/{len(all_processed_tools)} tools for '{server_slug}' "
|
||||
f"(filtered {len(disabled_tools)} by argument)"
|
||||
)
|
||||
return filtered_tools
|
||||
|
||||
return all_processed_tools
|
||||
|
||||
|
||||
async def get_tools_from_all_servers() -> list[Callable[..., Any]]:
|
||||
"""Get all tools from all configured MCP servers."""
|
||||
server_configs = await _load_enabled_mcp_server_configs()
|
||||
all_tools = []
|
||||
for server_slug in server_configs:
|
||||
tools = await get_mcp_tools(server_slug, additional_servers=server_configs)
|
||||
all_tools.extend(tools)
|
||||
return all_tools
|
||||
|
||||
|
||||
def clear_mcp_cache() -> None:
|
||||
"""Clear the MCP tools cache (useful for testing)."""
|
||||
global _mcp_tools_cache, _mcp_tools_stats
|
||||
_mcp_tools_cache = {}
|
||||
_mcp_tools_stats = {}
|
||||
|
||||
|
||||
def clear_mcp_server_tools_cache(server_slug: str) -> None:
|
||||
"""Clear the tools cache for a specific MCP server."""
|
||||
global _mcp_tools_cache, _mcp_tools_stats
|
||||
server_prefix = f"{server_slug}:"
|
||||
stale_keys = [key for key in _mcp_tools_cache if key.startswith(server_prefix)]
|
||||
for stale_key in stale_keys:
|
||||
_mcp_tools_cache.pop(stale_key, None)
|
||||
_mcp_tools_stats.pop(server_slug, None)
|
||||
logger.info(f"Cleared tools cache for MCP server '{server_slug}'")
|
||||
|
||||
|
||||
def get_mcp_tools_stats(server_slug: str) -> dict[str, int] | None:
|
||||
"""Get tools statistics for a MCP server.
|
||||
|
||||
Returns:
|
||||
dict with 'total', 'enabled', 'disabled' counts, or None if not available
|
||||
"""
|
||||
return _mcp_tools_stats.get(server_slug)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# === Server Config CRUD ===
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def get_mcp_server(db: AsyncSession, slug: str) -> MCPServer | None:
|
||||
"""Get single server configuration by slug."""
|
||||
result = await db.execute(select(MCPServer).filter(MCPServer.slug == slug))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_all_mcp_servers(db: AsyncSession) -> list[MCPServer]:
|
||||
"""Get all server configurations."""
|
||||
result = await db.execute(select(MCPServer))
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def create_mcp_server(
|
||||
db: AsyncSession,
|
||||
slug: str,
|
||||
name: str,
|
||||
transport: str,
|
||||
url: str = None,
|
||||
command: str = None,
|
||||
args: list = None,
|
||||
env: dict = None,
|
||||
description: str = None,
|
||||
headers: dict = None,
|
||||
timeout: int = None,
|
||||
sse_read_timeout: int = None,
|
||||
tags: list = None,
|
||||
icon: str = None,
|
||||
created_by: str = None,
|
||||
) -> MCPServer:
|
||||
"""Create server."""
|
||||
existing = await get_mcp_server(db, slug)
|
||||
if existing:
|
||||
raise ValueError(f"Server slug '{slug}' already exists")
|
||||
|
||||
server = MCPServer(
|
||||
slug=slug,
|
||||
name=name,
|
||||
description=description,
|
||||
transport=transport,
|
||||
url=url,
|
||||
command=command,
|
||||
args=args,
|
||||
env=env,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
sse_read_timeout=sse_read_timeout,
|
||||
tags=tags,
|
||||
icon=icon,
|
||||
enabled=1,
|
||||
created_by=created_by,
|
||||
updated_by=created_by,
|
||||
)
|
||||
db.add(server)
|
||||
await db.commit()
|
||||
await db.refresh(server)
|
||||
|
||||
clear_mcp_server_tools_cache(slug)
|
||||
|
||||
logger.info(f"Created MCP server '{slug}'")
|
||||
return server
|
||||
|
||||
|
||||
async def update_mcp_server(
|
||||
db: AsyncSession,
|
||||
slug: str,
|
||||
name: str = None,
|
||||
description: str = None,
|
||||
transport: str = None,
|
||||
url: str = None,
|
||||
command: str = None,
|
||||
args: list = None,
|
||||
env: Any = _UNSET,
|
||||
headers: dict = None,
|
||||
timeout: int = None,
|
||||
sse_read_timeout: int = None,
|
||||
tags: list = None,
|
||||
icon: str = None,
|
||||
updated_by: str = None,
|
||||
) -> MCPServer:
|
||||
"""Update server configuration."""
|
||||
server = await get_mcp_server(db, slug)
|
||||
if not server:
|
||||
raise ValueError(f"Server '{slug}' does not exist")
|
||||
|
||||
if name is not None:
|
||||
server.name = name
|
||||
if description is not None:
|
||||
server.description = description
|
||||
if transport is not None:
|
||||
server.transport = transport
|
||||
if url is not None:
|
||||
server.url = url
|
||||
if command is not None:
|
||||
server.command = command
|
||||
if args is not None:
|
||||
server.args = args
|
||||
if env is not _UNSET:
|
||||
server.env = env
|
||||
if headers is not None:
|
||||
server.headers = headers
|
||||
if timeout is not None:
|
||||
server.timeout = timeout
|
||||
if sse_read_timeout is not None:
|
||||
server.sse_read_timeout = sse_read_timeout
|
||||
if tags is not None:
|
||||
server.tags = tags
|
||||
if icon is not None:
|
||||
server.icon = icon
|
||||
if updated_by is not None:
|
||||
server.updated_by = updated_by
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(server)
|
||||
|
||||
clear_mcp_server_tools_cache(slug)
|
||||
|
||||
logger.info(f"Updated MCP server '{slug}'")
|
||||
return server
|
||||
|
||||
|
||||
async def delete_mcp_server(db: AsyncSession, slug: str) -> bool:
|
||||
"""Delete server."""
|
||||
server = await get_mcp_server(db, slug)
|
||||
if not server:
|
||||
return False
|
||||
|
||||
await db.delete(server)
|
||||
await db.commit()
|
||||
|
||||
clear_mcp_server_tools_cache(slug)
|
||||
|
||||
logger.info(f"Deleted MCP server '{slug}'")
|
||||
return True
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# === Tool Management ===
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def set_server_enabled(
|
||||
db: AsyncSession, slug: str, enabled: bool, updated_by: str = None
|
||||
) -> tuple[bool, MCPServer]:
|
||||
"""Set server enabled status."""
|
||||
server = await get_mcp_server(db, slug)
|
||||
if not server:
|
||||
raise ValueError(f"Server '{slug}' does not exist")
|
||||
|
||||
server.enabled = 1 if enabled else 0
|
||||
if updated_by is not None:
|
||||
server.updated_by = updated_by
|
||||
await db.commit()
|
||||
|
||||
is_enabled = bool(server.enabled)
|
||||
clear_mcp_server_tools_cache(slug)
|
||||
|
||||
logger.info(f"Set MCP server '{slug}' enabled={is_enabled}")
|
||||
return is_enabled, server
|
||||
|
||||
|
||||
async def toggle_tool_enabled(
|
||||
db: AsyncSession,
|
||||
server_slug: str,
|
||||
tool_name: str,
|
||||
updated_by: str = None,
|
||||
) -> tuple[bool, MCPServer]:
|
||||
"""Toggle single tool enabled status.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
server_slug: Server slug
|
||||
tool_name: Tool name
|
||||
updated_by: Updater
|
||||
|
||||
Returns:
|
||||
(enabled, server): Tool enabled status and updated server object
|
||||
"""
|
||||
server = await get_mcp_server(db, server_slug)
|
||||
if not server:
|
||||
raise ValueError(f"Server '{server_slug}' does not exist")
|
||||
|
||||
disabled_tools = list(server.disabled_tools or [])
|
||||
|
||||
if tool_name in disabled_tools:
|
||||
disabled_tools.remove(tool_name)
|
||||
enabled = True
|
||||
else:
|
||||
disabled_tools.append(tool_name)
|
||||
enabled = False
|
||||
|
||||
server.disabled_tools = disabled_tools
|
||||
if updated_by is not None:
|
||||
server.updated_by = updated_by
|
||||
await db.commit()
|
||||
|
||||
# Clear tool cache (re-filtered on next fetch)
|
||||
clear_mcp_server_tools_cache(server_slug)
|
||||
|
||||
logger.info(f"Toggled tool '{tool_name}' for server '{server_slug}' enabled={enabled}")
|
||||
return enabled, server
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# === Unified Entry Points (Wrappers) ===
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def get_enabled_mcp_tools(server_slug: str) -> list:
|
||||
"""Get MCP server tools (auto-filtering disabled_tools).
|
||||
|
||||
Unified entry point for Agents, automatically:
|
||||
1. Gets the latest server config from database
|
||||
2. Gets all tools
|
||||
3. Filters out disabled_tools
|
||||
|
||||
Args:
|
||||
server_slug: Server slug
|
||||
|
||||
Returns:
|
||||
List of enabled tools
|
||||
"""
|
||||
config = await get_enabled_mcp_server_config(server_slug)
|
||||
if config is None:
|
||||
logger.warning(f"MCP server '{server_slug}' not found in database or disabled")
|
||||
return []
|
||||
|
||||
disabled_tools = config.get("disabled_tools") or []
|
||||
return await get_mcp_tools(server_slug, additional_servers={server_slug: config}, disabled_tools=disabled_tools)
|
||||
|
||||
|
||||
async def get_servers_config(names: list[str]) -> dict[str, dict[str, Any]]:
|
||||
"""Batch get server configurations.
|
||||
|
||||
Args:
|
||||
names: List of server names
|
||||
|
||||
Returns:
|
||||
{name: config} dictionary, containing only found servers
|
||||
"""
|
||||
return await _load_enabled_mcp_server_configs(names=names)
|
||||
|
||||
|
||||
async def get_all_mcp_tools(server_slug: str) -> list:
|
||||
"""Get all tools of an MCP server (no filtering).
|
||||
|
||||
For management UI to display tool list, supports viewing all tools and their enabled status.
|
||||
Does NOT update the global tools cache to avoid polluting agent's filtered view.
|
||||
|
||||
Args:
|
||||
server_slug: Server slug
|
||||
|
||||
Returns:
|
||||
List of all tools (unfiltered)
|
||||
"""
|
||||
config = await get_enabled_mcp_server_config(server_slug)
|
||||
if config is None:
|
||||
logger.warning(f"MCP server '{server_slug}' not found in database or disabled")
|
||||
return []
|
||||
|
||||
# Get all tools (no filtering, force refresh, no cache update)
|
||||
return await get_mcp_tools(
|
||||
server_slug,
|
||||
additional_servers={server_slug: config},
|
||||
disabled_tools=[],
|
||||
cache=False,
|
||||
force_refresh=True,
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
from .attachment import inject_attachment_context, save_attachments_to_fs
|
||||
from .context import context_aware_prompt, context_based_model
|
||||
from .dynamic_tool import DynamicToolMiddleware
|
||||
from .summary import create_summary_middleware
|
||||
from .token_usage import TokenUsageMiddleware
|
||||
|
||||
__all__ = [
|
||||
"DynamicToolMiddleware",
|
||||
"TokenUsageMiddleware",
|
||||
"context_aware_prompt",
|
||||
"context_based_model",
|
||||
"create_summary_middleware",
|
||||
"inject_attachment_context", # 已废弃，使用 save_attachments_to_fs
|
||||
"save_attachments_to_fs",
|
||||
]
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Attachment prompt injection middleware.
|
||||
|
||||
Read uploaded file metadata from LangGraph state and inject readable paths
|
||||
into the system prompt so the model can use `read_file` on demand.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import NotRequired
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware, ModelRequest, ModelResponse
|
||||
from langchain_core.messages import SystemMessage
|
||||
|
||||
from yuxi.utils import logger
|
||||
|
||||
ATTACHMENT_PROMPT_MARKER = "<!-- attachment_context -->"
|
||||
|
||||
|
||||
class AttachmentState(AgentState):
|
||||
"""Extended state schema with uploaded files."""
|
||||
|
||||
uploads: NotRequired[list[dict]]
|
||||
|
||||
|
||||
def _build_attachment_prompt(uploads: Sequence[dict]) -> str | None:
|
||||
"""Render uploads into a concise prompt block."""
|
||||
if not uploads:
|
||||
return None
|
||||
|
||||
valid_uploads: list[tuple[str, str]] = []
|
||||
for upload in uploads:
|
||||
path = upload.get("path")
|
||||
if not isinstance(path, str) or not path.strip():
|
||||
continue
|
||||
file_name = upload.get("file_name", "未知文件")
|
||||
valid_uploads.append((str(file_name), path))
|
||||
|
||||
if not valid_uploads:
|
||||
return None
|
||||
|
||||
upload_infos = [f"- {file_name}: {path}" for file_name, path in valid_uploads]
|
||||
lines = [
|
||||
"用户上传了以下文件：",
|
||||
"",
|
||||
*upload_infos,
|
||||
"",
|
||||
"请优先使用 `read_file` 工具读取这些路径中的文件内容，再回答用户问题。",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class AttachmentMiddleware(AgentMiddleware[AttachmentState]):
|
||||
"""Inject upload context from state.uploads into system prompt."""
|
||||
|
||||
state_schema = AttachmentState
|
||||
|
||||
async def awrap_model_call(
|
||||
self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]
|
||||
) -> ModelResponse:
|
||||
uploads = request.state.get("uploads", [])
|
||||
logger.info(f"AttachmentMiddleware: found {len(uploads)} uploads in state")
|
||||
|
||||
if uploads:
|
||||
attachment_prompt = _build_attachment_prompt(uploads)
|
||||
if attachment_prompt:
|
||||
logger.info("AttachmentMiddleware: injecting attachment prompt")
|
||||
existing_blocks = list(request.system_message.content_blocks) if request.system_message else []
|
||||
existing_text = "\n".join(
|
||||
block.get("text", "")
|
||||
for block in existing_blocks
|
||||
if isinstance(block, dict) and block.get("type") == "text"
|
||||
)
|
||||
|
||||
if ATTACHMENT_PROMPT_MARKER in existing_text:
|
||||
logger.info("AttachmentMiddleware: attachment prompt already injected, skip")
|
||||
return await handler(request)
|
||||
|
||||
merged_blocks = existing_blocks + [
|
||||
{"type": "text", "text": f"{ATTACHMENT_PROMPT_MARKER}\n{attachment_prompt}"}
|
||||
]
|
||||
request = request.override(system_message=SystemMessage(content=merged_blocks))
|
||||
|
||||
return await handler(request)
|
||||
|
||||
|
||||
save_attachments_to_fs = AttachmentMiddleware()
|
||||
inject_attachment_context = save_attachments_to_fs
|
||||
@@ -0,0 +1,25 @@
|
||||
"""通用的 Context 相关中间件"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
from langchain.agents.middleware import ModelRequest, ModelResponse, dynamic_prompt, wrap_model_call
|
||||
|
||||
from yuxi.agents import load_chat_model, resolve_chat_model_spec
|
||||
from yuxi.utils import logger
|
||||
|
||||
|
||||
@dynamic_prompt
|
||||
def context_aware_prompt(request: ModelRequest) -> str:
|
||||
"""从 runtime context 动态生成系统提示词"""
|
||||
return request.runtime.context.system_prompt
|
||||
|
||||
|
||||
@wrap_model_call
|
||||
async def context_based_model(request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]) -> ModelResponse:
|
||||
"""从 runtime context 动态选择模型"""
|
||||
model_spec = resolve_chat_model_spec(request.runtime.context.model)
|
||||
model = load_chat_model(model_spec)
|
||||
|
||||
request = request.override(model=model)
|
||||
logger.debug(f"Using model {model_spec} for request {request.messages[-1].content[:200]}")
|
||||
return await handler(request)
|
||||
@@ -0,0 +1,69 @@
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from langchain.agents.middleware import AgentMiddleware, ModelRequest, ModelResponse
|
||||
|
||||
from yuxi.agents.mcp.service import get_mcp_tools
|
||||
from yuxi.utils import logger
|
||||
|
||||
|
||||
class DynamicToolMiddleware(AgentMiddleware):
|
||||
"""动态工具选择中间件 - 支持 MCP 工具的动态加载和注册
|
||||
|
||||
注意：所有可能用到的 MCP 工具必须在初始化时预加载并注册到 self.tools
|
||||
运行时只是根据配置筛选工具，不能动态添加新工具
|
||||
"""
|
||||
|
||||
def __init__(self, base_tools: list[Any], mcp_servers: list[str] | None = None):
|
||||
"""初始化中间件
|
||||
|
||||
Args:
|
||||
base_tools: 基础工具列表
|
||||
mcp_servers: 需要预加载的 MCP 服务器列表（可选）
|
||||
"""
|
||||
super().__init__()
|
||||
self.tools: list[Any] = base_tools
|
||||
self._all_mcp_tools: dict[str, list[Any]] = {} # 所有已加载的 MCP 工具
|
||||
self._mcp_servers = mcp_servers or []
|
||||
|
||||
async def initialize_mcp_tools(self) -> None:
|
||||
"""异步初始化：预加载所有可能用到的 MCP 工具"""
|
||||
for mcp_name in self._mcp_servers:
|
||||
if mcp_name not in self._all_mcp_tools:
|
||||
logger.info(f"Pre-loading MCP tools from: {mcp_name}")
|
||||
mcp_tools = await get_mcp_tools(mcp_name)
|
||||
self._all_mcp_tools[mcp_name] = mcp_tools
|
||||
# 将 MCP 工具注册到 middleware.tools
|
||||
self.tools.extend(mcp_tools)
|
||||
logger.info(f"Registered {len(mcp_tools)} tools from {mcp_name}")
|
||||
|
||||
async def awrap_model_call(
|
||||
self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]
|
||||
) -> ModelResponse:
|
||||
"""根据配置动态选择工具（从已注册的工具中筛选）"""
|
||||
# 从 runtime context 获取配置
|
||||
selected_tools = request.runtime.context.tools
|
||||
selected_mcps = request.runtime.context.mcps
|
||||
|
||||
enabled_tools = []
|
||||
|
||||
# 根据配置筛选基础工具
|
||||
if selected_tools and isinstance(selected_tools, list) and len(selected_tools) > 0:
|
||||
enabled_tools = [tool for tool in self.tools if tool.name in selected_tools]
|
||||
|
||||
# 根据配置筛选 MCP 工具（从已注册的工具中选择）
|
||||
if selected_mcps and isinstance(selected_mcps, list) and len(selected_mcps) > 0:
|
||||
for mcp in selected_mcps:
|
||||
if mcp in self._all_mcp_tools:
|
||||
enabled_tools.extend(self._all_mcp_tools[mcp])
|
||||
else:
|
||||
logger.warning(f"MCP server '{mcp}' not pre-loaded. Please add it to mcp_servers list.")
|
||||
|
||||
logger.info(
|
||||
f"Dynamic tool selection: {len(enabled_tools)} tools enabled: {[tool.name for tool in enabled_tools]}, "
|
||||
f"selected_tools: {selected_tools}, selected_mcps: {selected_mcps}"
|
||||
)
|
||||
|
||||
# 更新 request 中的工具列表
|
||||
request = request.override(tools=enabled_tools)
|
||||
return await handler(request)
|
||||
@@ -0,0 +1,491 @@
|
||||
"""Skills 中间件 - 处理 skills 提示词注入、依赖展开、动态激活"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Annotated, Any, NotRequired, TypedDict
|
||||
|
||||
from deepagents.middleware._utils import append_to_system_message
|
||||
from deepagents.middleware.skills import SKILLS_SYSTEM_PROMPT
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware, ModelRequest, ModelResponse
|
||||
from langchain.tools.tool_node import ToolCallRequest
|
||||
from langgraph.types import Command
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from yuxi.agents.mcp.service import get_enabled_mcp_tools
|
||||
from yuxi.agents.skills.repository import SkillRepository
|
||||
from yuxi.agents.skills.service import is_valid_skill_slug, list_accessible_skills, normalize_string_list
|
||||
from yuxi.agents.toolkits import get_all_tool_instances
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
# =============================================================================
|
||||
# 类型定义
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class SkillPromptMetadata(TypedDict):
|
||||
name: str
|
||||
description: str
|
||||
path: str
|
||||
|
||||
|
||||
class SkillDependencyNode(TypedDict):
|
||||
tools: list[str]
|
||||
mcps: list[str]
|
||||
skills: list[str]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 运行时数据加载函数
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def _list_skills_from_db(db: AsyncSession | None = None, user=None) -> list:
|
||||
"""从数据库加载 skills 列表"""
|
||||
if db is not None:
|
||||
if user is not None:
|
||||
return await list_accessible_skills(db, user)
|
||||
repo = SkillRepository(db)
|
||||
return await repo.list_enabled()
|
||||
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
if user is not None:
|
||||
return await list_accessible_skills(session, user)
|
||||
repo = SkillRepository(session)
|
||||
return await repo.list_enabled()
|
||||
|
||||
|
||||
def build_prompt_metadata(skills: list) -> dict[str, SkillPromptMetadata]:
|
||||
return {
|
||||
item.slug: {
|
||||
"name": item.name,
|
||||
"description": item.description,
|
||||
"path": f"/home/gem/skills/{item.slug}/SKILL.md",
|
||||
}
|
||||
for item in skills
|
||||
if item.slug
|
||||
}
|
||||
|
||||
|
||||
def build_dependency_map(skills: list) -> dict[str, SkillDependencyNode]:
|
||||
result: dict[str, SkillDependencyNode] = {}
|
||||
for item in skills:
|
||||
if not item.slug:
|
||||
continue
|
||||
result[item.slug] = {
|
||||
"tools": normalize_string_list(item.tool_dependencies or []),
|
||||
"mcps": normalize_string_list(item.mcp_dependencies or []),
|
||||
"skills": normalize_string_list(item.skill_dependencies or []),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
async def get_prompt_metadata(db: AsyncSession | None = None, user=None) -> dict[str, SkillPromptMetadata]:
|
||||
"""获取提示词元数据（直接从数据库加载）"""
|
||||
return build_prompt_metadata(await _list_skills_from_db(db, user))
|
||||
|
||||
|
||||
async def get_dependency_map(db: AsyncSession | None = None, user=None) -> dict[str, SkillDependencyNode]:
|
||||
"""获取依赖关系映射（直接从数据库加载）"""
|
||||
return build_dependency_map(await _list_skills_from_db(db, user))
|
||||
|
||||
|
||||
def expand_skill_closure(
|
||||
slugs: list[str] | None,
|
||||
dependency_map: dict[str, SkillDependencyNode],
|
||||
) -> list[str]:
|
||||
"""展开 skills 依赖闭包，返回包含所有依赖的列表"""
|
||||
ordered_roots = normalize_string_list(slugs)
|
||||
if not ordered_roots:
|
||||
return []
|
||||
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def dfs(slug: str, stack: set[str]) -> None:
|
||||
if slug in stack:
|
||||
logger.warning(f"Cycle detected in skill dependencies, skip: {' -> '.join([*stack, slug])}")
|
||||
return
|
||||
if slug in seen:
|
||||
return
|
||||
|
||||
node = dependency_map.get(slug)
|
||||
if not node:
|
||||
logger.warning(f"Skill dependency target not found in DB, skip: {slug}")
|
||||
return
|
||||
|
||||
seen.add(slug)
|
||||
result.append(slug)
|
||||
next_stack = set(stack)
|
||||
next_stack.add(slug)
|
||||
for dep in node.get("skills", []):
|
||||
dfs(dep, next_stack)
|
||||
|
||||
for root in ordered_roots:
|
||||
dfs(root, set())
|
||||
return result
|
||||
|
||||
|
||||
async def resolve_runtime_skills_for_context(context, *, db: AsyncSession | None = None, user=None) -> dict[str, Any]:
|
||||
skill_items = await _list_skills_from_db(db, user)
|
||||
dependency_map = build_dependency_map(skill_items)
|
||||
prompt_metadata = build_prompt_metadata(skill_items)
|
||||
available = set(dependency_map)
|
||||
selected = normalize_string_list(getattr(context, "skills", None))
|
||||
context_skills = [slug for slug in selected if slug in available]
|
||||
prompt_skills = expand_skill_closure(context_skills, dependency_map)
|
||||
return {
|
||||
"context_skills": context_skills,
|
||||
"prompt_skills": prompt_skills,
|
||||
"readable_skills": prompt_skills,
|
||||
"runtime_skill_metadata": prompt_metadata,
|
||||
"runtime_skill_dependency_map": dependency_map,
|
||||
}
|
||||
|
||||
|
||||
def resolve_skill_gated_tools(context) -> list:
|
||||
"""解析当前 Agent 所有可见 Skill 依赖的本地工具实例。
|
||||
|
||||
这些工具默认不会绑定给模型（由 SkillsMiddleware 在对应 Skill 激活后才放出），
|
||||
但必须在构建期注册进 create_agent 的 ToolNode，否则即便模型发起调用，执行器也会
|
||||
判定为 "not a valid tool"。调用方应自行排除已在基础工具集中的工具，避免重复门控。
|
||||
"""
|
||||
dependency_map = getattr(context, "_runtime_skill_dependency_map", {}) or {}
|
||||
readable_skills = getattr(context, "_readable_skills", []) or []
|
||||
tool_names: set[str] = set()
|
||||
for slug in readable_skills:
|
||||
node = dependency_map.get(slug) or {}
|
||||
tool_names.update(node.get("tools", []))
|
||||
if not tool_names:
|
||||
return []
|
||||
return [tool for tool in get_all_tool_instances() if tool.name in tool_names]
|
||||
|
||||
|
||||
def _activated_skills_reducer(left: list[str] | None, right: list[str] | None) -> list[str]:
|
||||
"""合并 activated_skills 列表"""
|
||||
merged: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for group in (left or [], right or []):
|
||||
for value in group:
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
slug = value.strip()
|
||||
if not slug or slug in seen:
|
||||
continue
|
||||
seen.add(slug)
|
||||
merged.append(slug)
|
||||
return merged
|
||||
|
||||
|
||||
class SkillsState(AgentState):
|
||||
"""Skills 状态定义"""
|
||||
|
||||
activated_skills: NotRequired[Annotated[list[str], _activated_skills_reducer]]
|
||||
|
||||
|
||||
class SkillsMiddleware(AgentMiddleware):
|
||||
"""Skills 中间件 - 处理 skills 提示词注入、依赖展开、动态激活
|
||||
|
||||
职责：
|
||||
- Skills 提示词注入（直接从数据库加载）
|
||||
- 依赖展开（用户配置 + 动态激活）
|
||||
- 工具/MCP 动态加载
|
||||
"""
|
||||
|
||||
state_schema = SkillsState
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
skills_context_name: str = "skills",
|
||||
enable_skills_prompt: bool = True,
|
||||
skills_sources_for_prompt: list[str] | None = None,
|
||||
):
|
||||
"""初始化中间件
|
||||
|
||||
Args:
|
||||
skills_context_name: 上下文中的 skills 列表字段名称（默认 "skills"）
|
||||
enable_skills_prompt: 是否启用 skills 提示段注入（默认 True）
|
||||
skills_sources_for_prompt: skills 来源路径（用于提示词展示，默认 ["/home/gem/skills/"]）
|
||||
"""
|
||||
super().__init__()
|
||||
self.skills_context_name = skills_context_name
|
||||
self.enable_skills_prompt = enable_skills_prompt
|
||||
self.skills_sources_for_prompt = skills_sources_for_prompt or ["/home/gem/skills/"]
|
||||
|
||||
async def awrap_model_call(
|
||||
self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]
|
||||
) -> ModelResponse:
|
||||
"""包装模型调用，处理 skills 提示词注入、动态激活和依赖展开"""
|
||||
runtime_context = request.runtime.context
|
||||
|
||||
if self.enable_skills_prompt:
|
||||
prompt_skills = getattr(runtime_context, "_prompt_skills", None)
|
||||
if isinstance(prompt_skills, list):
|
||||
prompt_skills = normalize_string_list(prompt_skills)
|
||||
if prompt_skills:
|
||||
skills_meta = self._collect_prompt_metadata(prompt_skills, runtime_context)
|
||||
skills_section = self._build_skills_section(skills_meta)
|
||||
system_message = append_to_system_message(getattr(request, "system_message", None), skills_section)
|
||||
request = request.override(system_message=system_message)
|
||||
|
||||
state = request.state if isinstance(request.state, dict) else {}
|
||||
activated = state.get("activated_skills", []) or []
|
||||
if not isinstance(activated, list):
|
||||
activated = []
|
||||
|
||||
readable_skills = self._get_readable_skills(runtime_context)
|
||||
activated = [slug for slug in normalize_string_list(activated) if slug in readable_skills]
|
||||
|
||||
deps_bundle = self._build_dependency_bundle(activated, runtime_context)
|
||||
activated_tool_names = set(deps_bundle["tools"])
|
||||
|
||||
# 门控：未激活 Skill 的依赖工具对模型不可见（保持按需加载）。
|
||||
# 这些工具已在构建期由 resolve_configured_runtime_tools 注册进 ToolNode，剔除只影响模型可见性、不影响可执行性。
|
||||
# 排除基础工具集中的工具（如 present_artifacts），它们始终可见、不受 Skill 激活影响。
|
||||
gated_tool_names = self._resolve_gated_tool_names(runtime_context) - activated_tool_names
|
||||
model_tools = list(request.tools or [])
|
||||
if gated_tool_names:
|
||||
model_tools = [t for t in model_tools if t.name not in gated_tool_names]
|
||||
|
||||
# 追加已激活 Skill 的依赖工具：本地工具确保绑定给模型，MCP 工具按需加载
|
||||
enabled_tools = []
|
||||
if activated_tool_names:
|
||||
enabled_tools = [t for t in get_all_tool_instances() if t.name in activated_tool_names]
|
||||
if deps_bundle["mcps"]:
|
||||
enabled_tools.extend(
|
||||
await self._get_mcp_tools_from_context(runtime_context, extra_mcps=deps_bundle["mcps"])
|
||||
)
|
||||
|
||||
existing_tool_names = {t.name for t in model_tools}
|
||||
for t in enabled_tools:
|
||||
if t.name not in existing_tool_names:
|
||||
model_tools.append(t)
|
||||
existing_tool_names.add(t.name)
|
||||
|
||||
if gated_tool_names or enabled_tools:
|
||||
request = request.override(tools=model_tools)
|
||||
|
||||
return await handler(request)
|
||||
|
||||
def _resolve_gated_tool_names(self, runtime_context) -> set[str]:
|
||||
"""所有可见 Skill 依赖、且不属于基础工具集的工具名集合（即「仅经 Skill 激活才放出」的工具）。"""
|
||||
dependency_map = self._get_runtime_dependency_map(runtime_context)
|
||||
readable_skills = self._get_readable_skills(runtime_context)
|
||||
base_tool_names = set(normalize_string_list(getattr(runtime_context, "tools", None)))
|
||||
gated: set[str] = set()
|
||||
for slug in readable_skills:
|
||||
gated.update(dependency_map.get(slug, {}).get("tools", []))
|
||||
return gated - base_tool_names
|
||||
|
||||
def _build_dependency_bundle(self, activated_skills: list[str], runtime_context) -> dict[str, list[str]]:
|
||||
"""根据直接激活的 skills 构建依赖包（不包含闭包展开的依赖）"""
|
||||
dependency_map = self._get_runtime_dependency_map(runtime_context)
|
||||
|
||||
tools: list[str] = []
|
||||
mcps: list[str] = []
|
||||
seen_tools: set[str] = set()
|
||||
seen_mcps: set[str] = set()
|
||||
|
||||
for slug in activated_skills:
|
||||
dep = dependency_map.get(slug, {})
|
||||
for tool_name in dep.get("tools", []):
|
||||
if tool_name in seen_tools:
|
||||
continue
|
||||
seen_tools.add(tool_name)
|
||||
tools.append(tool_name)
|
||||
for mcp_name in dep.get("mcps", []):
|
||||
if mcp_name in seen_mcps:
|
||||
continue
|
||||
seen_mcps.add(mcp_name)
|
||||
mcps.append(mcp_name)
|
||||
|
||||
return {"tools": tools, "mcps": mcps, "skills": activated_skills}
|
||||
|
||||
def _collect_prompt_metadata(self, slugs: list[str], runtime_context) -> list[SkillPromptMetadata]:
|
||||
"""收集指定 slugs 的提示词元数据"""
|
||||
prompt_metadata = self._get_runtime_prompt_metadata(runtime_context)
|
||||
|
||||
result: list[SkillPromptMetadata] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for slug in slugs:
|
||||
if not isinstance(slug, str):
|
||||
continue
|
||||
normalized = slug.strip()
|
||||
if not normalized or normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
|
||||
item = prompt_metadata.get(normalized)
|
||||
if not item:
|
||||
logger.debug(f"Skill slug not found in prompt metadata, skip: {normalized}")
|
||||
continue
|
||||
result.append(dict(item))
|
||||
|
||||
return result
|
||||
|
||||
async def _get_mcp_tools_from_context(
|
||||
self,
|
||||
context,
|
||||
*,
|
||||
extra_mcps: list[str] | None = None,
|
||||
) -> list:
|
||||
"""从上下文配置中获取 MCP 工具列表"""
|
||||
import asyncio
|
||||
|
||||
# MCP 工具（并行加载）
|
||||
mcps = getattr(context, "mcps", None) or []
|
||||
all_mcp_names: list[str] = []
|
||||
for server_name in mcps:
|
||||
if isinstance(server_name, str):
|
||||
all_mcp_names.append(server_name)
|
||||
for server_name in extra_mcps or []:
|
||||
if isinstance(server_name, str):
|
||||
all_mcp_names.append(server_name)
|
||||
|
||||
# 去重
|
||||
unique_mcp_names = list(dict.fromkeys(all_mcp_names))
|
||||
|
||||
async def load_mcp_tools(server_name: str) -> list:
|
||||
"""加载单个 MCP 服务器的工具"""
|
||||
try:
|
||||
mcp_tools = await get_enabled_mcp_tools(server_name)
|
||||
if not mcp_tools:
|
||||
logger.warning(f"SkillsMiddleware: mcp dependency unavailable, skip: {server_name}")
|
||||
return mcp_tools
|
||||
except Exception as e:
|
||||
logger.warning(f"SkillsMiddleware: failed to load mcp dependency '{server_name}': {e}")
|
||||
return []
|
||||
|
||||
# 并行加载所有 MCP 工具
|
||||
results = await asyncio.gather(*[load_mcp_tools(name) for name in unique_mcp_names])
|
||||
selected_tools = []
|
||||
for tools in results:
|
||||
selected_tools.extend(tools)
|
||||
|
||||
return selected_tools
|
||||
|
||||
def _process_tool_call_result(self, result: Any, request: ToolCallRequest) -> Any:
|
||||
"""处理工具调用结果，检查并处理 skill 动态激活"""
|
||||
if request.tool_call.get("name") != "read_file":
|
||||
return result
|
||||
|
||||
args = request.tool_call.get("args") or {}
|
||||
file_path = args.get("file_path") if isinstance(args, dict) else None
|
||||
slug = self._extract_skill_slug_from_skill_md_path(file_path)
|
||||
|
||||
if not slug:
|
||||
return result
|
||||
|
||||
if not self._is_visible_skill_slug(request, slug):
|
||||
logger.warning(f"SkillsMiddleware: deny skill activation for invisible slug: {slug}")
|
||||
return result
|
||||
|
||||
logger.debug(f"SkillsMiddleware: activated skill by read_file: {slug}")
|
||||
return self._merge_activated_skill_update(result, slug)
|
||||
|
||||
async def awrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], Any],
|
||||
):
|
||||
"""包装工具调用，处理 skill 动态激活"""
|
||||
result = await handler(request)
|
||||
return self._process_tool_call_result(result, request)
|
||||
|
||||
def wrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], Any],
|
||||
):
|
||||
"""同步版本的工具调用包装"""
|
||||
result = handler(request)
|
||||
return self._process_tool_call_result(result, request)
|
||||
|
||||
def _extract_skill_slug_from_skill_md_path(self, file_path: Any) -> str | None:
|
||||
"""从文件路径中提取 skill slug"""
|
||||
if not isinstance(file_path, str):
|
||||
return None
|
||||
raw = file_path.strip()
|
||||
if not raw:
|
||||
return None
|
||||
pure = PurePosixPath(raw if raw.startswith("/") else f"/{raw}")
|
||||
parts = [p for p in pure.parts if p not in ("/", "")]
|
||||
slug: str | None = None
|
||||
if (
|
||||
len(parts) == 5
|
||||
and parts[0] == "home"
|
||||
and parts[1] == "gem"
|
||||
and parts[2] == "skills"
|
||||
and parts[4] == "SKILL.md"
|
||||
):
|
||||
slug = parts[3]
|
||||
|
||||
if not is_valid_skill_slug(slug):
|
||||
return None
|
||||
return slug
|
||||
|
||||
def _get_readable_skills(self, runtime_context) -> set[str]:
|
||||
selected = getattr(runtime_context, "_readable_skills", [])
|
||||
return set(normalize_string_list(selected if isinstance(selected, list) else []))
|
||||
|
||||
def _get_runtime_prompt_metadata(self, runtime_context) -> dict[str, SkillPromptMetadata]:
|
||||
metadata = getattr(runtime_context, "_runtime_skill_metadata", {})
|
||||
return metadata if isinstance(metadata, dict) else {}
|
||||
|
||||
def _get_runtime_dependency_map(self, runtime_context) -> dict[str, SkillDependencyNode]:
|
||||
dependency_map = getattr(runtime_context, "_runtime_skill_dependency_map", {})
|
||||
return dependency_map if isinstance(dependency_map, dict) else {}
|
||||
|
||||
def _is_visible_skill_slug(self, request: ToolCallRequest, slug: str) -> bool:
|
||||
"""检查 slug 是否可见"""
|
||||
return slug in self._get_readable_skills(request.runtime.context)
|
||||
|
||||
def _merge_activated_skill_update(self, result: Any, slug: str):
|
||||
"""合并动态激活的 skill 更新"""
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
if isinstance(result, Command):
|
||||
update = dict(result.update or {})
|
||||
current = update.get("activated_skills") or []
|
||||
update["activated_skills"] = _activated_skills_reducer(current, [slug])
|
||||
return Command(graph=result.graph, update=update, resume=result.resume, goto=result.goto)
|
||||
|
||||
if isinstance(result, ToolMessage):
|
||||
return Command(update={"messages": [result], "activated_skills": [slug]})
|
||||
|
||||
return result
|
||||
|
||||
def _format_skills_locations(self, sources: list[str]) -> str:
|
||||
"""格式化 skills 位置信息"""
|
||||
locations = []
|
||||
for i, source_path in enumerate(sources):
|
||||
name = PurePosixPath(source_path.rstrip("/")).name.capitalize()
|
||||
suffix = " (higher priority)" if i == len(sources) - 1 else ""
|
||||
locations.append(f"**{name} Skills**: `{source_path}`{suffix}")
|
||||
return "\n".join(locations)
|
||||
|
||||
def _format_skills_list(self, skills_meta: list[dict[str, str]]) -> str:
|
||||
"""格式化 skills 列表"""
|
||||
if not skills_meta:
|
||||
return f"(No skills available yet. You can create skills in {' or '.join(self.skills_sources_for_prompt)})"
|
||||
|
||||
lines = []
|
||||
for skill in skills_meta:
|
||||
lines.append(f"- **{skill['name']}**: {skill['description']}")
|
||||
lines.append(f" -> Read `{skill['path']}` for full instructions")
|
||||
return "\n".join(lines)
|
||||
|
||||
def _build_skills_section(self, skills_meta: list[dict[str, str]]) -> str:
|
||||
"""构建 skills 提示段"""
|
||||
skills_locations = self._format_skills_locations(self.skills_sources_for_prompt)
|
||||
skills_list = self._format_skills_list(skills_meta)
|
||||
return SKILLS_SYSTEM_PROMPT.format(
|
||||
skills_locations=skills_locations,
|
||||
skills_load_warnings="",
|
||||
skills_list=skills_list,
|
||||
)
|
||||
@@ -0,0 +1,586 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated, Any
|
||||
|
||||
from deepagents import SubagentTransformer as DeepAgentsSubagentTransformer
|
||||
from deepagents.middleware._utils import append_to_system_message
|
||||
from langchain.agents.middleware.types import AgentMiddleware, ContextT, ModelRequest, ModelResponse, ResponseT
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.tools import StructuredTool
|
||||
from langgraph.prebuilt.tool_node import ToolRuntime
|
||||
from langgraph.types import Command
|
||||
|
||||
from yuxi.repositories.agent_repository import AgentRepository
|
||||
from yuxi.repositories.agent_run_repository import TERMINAL_RUN_STATUSES
|
||||
from yuxi.repositories.user_repository import UserRepository
|
||||
from yuxi.services.input_message_service import build_chat_input_message
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
from yuxi.storage.postgres.models_business import Agent
|
||||
|
||||
YUXI_SUBAGENTS_STREAM_KEY = "yuxi_subagents"
|
||||
|
||||
|
||||
def _subagent_run_service_module():
|
||||
from yuxi.services import subagent_run_service
|
||||
|
||||
return subagent_run_service
|
||||
|
||||
|
||||
def _async_only_tool(*, name: str, coroutine: Callable[..., Awaitable[Any]], description: str) -> StructuredTool:
|
||||
"""后台子智能体工具只在异步链路执行；仅声明 coroutine，同步调用由 LangChain 直接报错。"""
|
||||
return StructuredTool.from_function(name=name, coroutine=coroutine, description=description, infer_schema=True)
|
||||
|
||||
|
||||
class YuxiSubagentTransformer(DeepAgentsSubagentTransformer):
|
||||
def init(self) -> dict[str, Any]:
|
||||
return {YUXI_SUBAGENTS_STREAM_KEY: self._log}
|
||||
|
||||
|
||||
TASK_SYSTEM_PROMPT = """## `task`（子智能体任务工具）
|
||||
|
||||
你可以使用 `task` 工具把复杂、独立的子任务交给已配置的子智能体处理。子智能体只返回最终结果，你看不到它的中间步骤。
|
||||
工具结果会包含子智能体线程 ID，后续需要继续同一个子任务时，把该 ID 作为 `thread_id` 传回 `task`。
|
||||
|
||||
使用原则：
|
||||
- 任务足够复杂、可以独立完成、或需要隔离上下文时使用。
|
||||
- 多个互不依赖的子任务可以并行调用多个 `task`。
|
||||
- 继续既有子智能体任务时传入之前结果中的 `thread_id`；新任务不要填写 `thread_id`。
|
||||
- 不要并行调用同一个 `thread_id`，避免多个续跑请求同时写入同一子线程。
|
||||
- 简单问题或少量直接工具调用不要委派。
|
||||
- 调用时必须选择下方可用的 `subagent_slug`，并在 `description` 中写清目标、上下文和期望输出。
|
||||
- 不要通过 shell、curl、HTTP API 或命令行间接调用子智能体；需要子智能体时必须使用 `task` 工具。
|
||||
|
||||
后台子智能体：
|
||||
- 长任务或多个可并行任务优先使用 `subagent_start`，它会立即返回 `run_id` 和 `thread_id`，父智能体可以继续工作。
|
||||
- 后续用 `subagent_status` 查询状态，`subagent_events` 读取增量事件，
|
||||
`subagent_cancel` 取消，`subagent_await` 在明确需要结果时等待。
|
||||
- `thread_id` 是子智能体长期上下文 ID；同一个 `thread_id` 完成后可以继续创建新的 run。
|
||||
若同线程已有运行中 run，会返回 busy，不会隐藏排队。
|
||||
- 短任务且父智能体必须立刻依赖结果时继续使用 `task`。
|
||||
|
||||
Available subagent slugs:
|
||||
|
||||
{available_agents}"""
|
||||
|
||||
TASK_TOOL_DESCRIPTION = """Launch a configured Yuxi subagent to handle an isolated task.
|
||||
|
||||
Available subagent slugs:
|
||||
{available_agents}
|
||||
|
||||
Use `subagent_slug` to select one available subagent and put the full task brief in `description`.
|
||||
Omit `thread_id` for a new task. To continue a previous subagent task, pass the child thread ID returned by
|
||||
that prior task result as `thread_id`.
|
||||
Do not call subagents through shell, curl, HTTP APIs, or command-line indirection."""
|
||||
|
||||
SUBAGENT_START_DESCRIPTION = """Start a configured Yuxi subagent asynchronously.
|
||||
|
||||
Returns a child thread ID for future continuation and a run ID for status/events/cancel/result checks.
|
||||
Use this for long-running or parallelizable subagent work. If `thread_id` is provided, it continues that subagent
|
||||
thread when no active run is currently writing to it."""
|
||||
|
||||
SUBAGENT_STATUS_DESCRIPTION = """Check a subagent run status by run_id.
|
||||
|
||||
Returns the current run status, a compact progress summary with the latest 3 readable messages, and the final result
|
||||
when the run has reached a terminal status."""
|
||||
|
||||
SUBAGENT_EVENTS_DESCRIPTION = """Read recent events for a subagent run by run_id and Redis stream cursor."""
|
||||
|
||||
SUBAGENT_CANCEL_DESCRIPTION = """Cancel a running subagent run by run_id."""
|
||||
|
||||
SUBAGENT_AWAIT_DESCRIPTION = """Wait for a subagent run to finish and return its final result."""
|
||||
|
||||
TASK_DESCRIPTION_ARG = "需要子智能体独立完成的任务描述，包含必要上下文和期望输出。"
|
||||
SUBAGENT_SLUG_ARG = "要调用的子智能体 slug，必须是工具描述中列出的可用项之一。"
|
||||
TASK_THREAD_ID_ARG = "可选。要继续的既有子智能体线程 ID，通常来自之前 task 工具结果；新任务不要填写。"
|
||||
ASYNC_THREAD_ID_ARG = "可选。要继续的后台子智能体线程 ID，来自之前 subagent_start 返回的 thread_id；新任务不要填写。"
|
||||
SUBAGENT_RUN_ID_ARG = "子智能体运行 ID，由 subagent_start 返回。"
|
||||
SUBAGENT_AFTER_SEQ_ARG = "可选。事件流游标，首次读取传 0-0；后续传上次返回的 last_seq。"
|
||||
SUBAGENT_EVENT_LIMIT_ARG = "可选。读取事件数量，范围 1-50。"
|
||||
|
||||
|
||||
async def create_subagent_task_middleware(parent_context) -> YuxiSubAgentMiddleware | None:
|
||||
"""根据父智能体上下文加载可用子智能体，并在存在可调用项时创建 task 中间件。"""
|
||||
selected_slugs = [
|
||||
str(slug).strip() for slug in (getattr(parent_context, "subagents", None) or []) if str(slug).strip()
|
||||
]
|
||||
uid = str(getattr(parent_context, "uid", "") or "").strip()
|
||||
if not uid:
|
||||
return None
|
||||
|
||||
async with pg_manager.get_async_session_context() as db:
|
||||
user = await UserRepository().get_by_uid_with_db(db, uid)
|
||||
if user is None:
|
||||
return None
|
||||
repo = AgentRepository(db)
|
||||
if selected_slugs:
|
||||
subagents: list[Agent] = []
|
||||
seen: set[str] = set()
|
||||
for slug in selected_slugs:
|
||||
if slug in seen:
|
||||
continue
|
||||
seen.add(slug)
|
||||
agent = await repo.get_visible_by_slug(slug=slug, user=user, kind="subagent")
|
||||
if agent:
|
||||
subagents.append(agent)
|
||||
else:
|
||||
subagents = await repo.list_visible_subagents(user=user)
|
||||
|
||||
if not subagents:
|
||||
return None
|
||||
return YuxiSubAgentMiddleware(parent_context=parent_context, subagents=subagents)
|
||||
|
||||
|
||||
class YuxiSubAgentMiddleware(AgentMiddleware[Any, ContextT, ResponseT]):
|
||||
def __init__(self, *, parent_context, subagents: list[Agent]) -> None:
|
||||
super().__init__()
|
||||
self.parent_context = parent_context
|
||||
self.subagents = {agent.slug: agent for agent in subagents}
|
||||
available_agents = "\n".join(f"- {agent.slug}: {agent.description or agent.name}" for agent in subagents)
|
||||
self.system_prompt = TASK_SYSTEM_PROMPT.format(available_agents=available_agents)
|
||||
self.tools = [self._build_task_tool(available_agents), *self._build_async_subagent_tools(available_agents)]
|
||||
self.subagent_names = frozenset(self.subagents)
|
||||
self.transformers = [lambda scope: YuxiSubagentTransformer(scope, subagent_names=self.subagent_names)]
|
||||
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest[ContextT],
|
||||
handler: Callable[[ModelRequest[ContextT]], ModelResponse[ResponseT]],
|
||||
) -> ModelResponse[ResponseT]:
|
||||
return handler(
|
||||
request.override(system_message=append_to_system_message(request.system_message, self.system_prompt))
|
||||
)
|
||||
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest[ContextT],
|
||||
handler: Callable[[ModelRequest[ContextT]], Awaitable[ModelResponse[ResponseT]]],
|
||||
) -> ModelResponse[ResponseT]:
|
||||
return await handler(
|
||||
request.override(system_message=append_to_system_message(request.system_message, self.system_prompt))
|
||||
)
|
||||
|
||||
def _build_task_tool(self, available_agents: str) -> StructuredTool:
|
||||
"""构建 task 工具：启动子智能体后阻塞等待其最终结果。"""
|
||||
|
||||
async def atask(
|
||||
description: Annotated[str, TASK_DESCRIPTION_ARG],
|
||||
subagent_slug: Annotated[str, SUBAGENT_SLUG_ARG],
|
||||
runtime: ToolRuntime,
|
||||
thread_id: Annotated[str | None, TASK_THREAD_ID_ARG] = None,
|
||||
) -> str | Command:
|
||||
started, error = await self._start_subagent(
|
||||
description=description,
|
||||
subagent_slug=subagent_slug,
|
||||
runtime=runtime,
|
||||
thread_id=thread_id,
|
||||
error_prefix="无法调用子智能体",
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
|
||||
# 阻塞父智能体运行，直到子 run 终结再读取最终结果；运行失败时 result 含 error 信息
|
||||
parent_runtime = started.parent_runtime
|
||||
subagent_service = _subagent_run_service_module()
|
||||
try:
|
||||
from yuxi.services.agent_run_service import AgentRunWaitTimeout, await_agent_run_result
|
||||
|
||||
result = await await_agent_run_result(run_id=started.result.run.id, current_uid=parent_runtime.uid)
|
||||
run = await self._get_verified_subagent_run(
|
||||
run_id=started.result.run.id,
|
||||
uid=parent_runtime.uid,
|
||||
created_by_run_id=parent_runtime.created_by_run_id,
|
||||
)
|
||||
except AgentRunWaitTimeout as exc:
|
||||
try:
|
||||
run = await self._get_verified_subagent_run(
|
||||
run_id=started.result.run.id,
|
||||
uid=parent_runtime.uid,
|
||||
created_by_run_id=parent_runtime.created_by_run_id,
|
||||
)
|
||||
except ValueError as verify_exc:
|
||||
return str(verify_exc)
|
||||
subagent_run = subagent_service.serialize_subagent_run_state(run)
|
||||
return _task_wait_timeout_response(exc.result, runtime.tool_call_id, subagent_run)
|
||||
except ValueError as exc:
|
||||
return str(exc)
|
||||
|
||||
subagent_run = subagent_service.serialize_subagent_run_state(run)
|
||||
return _task_result_response(result, runtime.tool_call_id, subagent_run)
|
||||
|
||||
return _async_only_tool(
|
||||
name="task",
|
||||
coroutine=atask,
|
||||
description=TASK_TOOL_DESCRIPTION.format(available_agents=available_agents),
|
||||
)
|
||||
|
||||
def _build_async_subagent_tools(self, available_agents: str) -> list[StructuredTool]:
|
||||
"""构建后台子智能体生命周期工具：start/status/events/cancel/await。"""
|
||||
|
||||
async def asubagent_start(
|
||||
description: Annotated[str, TASK_DESCRIPTION_ARG],
|
||||
subagent_slug: Annotated[str, SUBAGENT_SLUG_ARG],
|
||||
runtime: ToolRuntime,
|
||||
thread_id: Annotated[str | None, ASYNC_THREAD_ID_ARG] = None,
|
||||
) -> str | Command:
|
||||
started, error = await self._start_subagent(
|
||||
description=description,
|
||||
subagent_slug=subagent_slug,
|
||||
runtime=runtime,
|
||||
thread_id=thread_id,
|
||||
error_prefix="无法启动子智能体",
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
|
||||
result, agent_item = started.result, started.agent_item
|
||||
subagent_service = _subagent_run_service_module()
|
||||
payload = {
|
||||
"status": "started" if result.created else "existing",
|
||||
"run_id": result.run.id,
|
||||
"thread_id": result.relation.child_thread_id,
|
||||
"subagent_slug": subagent_slug,
|
||||
"subagent_name": agent_item.name,
|
||||
"created_by_run_id": result.run.created_by_run_id,
|
||||
"run_status": result.run.status,
|
||||
"continuing": result.continuing,
|
||||
"subagent_thread_relation_id": result.relation.id,
|
||||
**subagent_service.subagent_run_urls(result.run.id),
|
||||
}
|
||||
subagent_run = subagent_service.serialize_subagent_run_state(result.run)
|
||||
return _json_tool_command(payload, runtime.tool_call_id, subagent_run=subagent_run)
|
||||
|
||||
async def asubagent_status(
|
||||
run_id: Annotated[str, SUBAGENT_RUN_ID_ARG],
|
||||
runtime: ToolRuntime,
|
||||
) -> str | Command:
|
||||
from yuxi.services.agent_run_service import get_agent_run_progress, get_agent_run_result
|
||||
|
||||
parent_runtime, runtime_error = self._require_async_parent_runtime("无法查询子智能体")
|
||||
if runtime_error:
|
||||
return runtime_error
|
||||
try:
|
||||
run = await self._get_verified_subagent_run(
|
||||
uid=parent_runtime.uid,
|
||||
created_by_run_id=parent_runtime.created_by_run_id,
|
||||
run_id=run_id,
|
||||
)
|
||||
|
||||
# 如果 run 已经终结，则尝试读取最终结果；否则 result 保持 None
|
||||
result = None
|
||||
if run.status in TERMINAL_RUN_STATUSES:
|
||||
async with pg_manager.get_async_session_context() as db:
|
||||
result = await get_agent_run_result(run_id=run.id, current_uid=parent_runtime.uid, db=db)
|
||||
|
||||
except ValueError as exc:
|
||||
return str(exc)
|
||||
|
||||
subagent_service = _subagent_run_service_module()
|
||||
payload = {
|
||||
"status": run.status,
|
||||
"run_id": run.id,
|
||||
"thread_id": run.conversation_thread_id,
|
||||
"subagent_slug": run.agent_slug,
|
||||
"error": run.error_message,
|
||||
"progress": await get_agent_run_progress(run.id),
|
||||
**subagent_service.subagent_run_urls(run.id),
|
||||
}
|
||||
if result:
|
||||
payload["result"] = result
|
||||
subagent_run = subagent_service.serialize_subagent_run_state(run)
|
||||
return _json_tool_command(payload, runtime.tool_call_id, subagent_run=subagent_run)
|
||||
|
||||
async def asubagent_events(
|
||||
run_id: Annotated[str, SUBAGENT_RUN_ID_ARG],
|
||||
runtime: ToolRuntime,
|
||||
after_seq: Annotated[str, SUBAGENT_AFTER_SEQ_ARG] = "0-0",
|
||||
limit: Annotated[int, SUBAGENT_EVENT_LIMIT_ARG] = 20,
|
||||
) -> str | Command:
|
||||
from yuxi.services.run_queue_service import list_run_stream_events, normalize_after_seq
|
||||
|
||||
parent_runtime, runtime_error = self._require_async_parent_runtime("无法读取子智能体事件")
|
||||
if runtime_error:
|
||||
return runtime_error
|
||||
try:
|
||||
await self._get_verified_subagent_run(
|
||||
run_id=run_id,
|
||||
uid=parent_runtime.uid,
|
||||
created_by_run_id=parent_runtime.created_by_run_id,
|
||||
) # 校验子智能体归属
|
||||
except ValueError as exc:
|
||||
return str(exc)
|
||||
|
||||
normalized_after_seq = normalize_after_seq(after_seq)
|
||||
event_limit = min(50, max(1, int(limit or 20)))
|
||||
events = await list_run_stream_events(run_id, after_seq=normalized_after_seq, limit=event_limit)
|
||||
payload = {
|
||||
"status": "ok",
|
||||
"run_id": run_id,
|
||||
"after_seq": normalized_after_seq,
|
||||
"last_seq": str(events[-1]["seq"]) if events else normalized_after_seq,
|
||||
"events": events,
|
||||
}
|
||||
return _json_tool_command(payload, runtime.tool_call_id)
|
||||
|
||||
async def asubagent_cancel(
|
||||
run_id: Annotated[str, SUBAGENT_RUN_ID_ARG],
|
||||
runtime: ToolRuntime,
|
||||
) -> str | Command:
|
||||
from yuxi.services.agent_run_service import request_cancel_agent_run
|
||||
|
||||
parent_runtime, runtime_error = self._require_async_parent_runtime("无法取消子智能体")
|
||||
if runtime_error:
|
||||
return runtime_error
|
||||
try:
|
||||
await self._get_verified_subagent_run(
|
||||
run_id=run_id,
|
||||
uid=parent_runtime.uid,
|
||||
created_by_run_id=parent_runtime.created_by_run_id,
|
||||
) # 校验子智能体归属
|
||||
|
||||
# 取消子智能体运行，返回最新 run 状态
|
||||
async with pg_manager.get_async_session_context() as db:
|
||||
run = await request_cancel_agent_run(run_id=run_id, current_uid=parent_runtime.uid, db=db)
|
||||
|
||||
except ValueError as exc:
|
||||
return str(exc)
|
||||
|
||||
subagent_service = _subagent_run_service_module()
|
||||
payload = {
|
||||
"status": run.status,
|
||||
"run_id": run.id,
|
||||
"thread_id": run.conversation_thread_id,
|
||||
**subagent_service.subagent_run_urls(run.id),
|
||||
}
|
||||
subagent_run = subagent_service.serialize_subagent_run_state(run)
|
||||
return _json_tool_command(payload, runtime.tool_call_id, subagent_run=subagent_run)
|
||||
|
||||
async def asubagent_await(
|
||||
run_id: Annotated[str, SUBAGENT_RUN_ID_ARG],
|
||||
runtime: ToolRuntime,
|
||||
) -> str | Command:
|
||||
from yuxi.services.agent_run_service import AgentRunWaitTimeout, await_agent_run_result
|
||||
|
||||
parent_runtime, runtime_error = self._require_async_parent_runtime("无法等待子智能体")
|
||||
if runtime_error:
|
||||
return runtime_error
|
||||
wait_timed_out = False
|
||||
try:
|
||||
# 等待前校验 run 归属，避免越权等待其它子任务
|
||||
await self._get_verified_subagent_run(
|
||||
run_id=run_id,
|
||||
uid=parent_runtime.uid,
|
||||
created_by_run_id=parent_runtime.created_by_run_id,
|
||||
)
|
||||
# 等待结束后重新读取已验证的最新 run 状态
|
||||
result = await await_agent_run_result(run_id=run_id, current_uid=parent_runtime.uid)
|
||||
run = await self._get_verified_subagent_run(
|
||||
run_id=run_id,
|
||||
uid=parent_runtime.uid,
|
||||
created_by_run_id=parent_runtime.created_by_run_id,
|
||||
)
|
||||
except AgentRunWaitTimeout as exc:
|
||||
wait_timed_out = True
|
||||
result = exc.result
|
||||
try:
|
||||
run = await self._get_verified_subagent_run(
|
||||
run_id=run_id,
|
||||
uid=parent_runtime.uid,
|
||||
created_by_run_id=parent_runtime.created_by_run_id,
|
||||
)
|
||||
except ValueError as verify_exc:
|
||||
return str(verify_exc)
|
||||
except ValueError as exc:
|
||||
return str(exc)
|
||||
|
||||
subagent_service = _subagent_run_service_module()
|
||||
payload = {
|
||||
"status": run.status,
|
||||
"run_id": run.id,
|
||||
"thread_id": run.conversation_thread_id,
|
||||
"result": result,
|
||||
}
|
||||
if wait_timed_out:
|
||||
payload["wait_timed_out"] = True
|
||||
payload["message"] = "子智能体仍在运行，等待最终结果超时；请稍后继续查询。"
|
||||
subagent_run = subagent_service.serialize_subagent_run_state(run)
|
||||
return _json_tool_command(payload, runtime.tool_call_id, subagent_run=subagent_run)
|
||||
|
||||
return [
|
||||
_async_only_tool(
|
||||
name="subagent_start",
|
||||
coroutine=asubagent_start,
|
||||
description=SUBAGENT_START_DESCRIPTION + "\n\nAvailable subagent slugs:\n" + available_agents,
|
||||
),
|
||||
_async_only_tool(
|
||||
name="subagent_status",
|
||||
coroutine=asubagent_status,
|
||||
description=SUBAGENT_STATUS_DESCRIPTION,
|
||||
),
|
||||
_async_only_tool(
|
||||
name="subagent_events",
|
||||
coroutine=asubagent_events,
|
||||
description=SUBAGENT_EVENTS_DESCRIPTION,
|
||||
),
|
||||
_async_only_tool(
|
||||
name="subagent_cancel",
|
||||
coroutine=asubagent_cancel,
|
||||
description=SUBAGENT_CANCEL_DESCRIPTION,
|
||||
),
|
||||
_async_only_tool(
|
||||
name="subagent_await",
|
||||
coroutine=asubagent_await,
|
||||
description=SUBAGENT_AWAIT_DESCRIPTION,
|
||||
),
|
||||
]
|
||||
|
||||
def _parent_runtime(self) -> _ParentRuntime:
|
||||
"""从父智能体 context 中抽取子智能体运行所需的最小父运行信息。"""
|
||||
parent_thread_id = str(getattr(self.parent_context, "parent_thread_id", None) or self.parent_context.thread_id)
|
||||
file_thread_id = str(getattr(self.parent_context, "file_thread_id", None) or parent_thread_id)
|
||||
uid = str(getattr(self.parent_context, "uid", "") or "").strip()
|
||||
created_by_run_id = str(getattr(self.parent_context, "run_id", "") or "").strip()
|
||||
return _ParentRuntime(
|
||||
file_thread_id=file_thread_id,
|
||||
uid=uid,
|
||||
created_by_run_id=created_by_run_id,
|
||||
)
|
||||
|
||||
def _require_async_parent_runtime(self, error_prefix: str) -> tuple[_ParentRuntime, str | None]:
|
||||
"""校验后台子智能体工具必须依赖的父运行上下文。"""
|
||||
parent_runtime = self._parent_runtime()
|
||||
if not parent_runtime.uid:
|
||||
return parent_runtime, f"{error_prefix}：当前运行时缺少 uid"
|
||||
if not parent_runtime.created_by_run_id:
|
||||
return parent_runtime, f"{error_prefix}：当前运行时缺少父运行 ID"
|
||||
return parent_runtime, None
|
||||
|
||||
async def _start_subagent(
|
||||
self,
|
||||
*,
|
||||
description: str,
|
||||
subagent_slug: str,
|
||||
runtime: ToolRuntime,
|
||||
thread_id: str | None,
|
||||
error_prefix: str,
|
||||
) -> tuple[_StartedSubagent | None, str | Command | None]:
|
||||
"""校验并启动（或继续）后台子智能体 run；成功返回启动结果，失败返回可直接回传的错误响应。"""
|
||||
if subagent_slug not in self.subagents:
|
||||
allowed = ", ".join(f"`{slug}`" for slug in self.subagents)
|
||||
return None, f"无法调用子智能体 {subagent_slug}，可用子智能体只有：{allowed}"
|
||||
if not runtime.tool_call_id:
|
||||
raise ValueError("Tool call ID is required for subagent invocation")
|
||||
|
||||
parent_runtime, runtime_error = self._require_async_parent_runtime(error_prefix)
|
||||
if runtime_error:
|
||||
return None, runtime_error
|
||||
|
||||
agent_item = self.subagents[subagent_slug]
|
||||
input_message = build_chat_input_message(description)
|
||||
subagent_service = _subagent_run_service_module()
|
||||
try:
|
||||
async with pg_manager.get_async_session_context() as db:
|
||||
result = await subagent_service.SubagentRunService(db).start(
|
||||
uid=parent_runtime.uid,
|
||||
created_by_run_id=parent_runtime.created_by_run_id,
|
||||
agent_item=agent_item,
|
||||
input_message=input_message,
|
||||
tool_call_id=runtime.tool_call_id,
|
||||
requested_thread_id=thread_id,
|
||||
file_thread_id=parent_runtime.file_thread_id,
|
||||
model_spec=self._subagent_model_override(agent_item),
|
||||
)
|
||||
except subagent_service.SubagentRunBusy as exc:
|
||||
return None, _json_tool_command(exc.to_payload(), runtime.tool_call_id)
|
||||
except ValueError as exc:
|
||||
return None, str(exc)
|
||||
return _StartedSubagent(result=result, parent_runtime=parent_runtime, agent_item=agent_item), None
|
||||
|
||||
def _subagent_model_override(self, agent_item: Agent) -> str | None:
|
||||
"""当子智能体未显式配置模型时，沿用父智能体当前模型。"""
|
||||
config_context = (
|
||||
(agent_item.config_json or {}).get("context") if isinstance(agent_item.config_json, dict) else None
|
||||
)
|
||||
configured_model = ""
|
||||
if isinstance(config_context, dict):
|
||||
configured_model = str(config_context.get("model") or "").strip()
|
||||
if configured_model:
|
||||
return None
|
||||
return str(getattr(self.parent_context, "model", None) or "").strip() or None
|
||||
|
||||
async def _get_verified_subagent_run(self, *, run_id: str, uid: str, created_by_run_id: str):
|
||||
"""在工具调用前按父 run 作用域校验子 run 归属。"""
|
||||
subagent_service = _subagent_run_service_module()
|
||||
async with pg_manager.get_async_session_context() as db:
|
||||
return await subagent_service.SubagentRunService(db).get_run_for_creator(
|
||||
uid=uid,
|
||||
created_by_run_id=created_by_run_id,
|
||||
run_id=run_id,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ParentRuntime:
|
||||
file_thread_id: str
|
||||
uid: str
|
||||
created_by_run_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _StartedSubagent:
|
||||
"""``_start_subagent`` 的结果：子 run 启动产物及其依赖的父运行上下文。"""
|
||||
|
||||
result: Any # SubagentStartResult
|
||||
parent_runtime: _ParentRuntime
|
||||
agent_item: Agent
|
||||
|
||||
|
||||
def _task_result_response(result: dict[str, Any], tool_call_id: str, subagent_run: dict[str, Any]) -> Command:
|
||||
"""把后台子智能体 run 的最终结果转换为同步 task 工具结果。"""
|
||||
output = str(result.get("output") or "").strip()
|
||||
error = result.get("error") if isinstance(result.get("error"), dict) else None
|
||||
if not output and error:
|
||||
output = str(error.get("message") or "子智能体运行失败")
|
||||
if not output:
|
||||
output = "子智能体已完成任务，但没有返回文本结果。"
|
||||
|
||||
tool_result = _tool_result_with_thread_id(subagent_run["child_thread_id"], output)
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(tool_result, tool_call_id=tool_call_id)], "subagent_runs": [subagent_run]}
|
||||
)
|
||||
|
||||
|
||||
def _task_wait_timeout_response(result: dict[str, Any], tool_call_id: str, subagent_run: dict[str, Any]) -> Command:
|
||||
"""同步 task 等待到达上限时，明确告诉父智能体子 run 仍未终结。"""
|
||||
status = str(result.get("status") or subagent_run.get("status") or "running")
|
||||
run_id = str(result.get("agent_run_id") or subagent_run["run_id"])
|
||||
output = (
|
||||
f"子智能体仍在运行（status: {status}），尚未返回最终文本结果。\n"
|
||||
f"run_id: {run_id}\n"
|
||||
"请稍后使用 subagent_status 或 subagent_await 查询结果；不要把当前结果视为任务已完成。"
|
||||
)
|
||||
tool_result = _tool_result_with_thread_id(subagent_run["child_thread_id"], output)
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(tool_result, tool_call_id=tool_call_id)], "subagent_runs": [subagent_run]}
|
||||
)
|
||||
|
||||
|
||||
def _json_tool_command(
|
||||
payload: dict[str, Any],
|
||||
tool_call_id: str,
|
||||
*,
|
||||
subagent_run: dict[str, Any] | None = None,
|
||||
) -> Command:
|
||||
"""把后台子智能体工具的结构化结果包装成 ToolMessage。"""
|
||||
content = json.dumps(payload, ensure_ascii=False, indent=2)
|
||||
update: dict[str, Any] = {"messages": [ToolMessage(content, tool_call_id=tool_call_id)]}
|
||||
if subagent_run is not None:
|
||||
update["subagent_runs"] = [subagent_run]
|
||||
return Command(update=update)
|
||||
|
||||
|
||||
def _tool_result_with_thread_id(child_thread_id: str, content: str) -> str:
|
||||
"""把子线程 ID 放进工具结果，方便后续继续同一子任务。"""
|
||||
return f"> 子智能体线程 ID: {child_thread_id}\n\n---\n\n{content}"
|
||||
@@ -0,0 +1,777 @@
|
||||
"""Yuxi adapter for DeepAgents conversation summarization middleware."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
import warnings
|
||||
from collections.abc import Awaitable, Callable, Iterable
|
||||
from contextvars import ContextVar
|
||||
from typing import Any
|
||||
|
||||
from deepagents.middleware.summarization import (
|
||||
Command,
|
||||
ContextOverflowError,
|
||||
SummarizationMiddleware,
|
||||
_aclip_overflow_tail,
|
||||
_clip_overflow_tail,
|
||||
)
|
||||
from langchain.agents.middleware.summarization import ContextSize
|
||||
from langchain.agents.middleware.types import ExtendedModelResponse, ModelRequest, ModelResponse
|
||||
from langchain.chat_models import BaseChatModel
|
||||
from langchain_core.messages import AIMessage, AnyMessage, ToolMessage, get_buffer_string
|
||||
from langchain_core.messages.utils import count_tokens_approximately
|
||||
from langgraph.config import get_stream_writer
|
||||
from langgraph.constants import TAG_NOSTREAM
|
||||
|
||||
from yuxi.agents.backends.composite import create_agent_composite_backend
|
||||
from yuxi.utils.logging_config import logger
|
||||
from yuxi.utils.paths import VIRTUAL_PATH_CONVERSATION_HISTORY, VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
|
||||
_APPROX_CHARS_PER_TOKEN = 4
|
||||
_DEFAULT_SUMMARY_TOOL_RESULT_LIMIT_TOKENS = 300
|
||||
_DEFAULT_L1_L2_TRIGGER_RATIO = 0.4
|
||||
_DEFAULT_TOOL_ARG_MAX_LENGTH = 2000
|
||||
_TRUNCATED_TOOL_ARG_TEXT = "...(argument truncated for context view)"
|
||||
_TOOL_RESULT_SAVED_MARKER = "yuxi_tool_result_saved"
|
||||
_SUMMARY_BACKEND: ContextVar[Any | None] = ContextVar("yuxi_summary_backend", default=None)
|
||||
_SUMMARY_SANITIZED_MESSAGES: ContextVar[dict[tuple[int, ...], list[AnyMessage]] | None] = ContextVar(
|
||||
"yuxi_summary_sanitized_messages",
|
||||
default=None,
|
||||
)
|
||||
_SUMMARY_COMPRESSION_STATE: ContextVar[dict[str, bool] | None] = ContextVar(
|
||||
"yuxi_summary_compression_state",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def _emit_compression(status: str, **extra: Any) -> None:
|
||||
try:
|
||||
writer = get_stream_writer()
|
||||
except RuntimeError:
|
||||
return
|
||||
writer({"type": "yuxi.context_compression", "status": status, **extra})
|
||||
|
||||
|
||||
def _emit_compression_started_once() -> None:
|
||||
state = _SUMMARY_COMPRESSION_STATE.get()
|
||||
if state is not None and state.get("started"):
|
||||
return
|
||||
if state is not None:
|
||||
state["started"] = True
|
||||
_emit_compression("started")
|
||||
|
||||
|
||||
def _count_tokens_for_summary_trigger(messages: Iterable[Any], **kwargs: Any) -> int:
|
||||
kwargs.pop("use_usage_metadata_scaling", None)
|
||||
return count_tokens_approximately(messages, use_usage_metadata_scaling=False, **kwargs)
|
||||
|
||||
|
||||
def _extract_text_content(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
parts.append(item)
|
||||
elif isinstance(item, dict) and isinstance(item.get("text"), str):
|
||||
parts.append(item["text"])
|
||||
return "\n".join(part for part in parts if part)
|
||||
return "" if content is None else str(content)
|
||||
|
||||
|
||||
def _tool_result_path(tool_name: str | None, content: str, large_tool_results_prefix: str) -> str:
|
||||
safe_tool_name = re.sub(r"[^A-Za-z0-9_.-]+", "-", (tool_name or "").strip()).strip(".-") or "tool-result"
|
||||
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()[:16]
|
||||
return f"{large_tool_results_prefix}/{safe_tool_name}-{digest}.txt"
|
||||
|
||||
|
||||
def _preview_tool_result(content: str, token_limit: int | None) -> tuple[str, int]:
|
||||
text = content.strip()
|
||||
if token_limit is None:
|
||||
return text, 0
|
||||
if token_limit <= 0:
|
||||
return "", len(text)
|
||||
|
||||
max_chars = token_limit * _APPROX_CHARS_PER_TOKEN
|
||||
if len(text) <= max_chars:
|
||||
return text, 0
|
||||
|
||||
preview = text[:max_chars].rstrip()
|
||||
return preview, len(text) - len(preview)
|
||||
|
||||
|
||||
def _write_tool_result(backend, path: str, content: str) -> str | None:
|
||||
if backend is None:
|
||||
return None
|
||||
|
||||
result = backend.write(path, content)
|
||||
error = getattr(result, "error", None)
|
||||
if not error:
|
||||
return path
|
||||
if "already exists" in str(error).lower():
|
||||
return path
|
||||
raise RuntimeError(f"Failed to write tool result to {path}: {error}")
|
||||
|
||||
|
||||
def _estimated_content_tokens(content: str) -> int:
|
||||
return max((len(content) + _APPROX_CHARS_PER_TOKEN - 1) // _APPROX_CHARS_PER_TOKEN, 1)
|
||||
|
||||
|
||||
def _tool_result_replacement_content(
|
||||
message: ToolMessage,
|
||||
*,
|
||||
backend,
|
||||
tool_result_offload_token_limit: int | None,
|
||||
large_tool_results_prefix: str,
|
||||
) -> str:
|
||||
content = _extract_text_content(message.content)
|
||||
approx_tokens = max((len(content) + _APPROX_CHARS_PER_TOKEN - 1) // _APPROX_CHARS_PER_TOKEN, 1)
|
||||
tool_name = message.name if isinstance(message.name, str) and message.name else None
|
||||
path = _write_tool_result(
|
||||
backend,
|
||||
_tool_result_path(tool_name, content, large_tool_results_prefix),
|
||||
content,
|
||||
)
|
||||
preview, omitted_chars = _preview_tool_result(content, tool_result_offload_token_limit)
|
||||
|
||||
lines = [
|
||||
"[Tool result saved]",
|
||||
f"Tool: {tool_name or 'unknown'}",
|
||||
f"Approx tokens: {approx_tokens}",
|
||||
]
|
||||
if path:
|
||||
lines.append(f"Full output path: {path}")
|
||||
if preview:
|
||||
lines.extend(["", "Output preview:", preview])
|
||||
if omitted_chars:
|
||||
lines.append(f"\n[Truncated {omitted_chars} chars. Read the full output from the saved file.]")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _should_offload_tool_message(message: ToolMessage, trigger_tokens: int | None) -> bool:
|
||||
if trigger_tokens is None:
|
||||
return True
|
||||
if trigger_tokens <= 0:
|
||||
return True
|
||||
content = _extract_text_content(message.content)
|
||||
return _estimated_content_tokens(content) > trigger_tokens
|
||||
|
||||
|
||||
def _replace_tool_message_content(
|
||||
message: ToolMessage,
|
||||
*,
|
||||
backend,
|
||||
tool_result_offload_token_limit: int | None,
|
||||
large_tool_results_prefix: str,
|
||||
) -> ToolMessage:
|
||||
additional_kwargs = dict(getattr(message, "additional_kwargs", {}) or {})
|
||||
additional_kwargs[_TOOL_RESULT_SAVED_MARKER] = True
|
||||
return message.model_copy(
|
||||
update={
|
||||
"content": _tool_result_replacement_content(
|
||||
message,
|
||||
backend=backend,
|
||||
tool_result_offload_token_limit=tool_result_offload_token_limit,
|
||||
large_tool_results_prefix=large_tool_results_prefix,
|
||||
),
|
||||
"additional_kwargs": additional_kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _truncate_string_arg(value: str, max_length: int) -> str:
|
||||
if len(value) <= max_length:
|
||||
return value
|
||||
return f"{value[:20]}{_TRUNCATED_TOOL_ARG_TEXT}"
|
||||
|
||||
|
||||
def _truncate_tool_call_args(tool_call: dict[str, Any], max_length: int) -> tuple[dict[str, Any], bool]:
|
||||
if tool_call.get("name") not in {"write_file", "edit_file"}:
|
||||
return tool_call, False
|
||||
args = tool_call.get("args")
|
||||
if not isinstance(args, dict):
|
||||
return tool_call, False
|
||||
|
||||
truncated_args: dict[str, Any] = {}
|
||||
modified = False
|
||||
for key, value in args.items():
|
||||
if isinstance(value, str) and len(value) > max_length:
|
||||
truncated_args[key] = _truncate_string_arg(value, max_length)
|
||||
modified = True
|
||||
else:
|
||||
truncated_args[key] = value
|
||||
if not modified:
|
||||
return tool_call, False
|
||||
return {**tool_call, "args": truncated_args}, True
|
||||
|
||||
|
||||
def _truncate_provider_tool_calls(additional_kwargs: dict[str, Any], max_length: int) -> tuple[dict[str, Any], bool]:
|
||||
raw_tool_calls = additional_kwargs.get("tool_calls")
|
||||
if not isinstance(raw_tool_calls, list):
|
||||
return additional_kwargs, False
|
||||
|
||||
updated_tool_calls: list[Any] = []
|
||||
modified = False
|
||||
for raw_call in raw_tool_calls:
|
||||
if not isinstance(raw_call, dict):
|
||||
updated_tool_calls.append(raw_call)
|
||||
continue
|
||||
function = raw_call.get("function")
|
||||
if not isinstance(function, dict):
|
||||
updated_tool_calls.append(raw_call)
|
||||
continue
|
||||
if function.get("name") not in {"write_file", "edit_file"}:
|
||||
updated_tool_calls.append(raw_call)
|
||||
continue
|
||||
arguments = function.get("arguments")
|
||||
if not isinstance(arguments, str) or len(arguments) <= max_length:
|
||||
updated_tool_calls.append(raw_call)
|
||||
continue
|
||||
|
||||
updated_function = {**function, "arguments": _truncate_string_arg(arguments, max_length)}
|
||||
updated_tool_calls.append({**raw_call, "function": updated_function})
|
||||
modified = True
|
||||
|
||||
if not modified:
|
||||
return additional_kwargs, False
|
||||
return {**additional_kwargs, "tool_calls": updated_tool_calls}, True
|
||||
|
||||
|
||||
def _truncate_ai_tool_call_args(message: AIMessage, *, max_length: int) -> AIMessage:
|
||||
if not message.tool_calls and not getattr(message, "additional_kwargs", None):
|
||||
return message
|
||||
|
||||
updated_tool_calls: list[Any] = []
|
||||
tool_calls_modified = False
|
||||
for tool_call in message.tool_calls or []:
|
||||
if not isinstance(tool_call, dict):
|
||||
updated_tool_calls.append(tool_call)
|
||||
continue
|
||||
updated, modified = _truncate_tool_call_args(tool_call, max_length)
|
||||
updated_tool_calls.append(updated)
|
||||
tool_calls_modified = tool_calls_modified or modified
|
||||
|
||||
additional_kwargs = dict(getattr(message, "additional_kwargs", {}) or {})
|
||||
additional_kwargs, additional_kwargs_modified = _truncate_provider_tool_calls(additional_kwargs, max_length)
|
||||
|
||||
if not tool_calls_modified and not additional_kwargs_modified:
|
||||
return message
|
||||
|
||||
updated_message = message.model_copy(update={"additional_kwargs": additional_kwargs})
|
||||
if tool_calls_modified:
|
||||
updated_message.tool_calls = updated_tool_calls
|
||||
return updated_message
|
||||
|
||||
|
||||
def sanitize_messages_for_summary(
|
||||
messages: list[AnyMessage],
|
||||
*,
|
||||
backend=None,
|
||||
tool_result_offload_token_limit: int | None = _DEFAULT_SUMMARY_TOOL_RESULT_LIMIT_TOKENS,
|
||||
large_tool_results_prefix: str = VIRTUAL_PATH_LARGE_TOOL_RESULTS,
|
||||
) -> list[AnyMessage]:
|
||||
"""Build a compact summary/offload view by replacing only ToolMessage content."""
|
||||
sanitized: list[AnyMessage] = []
|
||||
for message in messages:
|
||||
if isinstance(message, ToolMessage):
|
||||
if getattr(message, "additional_kwargs", {}).get(_TOOL_RESULT_SAVED_MARKER) is True:
|
||||
sanitized.append(message)
|
||||
continue
|
||||
sanitized.append(
|
||||
_replace_tool_message_content(
|
||||
message,
|
||||
backend=backend,
|
||||
tool_result_offload_token_limit=tool_result_offload_token_limit,
|
||||
large_tool_results_prefix=large_tool_results_prefix,
|
||||
)
|
||||
)
|
||||
continue
|
||||
sanitized.append(message)
|
||||
return sanitized
|
||||
|
||||
|
||||
class YuxiSummarizationMiddleware(SummarizationMiddleware):
|
||||
"""DeepAgents summarization middleware with Yuxi-specific tool-call sanitization."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
tool_result_offload_token_limit: int | None = _DEFAULT_SUMMARY_TOOL_RESULT_LIMIT_TOKENS,
|
||||
l1_l2_trigger_ratio: float = _DEFAULT_L1_L2_TRIGGER_RATIO,
|
||||
tool_arg_max_length: int = _DEFAULT_TOOL_ARG_MAX_LENGTH,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.tool_result_offload_token_limit = tool_result_offload_token_limit
|
||||
self.l1_l2_trigger_ratio = l1_l2_trigger_ratio
|
||||
self.tool_arg_max_length = tool_arg_max_length
|
||||
|
||||
def _should_summarize(self, messages: list[AnyMessage], total_tokens: int) -> bool:
|
||||
if not self._lc_helper._trigger_clauses:
|
||||
return False
|
||||
|
||||
for clause in self._lc_helper._trigger_clauses:
|
||||
clause_met = True
|
||||
for kind, value in clause.items():
|
||||
if kind == "messages":
|
||||
if len(messages) < value:
|
||||
clause_met = False
|
||||
break
|
||||
elif kind == "tokens":
|
||||
if total_tokens < value:
|
||||
clause_met = False
|
||||
break
|
||||
elif kind == "fraction":
|
||||
max_input_tokens = self._get_profile_limits()
|
||||
if max_input_tokens is None:
|
||||
clause_met = False
|
||||
break
|
||||
threshold = int(max_input_tokens * value)
|
||||
if threshold <= 0:
|
||||
threshold = 1
|
||||
if total_tokens < threshold:
|
||||
clause_met = False
|
||||
break
|
||||
if clause_met:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _sanitize_messages_for_summary(
|
||||
self,
|
||||
messages: list[AnyMessage],
|
||||
*,
|
||||
backend,
|
||||
) -> list[AnyMessage]:
|
||||
_ = backend
|
||||
cache = _SUMMARY_SANITIZED_MESSAGES.get()
|
||||
cache_key = tuple(id(message) for message in messages)
|
||||
if cache is not None and cache_key in cache:
|
||||
return cache[cache_key]
|
||||
|
||||
sanitized = messages
|
||||
if cache is not None:
|
||||
cache[cache_key] = sanitized
|
||||
return sanitized
|
||||
|
||||
def _sanitize_messages_for_l1(
|
||||
self,
|
||||
messages: list[AnyMessage],
|
||||
*,
|
||||
backend,
|
||||
) -> list[AnyMessage]:
|
||||
compacted: list[AnyMessage] = []
|
||||
modified = False
|
||||
for message in messages:
|
||||
if isinstance(message, AIMessage):
|
||||
updated = _truncate_ai_tool_call_args(message, max_length=self.tool_arg_max_length)
|
||||
compacted.append(updated)
|
||||
modified = modified or updated is not message
|
||||
continue
|
||||
if isinstance(message, ToolMessage) and _should_offload_tool_message(
|
||||
message,
|
||||
self.tool_result_offload_token_limit,
|
||||
):
|
||||
updated = _replace_tool_message_content(
|
||||
message,
|
||||
backend=backend,
|
||||
tool_result_offload_token_limit=self.tool_result_offload_token_limit,
|
||||
large_tool_results_prefix=self._large_tool_results_prefix,
|
||||
)
|
||||
compacted.append(updated)
|
||||
modified = modified or updated is not message
|
||||
continue
|
||||
compacted.append(message)
|
||||
return compacted if modified else messages
|
||||
|
||||
def _count_request_tokens(
|
||||
self,
|
||||
messages: list[AnyMessage],
|
||||
*,
|
||||
system_message,
|
||||
tools,
|
||||
) -> int:
|
||||
counted_messages = [system_message, *messages] if system_message is not None else messages
|
||||
try:
|
||||
return self.token_counter(counted_messages, tools=tools) # ty: ignore[unknown-argument]
|
||||
except TypeError:
|
||||
return self.token_counter(counted_messages)
|
||||
|
||||
def _entry_trigger_tokens(self) -> int | None:
|
||||
token_thresholds: list[int] = []
|
||||
for clause in getattr(self._lc_helper, "_trigger_clauses", []) or []:
|
||||
value = clause.get("tokens")
|
||||
if isinstance(value, int) and value > 0:
|
||||
token_thresholds.append(value)
|
||||
fraction = clause.get("fraction")
|
||||
if isinstance(fraction, int | float):
|
||||
max_input_tokens = self._get_profile_limits()
|
||||
if max_input_tokens is not None:
|
||||
threshold = int(max_input_tokens * fraction)
|
||||
token_thresholds.append(max(threshold, 1))
|
||||
return min(token_thresholds) if token_thresholds else None
|
||||
|
||||
def _should_run_l1(self, messages: list[AnyMessage], total_tokens: int) -> bool:
|
||||
return self._should_summarize(messages, total_tokens)
|
||||
|
||||
def _should_run_l2(self, compacted_total_tokens: int, entry_threshold_tokens: int | None) -> bool:
|
||||
if entry_threshold_tokens is None:
|
||||
return True
|
||||
threshold = max(int(entry_threshold_tokens * self.l1_l2_trigger_ratio), 1)
|
||||
return compacted_total_tokens > threshold
|
||||
|
||||
def _backend_for_request(self, request: ModelRequest):
|
||||
try:
|
||||
return self._get_backend(request.state, request.runtime)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _summarization_event_from_result(result: Any) -> dict | None:
|
||||
if not isinstance(result, ExtendedModelResponse):
|
||||
return None
|
||||
command = getattr(result, "command", None)
|
||||
update = getattr(command, "update", None) if command is not None else None
|
||||
if not isinstance(update, dict):
|
||||
return None
|
||||
event = update.get("_summarization_event")
|
||||
return event if isinstance(event, dict) else None
|
||||
|
||||
def _emit_completed(self, result: Any) -> None:
|
||||
event = self._summarization_event_from_result(result)
|
||||
if event is not None:
|
||||
_emit_compression(
|
||||
"completed",
|
||||
cutoff_index=event.get("cutoff_index"),
|
||||
file_path=event.get("file_path"),
|
||||
)
|
||||
|
||||
# 重写 _create_summary/_acreate_summary 以在摘要 LLM 调用上挂 TAG_NOSTREAM：父类
|
||||
# 的 model.invoke 带 lc_source 元数据但无 nostream 标记，其 token 流会被 LangGraph
|
||||
# messages stream 捕获并广播到前端，形成 phantom 摘要消息。带 TAG_NOSTREAM 后流式
|
||||
# 层在源头跳过该调用，无需 chat_service 下游过滤，主 messages 流天然只含用户可见回复。
|
||||
# 父类硬编码 invoke config 且无 tags 钩子（self.model 为中间件实例共享属性，并发下不能
|
||||
# 临时换绑 bind(tags=...)），故只能重写；trim/format 是纯同步逻辑，抽到 _build_summary_prompt
|
||||
# 供 sync/async 两条路径共用，避免逐字重复。
|
||||
_SUMMARY_INVOKE_CONFIG = {"metadata": {"lc_source": "summarization"}, "tags": [TAG_NOSTREAM]}
|
||||
|
||||
def _build_summary_prompt(self, sanitized: list[AnyMessage]) -> str | None:
|
||||
trimmed = self._lc_helper._trim_messages_for_summary(sanitized)
|
||||
if not trimmed:
|
||||
return None
|
||||
return self._lc_helper.summary_prompt.format(messages=get_buffer_string(trimmed, format="xml")).rstrip()
|
||||
|
||||
def _create_summary(self, messages_to_summarize: list[AnyMessage]) -> str:
|
||||
sanitized = self._sanitize_messages_for_summary(
|
||||
messages_to_summarize,
|
||||
backend=_SUMMARY_BACKEND.get(),
|
||||
)
|
||||
if not sanitized:
|
||||
return "No previous conversation history."
|
||||
prompt = self._build_summary_prompt(sanitized)
|
||||
if prompt is None:
|
||||
return "Previous conversation was too long to summarize."
|
||||
try:
|
||||
return self.model.invoke(prompt, config=self._SUMMARY_INVOKE_CONFIG).text.strip()
|
||||
except Exception as e:
|
||||
return f"Error generating summary: {e!s}"
|
||||
|
||||
async def _acreate_summary(self, messages_to_summarize: list[AnyMessage]) -> str:
|
||||
sanitized = self._sanitize_messages_for_summary(
|
||||
messages_to_summarize,
|
||||
backend=_SUMMARY_BACKEND.get(),
|
||||
)
|
||||
if not sanitized:
|
||||
return "No previous conversation history."
|
||||
prompt = self._build_summary_prompt(sanitized)
|
||||
if prompt is None:
|
||||
return "Previous conversation was too long to summarize."
|
||||
try:
|
||||
response = await self.model.ainvoke(prompt, config=self._SUMMARY_INVOKE_CONFIG)
|
||||
return response.text.strip()
|
||||
except Exception as e:
|
||||
return f"Error generating summary: {e!s}"
|
||||
|
||||
def _offload_to_backend(self, backend, messages: list[AnyMessage]) -> str | None:
|
||||
_emit_compression_started_once()
|
||||
return super()._offload_to_backend(
|
||||
backend,
|
||||
self._sanitize_messages_for_summary(
|
||||
messages,
|
||||
backend=backend,
|
||||
),
|
||||
)
|
||||
|
||||
async def _aoffload_to_backend(self, backend, messages: list[AnyMessage]) -> str | None:
|
||||
_emit_compression_started_once()
|
||||
return await super()._aoffload_to_backend(
|
||||
backend,
|
||||
self._sanitize_messages_for_summary(
|
||||
messages,
|
||||
backend=backend,
|
||||
),
|
||||
)
|
||||
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelResponse:
|
||||
backend_token = _SUMMARY_BACKEND.set(self._backend_for_request(request))
|
||||
sanitized_token = _SUMMARY_SANITIZED_MESSAGES.set({})
|
||||
compression_state: dict[str, bool] = {"started": False}
|
||||
compression_token = _SUMMARY_COMPRESSION_STATE.set(compression_state)
|
||||
try:
|
||||
try:
|
||||
result = self._wrap_model_call_with_l1(request, handler)
|
||||
except Exception as exc:
|
||||
if compression_state.get("started"):
|
||||
_emit_compression("failed", error=repr(exc))
|
||||
raise
|
||||
self._emit_completed(result)
|
||||
return result
|
||||
finally:
|
||||
_SUMMARY_COMPRESSION_STATE.reset(compression_token)
|
||||
_SUMMARY_SANITIZED_MESSAGES.reset(sanitized_token)
|
||||
_SUMMARY_BACKEND.reset(backend_token)
|
||||
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelResponse:
|
||||
backend_token = _SUMMARY_BACKEND.set(self._backend_for_request(request))
|
||||
sanitized_token = _SUMMARY_SANITIZED_MESSAGES.set({})
|
||||
compression_state: dict[str, bool] = {"started": False}
|
||||
compression_token = _SUMMARY_COMPRESSION_STATE.set(compression_state)
|
||||
try:
|
||||
try:
|
||||
result = await self._awrap_model_call_with_l1(request, handler)
|
||||
except Exception as exc:
|
||||
if compression_state.get("started"):
|
||||
_emit_compression("failed", error=repr(exc))
|
||||
raise
|
||||
self._emit_completed(result)
|
||||
return result
|
||||
finally:
|
||||
_SUMMARY_COMPRESSION_STATE.reset(compression_token)
|
||||
_SUMMARY_SANITIZED_MESSAGES.reset(sanitized_token)
|
||||
_SUMMARY_BACKEND.reset(backend_token)
|
||||
|
||||
def _wrap_model_call_with_l1(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelResponse | ExtendedModelResponse:
|
||||
effective_messages = self._get_effective_messages(request)
|
||||
truncated_messages, _ = self._truncate_args(
|
||||
effective_messages,
|
||||
request.system_message,
|
||||
request.tools,
|
||||
)
|
||||
total_tokens = self._count_request_tokens(
|
||||
truncated_messages,
|
||||
system_message=request.system_message,
|
||||
tools=request.tools,
|
||||
)
|
||||
should_run_l1 = self._should_run_l1(truncated_messages, total_tokens)
|
||||
|
||||
overflow_triggered = False
|
||||
if not should_run_l1:
|
||||
try:
|
||||
return handler(request.override(messages=truncated_messages))
|
||||
except ContextOverflowError:
|
||||
overflow_triggered = True
|
||||
|
||||
backend = self._get_backend(request.state, request.runtime)
|
||||
l1_messages = self._sanitize_messages_for_l1(truncated_messages, backend=backend)
|
||||
if should_run_l1:
|
||||
_emit_compression_started_once()
|
||||
|
||||
l1_total_tokens = self._count_request_tokens(
|
||||
l1_messages,
|
||||
system_message=request.system_message,
|
||||
tools=request.tools,
|
||||
)
|
||||
should_run_l2 = overflow_triggered or self._should_run_l2(l1_total_tokens, self._entry_trigger_tokens())
|
||||
|
||||
if not should_run_l2:
|
||||
try:
|
||||
response = handler(request.override(messages=l1_messages))
|
||||
except ContextOverflowError:
|
||||
overflow_triggered = True
|
||||
else:
|
||||
if should_run_l1:
|
||||
_emit_compression("completed")
|
||||
return response
|
||||
|
||||
cutoff_index = self._determine_cutoff_index(l1_messages)
|
||||
if cutoff_index <= 0:
|
||||
response = handler(request.override(messages=l1_messages))
|
||||
if should_run_l1:
|
||||
_emit_compression("completed")
|
||||
return response
|
||||
|
||||
messages_to_summarize, preserved_messages = self._partition_messages(l1_messages, cutoff_index)
|
||||
new_state_tail: list[AnyMessage] = []
|
||||
if overflow_triggered:
|
||||
preserved_messages, new_state_tail = _clip_overflow_tail(
|
||||
preserved_messages,
|
||||
backend,
|
||||
keep=self._lc_helper.keep,
|
||||
max_input_tokens=self._get_profile_limits(),
|
||||
token_counter=self.token_counter,
|
||||
large_tool_results_prefix=self._large_tool_results_prefix,
|
||||
)
|
||||
|
||||
file_path = self._offload_to_backend(backend, messages_to_summarize)
|
||||
if file_path is None:
|
||||
msg = (
|
||||
"Offloading conversation history to backend failed during summarization. "
|
||||
"Older messages will not be recoverable."
|
||||
)
|
||||
logger.error(msg)
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
|
||||
summary = self._create_summary(messages_to_summarize)
|
||||
new_messages = self._build_new_messages_with_path(summary, file_path)
|
||||
previous_event = request.state.get("_summarization_event")
|
||||
state_cutoff_index = self._compute_state_cutoff(previous_event, cutoff_index)
|
||||
new_event = {
|
||||
"cutoff_index": state_cutoff_index,
|
||||
"summary_message": new_messages[0],
|
||||
"file_path": file_path,
|
||||
}
|
||||
|
||||
response = handler(request.override(messages=[*new_messages, *preserved_messages]))
|
||||
update: dict[str, Any] = {"_summarization_event": new_event}
|
||||
if new_state_tail:
|
||||
update["messages"] = list(new_state_tail)
|
||||
return ExtendedModelResponse(model_response=response, command=Command(update=update))
|
||||
|
||||
async def _awrap_model_call_with_l1(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelResponse | ExtendedModelResponse:
|
||||
effective_messages = self._get_effective_messages(request)
|
||||
truncated_messages, _ = self._truncate_args(
|
||||
effective_messages,
|
||||
request.system_message,
|
||||
request.tools,
|
||||
)
|
||||
total_tokens = self._count_request_tokens(
|
||||
truncated_messages,
|
||||
system_message=request.system_message,
|
||||
tools=request.tools,
|
||||
)
|
||||
should_run_l1 = self._should_run_l1(truncated_messages, total_tokens)
|
||||
|
||||
overflow_triggered = False
|
||||
if not should_run_l1:
|
||||
try:
|
||||
return await handler(request.override(messages=truncated_messages))
|
||||
except ContextOverflowError:
|
||||
overflow_triggered = True
|
||||
|
||||
backend = self._get_backend(request.state, request.runtime)
|
||||
l1_messages = self._sanitize_messages_for_l1(truncated_messages, backend=backend)
|
||||
if should_run_l1:
|
||||
_emit_compression_started_once()
|
||||
|
||||
l1_total_tokens = self._count_request_tokens(
|
||||
l1_messages,
|
||||
system_message=request.system_message,
|
||||
tools=request.tools,
|
||||
)
|
||||
should_run_l2 = overflow_triggered or self._should_run_l2(l1_total_tokens, self._entry_trigger_tokens())
|
||||
|
||||
if not should_run_l2:
|
||||
try:
|
||||
response = await handler(request.override(messages=l1_messages))
|
||||
except ContextOverflowError:
|
||||
overflow_triggered = True
|
||||
else:
|
||||
if should_run_l1:
|
||||
_emit_compression("completed")
|
||||
return response
|
||||
|
||||
cutoff_index = self._determine_cutoff_index(l1_messages)
|
||||
if cutoff_index <= 0:
|
||||
response = await handler(request.override(messages=l1_messages))
|
||||
if should_run_l1:
|
||||
_emit_compression("completed")
|
||||
return response
|
||||
|
||||
messages_to_summarize, preserved_messages = self._partition_messages(l1_messages, cutoff_index)
|
||||
new_state_tail: list[AnyMessage] = []
|
||||
if overflow_triggered:
|
||||
preserved_messages, new_state_tail = await _aclip_overflow_tail(
|
||||
preserved_messages,
|
||||
backend,
|
||||
keep=self._lc_helper.keep,
|
||||
max_input_tokens=self._get_profile_limits(),
|
||||
token_counter=self.token_counter,
|
||||
large_tool_results_prefix=self._large_tool_results_prefix,
|
||||
)
|
||||
|
||||
# Offload 与 summary 互相独立，并发执行以避免串行等待一次文件 I/O + 一次
|
||||
# LLM 调用；_SUMMARY_SANITIZED_MESSAGES 的 id 缓存保证两路 sanitize 不会重复
|
||||
# 写入工具结果文件，offload 失败返回 None 时 summary 仍可独立完成。
|
||||
file_path, summary = await asyncio.gather(
|
||||
self._aoffload_to_backend(backend, messages_to_summarize),
|
||||
self._acreate_summary(messages_to_summarize),
|
||||
)
|
||||
if file_path is None:
|
||||
msg = (
|
||||
"Offloading conversation history to backend failed during summarization. "
|
||||
"Older messages will not be recoverable."
|
||||
)
|
||||
logger.error(msg)
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
|
||||
new_messages = self._build_new_messages_with_path(summary, file_path)
|
||||
previous_event = request.state.get("_summarization_event")
|
||||
state_cutoff_index = self._compute_state_cutoff(previous_event, cutoff_index)
|
||||
new_event = {
|
||||
"cutoff_index": state_cutoff_index,
|
||||
"summary_message": new_messages[0],
|
||||
"file_path": file_path,
|
||||
}
|
||||
|
||||
response = await handler(request.override(messages=[*new_messages, *preserved_messages]))
|
||||
update: dict[str, Any] = {"_summarization_event": new_event}
|
||||
if new_state_tail:
|
||||
update["messages"] = list(new_state_tail)
|
||||
return ExtendedModelResponse(model_response=response, command=Command(update=update))
|
||||
|
||||
|
||||
def create_summary_middleware(
|
||||
model: str | BaseChatModel,
|
||||
*,
|
||||
trigger: ContextSize | list[ContextSize] | None,
|
||||
keep: ContextSize | list[ContextSize] | None,
|
||||
summary_prompt: str | None = None,
|
||||
trim_tokens_to_summarize: int | None = None,
|
||||
tool_result_offload_token_limit: int | None = _DEFAULT_SUMMARY_TOOL_RESULT_LIMIT_TOKENS,
|
||||
l1_l2_trigger_ratio: float = _DEFAULT_L1_L2_TRIGGER_RATIO,
|
||||
) -> SummarizationMiddleware:
|
||||
"""Create DeepAgents summarization middleware using Yuxi's virtual outputs root."""
|
||||
middleware_kwargs = {
|
||||
"model": model,
|
||||
"backend": create_agent_composite_backend,
|
||||
"trigger": trigger,
|
||||
"keep": keep,
|
||||
"token_counter": _count_tokens_for_summary_trigger,
|
||||
"trim_tokens_to_summarize": trim_tokens_to_summarize,
|
||||
"tool_result_offload_token_limit": tool_result_offload_token_limit,
|
||||
"l1_l2_trigger_ratio": l1_l2_trigger_ratio,
|
||||
}
|
||||
if summary_prompt and summary_prompt.strip():
|
||||
middleware_kwargs["summary_prompt"] = summary_prompt
|
||||
middleware = YuxiSummarizationMiddleware(**middleware_kwargs)
|
||||
middleware._history_path_prefix = VIRTUAL_PATH_CONVERSATION_HISTORY
|
||||
middleware._large_tool_results_prefix = VIRTUAL_PATH_LARGE_TOOL_RESULTS
|
||||
return middleware
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Token usage observation middleware for Yuxi agents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, NotRequired, TypedDict
|
||||
|
||||
from langchain.agents.middleware.types import (
|
||||
AgentMiddleware,
|
||||
AgentState,
|
||||
ExtendedModelResponse,
|
||||
ModelRequest,
|
||||
ModelResponse,
|
||||
)
|
||||
from langchain_core.messages import AIMessage, AnyMessage
|
||||
from langchain_core.messages.utils import count_tokens_approximately
|
||||
from langgraph.types import Command
|
||||
|
||||
|
||||
class TokenUsagePayload(TypedDict, total=False):
|
||||
"""Serializable token usage snapshot stored in LangGraph state."""
|
||||
|
||||
state_message_count: int
|
||||
state_message_count_before_call: int
|
||||
state_messages_tokens: int
|
||||
state_messages_tokens_before_call: int
|
||||
llm_message_count: int
|
||||
llm_messages_tokens: int
|
||||
llm_content_message_count: int
|
||||
llm_content_message_tokens: int
|
||||
llm_tool_message_count: int
|
||||
llm_tool_message_tokens: int
|
||||
llm_input_tokens: int
|
||||
system_tokens: int
|
||||
tools_tokens: int
|
||||
tool_count: int
|
||||
context_window: int | None
|
||||
context_usage_ratio: float | None
|
||||
remaining_context_tokens: int | None
|
||||
summary_active: bool
|
||||
summary_message_tokens: int
|
||||
summary_trigger_tokens: int | None
|
||||
model_usage: dict[str, int]
|
||||
counter: str
|
||||
estimate: bool
|
||||
measured_at: str
|
||||
|
||||
|
||||
class TokenUsageState(AgentState):
|
||||
"""Agent state extension with the latest token usage snapshot."""
|
||||
|
||||
token_usage: NotRequired[TokenUsagePayload]
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float) and value.is_integer():
|
||||
return int(value)
|
||||
return None
|
||||
|
||||
|
||||
def _model_context_window(model: Any) -> int | None:
|
||||
profile = getattr(model, "profile", None)
|
||||
if not isinstance(profile, Mapping):
|
||||
return None
|
||||
max_input_tokens = profile.get("max_input_tokens")
|
||||
return max_input_tokens if isinstance(max_input_tokens, int) and max_input_tokens > 0 else None
|
||||
|
||||
|
||||
def _summary_trigger_tokens(runtime_context: Any) -> int | None:
|
||||
threshold = _safe_int(getattr(runtime_context, "summary_threshold", None))
|
||||
if threshold is None or threshold <= 0:
|
||||
return None
|
||||
return threshold * 1024
|
||||
|
||||
|
||||
def _is_summary_message(message: AnyMessage) -> bool:
|
||||
return getattr(message, "additional_kwargs", {}).get("lc_source") == "summarization"
|
||||
|
||||
|
||||
def _is_tool_message(message: AnyMessage) -> bool:
|
||||
return getattr(message, "type", None) == "tool" or getattr(message, "role", None) == "tool"
|
||||
|
||||
|
||||
def _model_usage_from_response(response: ModelResponse) -> dict[str, int]:
|
||||
for message in reversed(response.result):
|
||||
if not isinstance(message, AIMessage):
|
||||
continue
|
||||
usage = getattr(message, "usage_metadata", None)
|
||||
if not isinstance(usage, Mapping):
|
||||
continue
|
||||
return {str(key): value for key, value in usage.items() if isinstance(value, int)}
|
||||
return {}
|
||||
|
||||
|
||||
class TokenUsageMiddleware(AgentMiddleware[TokenUsageState]):
|
||||
"""Record approximate context token usage for the current model request."""
|
||||
|
||||
state_schema = TokenUsageState
|
||||
|
||||
def __init__(self, token_counter=count_tokens_approximately) -> None:
|
||||
super().__init__()
|
||||
self.token_counter = token_counter
|
||||
|
||||
def _count_tokens(self, messages: Iterable[Any], *, tools: list[Any] | None = None) -> int:
|
||||
message_list = list(messages)
|
||||
if tools is not None:
|
||||
return int(self.token_counter(message_list, tools=tools))
|
||||
return int(self.token_counter(message_list))
|
||||
|
||||
def _build_snapshot(self, request: ModelRequest, response: ModelResponse) -> TokenUsagePayload:
|
||||
state_messages = list(request.state.get("messages") or [])
|
||||
llm_messages = list(request.messages or [])
|
||||
system_messages = [request.system_message] if request.system_message is not None else []
|
||||
tools = list(request.tools or [])
|
||||
response_messages = list(response.result or [])
|
||||
|
||||
state_tokens_before_call = self._count_tokens(state_messages)
|
||||
next_state_messages = [*state_messages, *response_messages]
|
||||
state_messages_tokens = self._count_tokens(next_state_messages)
|
||||
llm_messages_tokens = self._count_tokens(llm_messages)
|
||||
system_tokens = self._count_tokens(system_messages)
|
||||
tools_tokens = self._count_tokens([], tools=tools) if tools else 0
|
||||
llm_input_tokens = self._count_tokens([*system_messages, *llm_messages], tools=tools)
|
||||
|
||||
context_window = _model_context_window(request.model)
|
||||
context_usage_ratio = None
|
||||
remaining_context_tokens = None
|
||||
if context_window:
|
||||
context_usage_ratio = min(1.0, round(llm_input_tokens / context_window, 4))
|
||||
remaining_context_tokens = max(context_window - llm_input_tokens, 0)
|
||||
|
||||
summary_message = llm_messages[0] if llm_messages and _is_summary_message(llm_messages[0]) else None
|
||||
llm_tool_messages = [message for message in llm_messages if _is_tool_message(message)]
|
||||
llm_content_messages = [
|
||||
message for message in llm_messages if not _is_tool_message(message) and not _is_summary_message(message)
|
||||
]
|
||||
summary_trigger_tokens = _summary_trigger_tokens(getattr(request.runtime, "context", None))
|
||||
|
||||
return {
|
||||
"state_message_count": len(next_state_messages),
|
||||
"state_message_count_before_call": len(state_messages),
|
||||
"state_messages_tokens": state_messages_tokens,
|
||||
"state_messages_tokens_before_call": state_tokens_before_call,
|
||||
"llm_message_count": len(llm_messages),
|
||||
"llm_messages_tokens": llm_messages_tokens,
|
||||
"llm_content_message_count": len(llm_content_messages),
|
||||
"llm_content_message_tokens": self._count_tokens(llm_content_messages),
|
||||
"llm_tool_message_count": len(llm_tool_messages),
|
||||
"llm_tool_message_tokens": self._count_tokens(llm_tool_messages),
|
||||
"llm_input_tokens": llm_input_tokens,
|
||||
"system_tokens": system_tokens,
|
||||
"tools_tokens": tools_tokens,
|
||||
"tool_count": len(tools),
|
||||
"context_window": context_window,
|
||||
"context_usage_ratio": context_usage_ratio,
|
||||
"remaining_context_tokens": remaining_context_tokens,
|
||||
"summary_active": summary_message is not None,
|
||||
"summary_message_tokens": self._count_tokens([summary_message]) if summary_message else 0,
|
||||
"summary_trigger_tokens": summary_trigger_tokens,
|
||||
"model_usage": _model_usage_from_response(response),
|
||||
"counter": "langchain.count_tokens_approximately",
|
||||
"estimate": True,
|
||||
"measured_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ExtendedModelResponse:
|
||||
response = handler(request)
|
||||
return ExtendedModelResponse(
|
||||
model_response=response,
|
||||
command=Command(update={"token_usage": self._build_snapshot(request, response)}),
|
||||
)
|
||||
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ExtendedModelResponse:
|
||||
response = await handler(request)
|
||||
return ExtendedModelResponse(
|
||||
model_response=response,
|
||||
command=Command(update={"token_usage": self._build_snapshot(request, response)}),
|
||||
)
|
||||
@@ -0,0 +1,193 @@
|
||||
from typing import Any
|
||||
|
||||
from langchain.chat_models import BaseChatModel
|
||||
from langchain_openai import ChatOpenAI
|
||||
from pydantic import SecretStr
|
||||
|
||||
from yuxi import config as sys_config
|
||||
from yuxi.models.providers.cache import model_cache
|
||||
from yuxi.utils import get_docker_safe_url
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
_TOOL_IMAGE_USER_TEXT = "Images returned by read_file are attached below. Inspect them when answering."
|
||||
|
||||
|
||||
def resolve_chat_model_spec(model_spec: str | None, *, fallback: str | None = None) -> str:
|
||||
"""解析空模型配置，不吞掉已经配置但无效的模型值。
|
||||
|
||||
这里仅处理模型为空时的优先级：请求或配置值、调用方 fallback、系统默认模型；
|
||||
具体模型是否存在、是否为聊天模型仍由 model_cache 校验。
|
||||
"""
|
||||
for candidate in (model_spec, fallback, sys_config.default_model):
|
||||
if isinstance(candidate, str) and candidate.strip():
|
||||
return candidate.strip()
|
||||
raise ValueError("model spec 不能为空")
|
||||
|
||||
|
||||
def load_chat_model(fully_specified_name: str | None, **kwargs) -> BaseChatModel:
|
||||
fully_specified_name = resolve_chat_model_spec(fully_specified_name)
|
||||
|
||||
info = model_cache.get_model_info(fully_specified_name)
|
||||
if not info:
|
||||
available_specs = model_cache.get_all_specs("chat")
|
||||
available_ids = [item.spec for item in available_specs[:10]]
|
||||
raise ValueError(
|
||||
f"Unknown model spec: '{fully_specified_name}'. "
|
||||
f"Available chat models ({len(available_specs)}): {available_ids}"
|
||||
)
|
||||
|
||||
if info.model_type != "chat":
|
||||
raise ValueError(f"Model {fully_specified_name} is not a chat model (type={info.model_type})")
|
||||
|
||||
api_key = info.api_key
|
||||
base_url = get_docker_safe_url(info.base_url)
|
||||
|
||||
logger.debug(f"Loading model {fully_specified_name} with provider_type={info.provider_type}")
|
||||
|
||||
if info.provider_type == "anthropic":
|
||||
from langchain_anthropic import ChatAnthropic
|
||||
|
||||
return ChatAnthropic(
|
||||
model=info.model_id,
|
||||
api_key=SecretStr(api_key),
|
||||
base_url=base_url,
|
||||
**kwargs,
|
||||
)
|
||||
if info.provider_type == "gemini":
|
||||
from langchain_google_genai import ChatGoogleGenerativeAI
|
||||
|
||||
return ChatGoogleGenerativeAI(
|
||||
model=info.model_id,
|
||||
google_api_key=SecretStr(api_key),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
return _ToolCallChunkFixChatOpenAI(
|
||||
model=info.model_id,
|
||||
api_key=SecretStr(api_key),
|
||||
base_url=base_url,
|
||||
stream_usage=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class _ToolCallChunkFixChatOpenAI(ChatOpenAI):
|
||||
"""归一化流式 tool_call 续片中的空串 name/id，规避 v3 流式累积缺陷。"""
|
||||
|
||||
def _get_request_payload(self, input_, *, stop=None, **kwargs):
|
||||
"""Override to bridge tool image blocks to user messages."""
|
||||
payload = super()._get_request_payload(input_, stop=stop, **kwargs)
|
||||
return _bridge_tool_images_to_user_messages(payload)
|
||||
|
||||
async def _astream(self, *args, **kwargs):
|
||||
async for chunk in super()._astream(*args, **kwargs):
|
||||
_normalize_tool_call_chunks(chunk.message)
|
||||
yield chunk
|
||||
|
||||
def _stream(self, *args, **kwargs):
|
||||
for chunk in super()._stream(*args, **kwargs):
|
||||
_normalize_tool_call_chunks(chunk.message)
|
||||
yield chunk
|
||||
|
||||
|
||||
def _bridge_tool_images_to_user_messages(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""将工具调用返回的 image_url 块桥接到用户消息中，避免工具消息中包含图片导致的渲染问题。"""
|
||||
messages = payload.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
return payload
|
||||
if not any(isinstance(m, dict) and m.get("role") == "tool" and _tool_image_blocks(m) for m in messages):
|
||||
return payload
|
||||
|
||||
bridged_messages: list[dict[str, Any]] = []
|
||||
pending_images: list[dict[str, Any]] = []
|
||||
|
||||
def flush_pending_images() -> None:
|
||||
nonlocal pending_images
|
||||
if not pending_images:
|
||||
return
|
||||
|
||||
bridged_messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": _TOOL_IMAGE_USER_TEXT}, *pending_images],
|
||||
}
|
||||
)
|
||||
pending_images = []
|
||||
|
||||
for message in messages:
|
||||
if not isinstance(message, dict):
|
||||
flush_pending_images()
|
||||
bridged_messages.append(message)
|
||||
continue
|
||||
|
||||
role = message.get("role")
|
||||
if role != "tool":
|
||||
flush_pending_images()
|
||||
|
||||
image_blocks = _tool_image_blocks(message) if role == "tool" else []
|
||||
if image_blocks:
|
||||
pending_images.extend(image_blocks)
|
||||
|
||||
content = _text_without_images(message.get("content"), image_blocks)
|
||||
if not content:
|
||||
content = (
|
||||
f"read_file returned {len(image_blocks)} image(s). "
|
||||
"The image content is attached in the following user message for visual inspection."
|
||||
)
|
||||
message = {**message, "content": content}
|
||||
|
||||
bridged_messages.append(message)
|
||||
|
||||
flush_pending_images()
|
||||
|
||||
return {**payload, "messages": bridged_messages}
|
||||
|
||||
|
||||
def _normalize_tool_call_chunks(message) -> None:
|
||||
"""把工具调用续片里空字符串的 name/id 归一化为 None。
|
||||
|
||||
LangGraph v3 流式累积对 tool_call 字段是“后值覆盖”：部分 OpenAI 兼容提供商
|
||||
（siliconflow、阿里云百炼等）在续片里把 name/id 下发为空字符串 ""，会覆盖首片
|
||||
的真实值（siliconflow 丢 name、百炼丢 id），导致工具结果无法按 tool_call_id
|
||||
关联、工具状态停留在“进行中”。OpenAI 官方在续片里发 None 不会触发覆盖，这里
|
||||
把空串归一化为 None 对齐该行为。待上游修复 v3 协议后可移除。
|
||||
"""
|
||||
for chunk in message.tool_call_chunks:
|
||||
if chunk.get("name") == "":
|
||||
chunk["name"] = None
|
||||
if chunk.get("id") == "":
|
||||
chunk["id"] = None
|
||||
|
||||
|
||||
def _tool_image_blocks(message: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
content = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return []
|
||||
return [
|
||||
block
|
||||
for block in content
|
||||
if isinstance(block, dict)
|
||||
and block.get("type") == "image_url"
|
||||
and isinstance(block.get("image_url"), dict)
|
||||
and isinstance(block["image_url"].get("url"), str)
|
||||
]
|
||||
|
||||
|
||||
def _text_without_images(content: Any, image_blocks: list[dict[str, Any]]) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if not isinstance(content, list):
|
||||
return ""
|
||||
|
||||
image_ids = {id(block) for block in image_blocks}
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if id(block) in image_ids:
|
||||
continue
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict) and block.get("type") in {"text", "input_text"}:
|
||||
text = block.get("text")
|
||||
if isinstance(text, str):
|
||||
parts.append(text)
|
||||
return "\n".join(part for part in parts if part)
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuiltinSkillSpec:
|
||||
slug: str
|
||||
source_dir: Path
|
||||
description: str = ""
|
||||
version: str = "1.0.0"
|
||||
tool_dependencies: tuple[str, ...] = ()
|
||||
mcp_dependencies: tuple[str, ...] = ()
|
||||
skill_dependencies: tuple[str, ...] = ()
|
||||
|
||||
|
||||
_SKILLS_ROOT = Path(__file__).resolve().parent
|
||||
|
||||
BUILTIN_SKILLS: list[BuiltinSkillSpec] = [
|
||||
BuiltinSkillSpec(
|
||||
slug="image-gen",
|
||||
source_dir=_SKILLS_ROOT / "image-gen",
|
||||
description="在 Agent 沙盒中生成图片并保存到 outputs，默认支持 Qwen-Image，也可接入其它图片生成接口。",
|
||||
version="2026.06.02",
|
||||
tool_dependencies=("present_artifacts",),
|
||||
),
|
||||
BuiltinSkillSpec(
|
||||
slug="deep-research",
|
||||
source_dir=_SKILLS_ROOT / "deep-research",
|
||||
description="深度研究编排方法论：澄清范围、拆解规划、并行调度子智能体调研、对抗式核验、综合成带引用的结构化报告。",
|
||||
version="2026.06.05",
|
||||
tool_dependencies=("tavily_search",),
|
||||
),
|
||||
BuiltinSkillSpec(
|
||||
slug="knowledge-base",
|
||||
source_dir=_SKILLS_ROOT / "knowledge-base",
|
||||
description="使用 Yuxi 知识库进行检索、打开文档、文档内定位和查看思维导图。",
|
||||
version="2026.06.24",
|
||||
tool_dependencies=(
|
||||
"list_kbs",
|
||||
"query_kb",
|
||||
"find_kb_document",
|
||||
"open_kb_document",
|
||||
"get_mindmap",
|
||||
"search_file",
|
||||
),
|
||||
),
|
||||
BuiltinSkillSpec(
|
||||
slug="mysql-reporter",
|
||||
source_dir=_SKILLS_ROOT / "mysql-reporter",
|
||||
description="基于 MySQL 数据库生成查询报表和可视化图表，适合分析业务指标、统计趋势，并用 Charts MCP 展示结果。",
|
||||
version="2026.06.05",
|
||||
mcp_dependencies=("mcp-server-chart",),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,50 @@
|
||||
---
|
||||
name: deep-research
|
||||
description: "深度研究编排方法论：澄清范围、拆解规划、并行调度子智能体调研、对抗式核验、综合成带引用的结构化报告。当任务需要多来源、可追溯、需事实核查的深度研究时使用此技能。"
|
||||
---
|
||||
|
||||
# 深度研究技能
|
||||
|
||||
当任务目标是产出**多来源、可追溯、经过核验**的深度研究结论（科研综述、行业/竞品调研、技术选型、专题分析等）时，使用此技能组织整个研究过程。本技能的核心是**编排**：你负责整体把控与子智能体调度，把繁重的检索与核验工作派发出去，自己专注规划与综合。
|
||||
|
||||
## 可用子智能体
|
||||
|
||||
通过 `task` 工具调度（可并行多开，互不依赖的子任务同时派发）：
|
||||
|
||||
- `research-explorer`（调研探索员）：围绕一个明确子问题做多轮网页/知识库检索，返回按要点组织、带 `<cite>` 引用的结构化发现。**这是主力，按子问题并行多开。**
|
||||
- `fact-verifier`（事实核查员）：对给定的关键论断做对抗式核验，逐条给出 支持 / 存疑 / 反驳 + 依据来源 + 置信度，并标注冲突。
|
||||
|
||||
## 编排流程
|
||||
|
||||
### 1. 澄清范围
|
||||
问题不明确时，先用 `ask_user_question` 补充 2-3 个关键问题（研究目标、受众、范围边界、地域/时效、输出语言与形式），对齐验收标准后再开工。已经清晰的任务不要反复追问。
|
||||
|
||||
### 2. 规划拆解
|
||||
用 `write_todos` 把研究目标拆成**可独立调研**的子问题，每个子问题写明产出标准（要回答什么、需要哪类证据）。子问题应正交、覆盖完整，避免重叠或遗漏关键角度。
|
||||
|
||||
### 3. 并行派发调研
|
||||
- 把互不依赖的子问题用**多个 `task` 调用并行**派发给 `research-explorer`。
|
||||
- 每次派发在 `description` 中写清：子问题目标、已知上下文、期望输出格式（要点 + `<cite source="$URL" type="url">$INDEX</cite>` 引用 + 参考来源列表）。
|
||||
- 何时派发 vs 自己直检：子问题复杂、需多轮检索、可隔离上下文、可并行时一律派发子智能体；仅在澄清范围、补一两个零散事实、或快速校正方向时才自己少量直接检索。
|
||||
- 子问题之间有依赖时，先派发前置子问题，拿到结果后再派发后续。
|
||||
|
||||
### 4. 核验关键结论
|
||||
对**影响最终结论的关键论断**、数字、以及子智能体之间相互冲突的发现，派发 `fact-verifier` 做对抗式核验。要求其默认倾向「证据不足即标注存疑」。核验未通过的结论不要写进正文，或必须明确降级标注。
|
||||
|
||||
### 5. 综合成稿
|
||||
证据充分后，由你统一综合为结构化报告，**不要**简单拼接子智能体返回的原文。组织顺序：问题定义 → 证据整理 → 分析比较 → 结论与建议 → 来源。围绕「论证」而非「资料堆砌」，每个结论都要有证据支撑。
|
||||
|
||||
### 6. 停止准则
|
||||
信息饱和、或确认无法获取更多有效信息即停。明确标注证据缺口与不确定性，不臆断、不编造来源。
|
||||
|
||||
## 引用规范
|
||||
|
||||
- 报告中关键结论、数据、观点必须绑定来源。
|
||||
- 沿用 `<cite source="$URL" type="url">$INDEX</cite>` 标注，$INDEX 从 1 起递增，引用紧跟结论后、不单独成行。
|
||||
- 文末单列「来源」章节，逐条列出标题与 URL；引用用户附件/知识库时标明文件名或路径。
|
||||
|
||||
## 输出约束
|
||||
|
||||
- 最终交付的是一份可直接使用的报告，而不是「我打算怎么研究」。
|
||||
- 不要外泄中间推理过程、原始检索日志，也不要把待办清单原样输出成正文。
|
||||
- 报告语言与用户提问语言一致，使用正式、克制、可复核的书面表达。
|
||||
@@ -0,0 +1,82 @@
|
||||
---
|
||||
name: image-gen
|
||||
description: "在 Agent 沙盒中生成图片并保存到 outputs。当用户要求生成图片、海报、插画、文生图，或指定 Qwen-Image、其它兼容图片生成接口时使用此技能。"
|
||||
---
|
||||
|
||||
# 图片生成技能
|
||||
|
||||
当用户要求生成图片、海报、插画、文生图，或明确提到 Qwen-Image 时，使用此技能组织图片生成流程。
|
||||
|
||||
## 默认生成接口
|
||||
|
||||
默认使用 SiliconFlow 的 Qwen-Image 接口：
|
||||
|
||||
- Endpoint: `POST https://api.siliconflow.cn/v1/images/generations`
|
||||
- Model: `Qwen/Qwen-Image`
|
||||
- 默认参数：
|
||||
- `negative_prompt`: `""`
|
||||
- `num_inference_steps`: `20`
|
||||
- `guidance_scale`: `7.5`
|
||||
|
||||
调用外部接口时，必须在 Agent 沙盒执行环境中读取 `SILICONFLOW_API_KEY`。不要依赖后端进程环境变量。
|
||||
|
||||
## 操作流程
|
||||
|
||||
1. 明确用户要生成的图片内容、风格、尺寸或约束；信息不足但不影响生成时，使用合理默认值，不要反复追问。
|
||||
2. 使用可用的执行工具在沙盒中运行脚本，调用图片生成接口，传入用户需求整理后的 `prompt`，并按需传入 `negative_prompt`、`num_inference_steps`、`guidance_scale`。
|
||||
3. 从生成接口响应中读取图片地址，默认路径为 `images[0].url`。
|
||||
4. 在同一个沙盒脚本中用 `Authorization: Bearer $SILICONFLOW_API_KEY` 下载该图片地址；如果接口直接返回 base64，则直接解码保存。
|
||||
5. 将最终图片保存到 `/home/gem/user-data/outputs/` 下，例如 `/home/gem/user-data/outputs/generated-image.png`。
|
||||
6. 调用 `present_artifacts`，传入保存后的 outputs 虚拟路径，让前端展示图片产物。
|
||||
7. 最终回复简要说明图片已生成，不要把外部临时 URL 当作最终结果展示。
|
||||
|
||||
## 脚本示例
|
||||
|
||||
可根据用户需求调整 prompt 和输出文件名：
|
||||
|
||||
```python
|
||||
import os
|
||||
import requests
|
||||
from pathlib import Path
|
||||
|
||||
api_key = os.environ["SILICONFLOW_API_KEY"]
|
||||
prompt = "根据用户需求整理后的图片提示词"
|
||||
|
||||
response = requests.post(
|
||||
"https://api.siliconflow.cn/v1/images/generations",
|
||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||
json={
|
||||
"model": "Qwen/Qwen-Image",
|
||||
"prompt": prompt,
|
||||
"negative_prompt": "",
|
||||
"num_inference_steps": 20,
|
||||
"guidance_scale": 7.5,
|
||||
},
|
||||
timeout=120,
|
||||
)
|
||||
response.raise_for_status()
|
||||
image_url = response.json()["images"][0]["url"]
|
||||
|
||||
image_response = requests.get(
|
||||
image_url,
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
timeout=120,
|
||||
)
|
||||
image_response.raise_for_status()
|
||||
|
||||
output_path = Path("/home/gem/user-data/outputs/generated-image.png")
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_bytes(image_response.content)
|
||||
print(output_path.as_posix())
|
||||
```
|
||||
|
||||
## 多模型扩展
|
||||
|
||||
如果用户指定其它图片生成模型或兼容接口，可以按该接口的协议先生成图片。只要最终拿到图片 bytes 或 base64，就保存到 `/home/gem/user-data/outputs/`，再调用 `present_artifacts` 展示。
|
||||
|
||||
## 关键约束
|
||||
|
||||
- 不要把外部生成接口返回的临时 URL 当作最终结果直接展示给用户。
|
||||
- 不要调用后端 MinIO 上传工具；图片生成和下载都应在沙盒内完成。
|
||||
- 如果 `SILICONFLOW_API_KEY` 缺失，应明确提示用户需要在 Agent 沙盒环境变量中配置。
|
||||
- 保存到 outputs 后必须调用 `present_artifacts`，否则前端不会自动展示生成图片。
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
name: knowledge-base
|
||||
slug: knowledge-base
|
||||
description: "使用 Yuxi 知识库进行检索、打开文档、文档内定位和查看思维导图。当用户需要基于已配置知识库回答问题、核验资料或引用文档内容时使用此技能。"
|
||||
---
|
||||
|
||||
# 知识库技能
|
||||
|
||||
当用户要求基于项目知识库、内部资料、上传入库文档或知识图谱相关内容回答问题时，使用此技能。
|
||||
|
||||
## 可用工具
|
||||
|
||||
- `list_kbs`：列出当前会话可访问且已启用的知识库。
|
||||
- `query_kb`：按 `kb_id` 在指定知识库中检索内容，返回 `file_id` 和相关片段。
|
||||
- `open_kb_document`：按 `kb_id` 和 `file_id` 打开文档原文窗口，适合查看更完整上下文。
|
||||
- `find_kb_document`：在已知文档内用关键词或正则定位段落。
|
||||
- `get_mindmap`：查看知识库思维导图结构。
|
||||
- `search_file`：按文件名关键词搜索知识库中的文件，支持指定知识库或跨知识库，返回文件列表与分页信息。
|
||||
|
||||
## 操作流程
|
||||
|
||||
1. 需要先确认当前会话有哪些知识库可用；不确定时调用 `list_kbs`。
|
||||
2. 针对用户问题选择最相关的知识库，使用 `query_kb` 检索。
|
||||
3. 如果检索片段不足以回答，使用返回的 `file_id` 调用 `open_kb_document` 查看上下文。
|
||||
4. 如果用户要求定位术语、指标、章节或原文证据，使用 `find_kb_document` 在候选文档内查找。
|
||||
5. 当用户关心知识库结构、文件分类或知识框架时，使用 `get_mindmap`。
|
||||
|
||||
## 关键约束
|
||||
|
||||
- 只能访问当前会话配置和用户权限允许的知识库。
|
||||
- 不要编造 `kb_id` 或 `file_id`；优先从 `list_kbs` 和 `query_kb` 的返回结果中获取。
|
||||
- 回答需要可追溯时，应说明依据来自哪个知识库、文件或检索片段。
|
||||
- Dify 等外部只读知识库可能只支持检索，不一定支持打开全文或文档内查找；遇到工具返回限制说明时，应如实告知用户。
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
name: mysql reporter
|
||||
slug: mysql-reporter
|
||||
description: "生成 MySQL 查询报表并生成可视化图表。当用户需要查询 MySQL 数据库并以报表形式展示结果时使用此技能，包括：统计销售数据、分析用户行为、生成业务报表、查询业务指标等。"
|
||||
---
|
||||
|
||||
# MySQL 报表技能
|
||||
|
||||
根据用户的指令，通过终端脚本访问 MySQL 数据库，并结合图表绘制工具构建 SQL 查询报告。
|
||||
|
||||
## 操作流程
|
||||
|
||||
1. 理解用户的指令，明确报表的需求和目标
|
||||
2. 通过 terminal 进入技能目录：`cd /home/gem/skills/mysql-reporter`
|
||||
3. 使用 `uv run scripts/list_tables.py` 查看可用表；如果脚本提示缺少 MySQL 配置，按“环境变量缺失处理”回复用户
|
||||
4. 必要时用 `uv run scripts/describe_table.py --table 表名` 查看表结构
|
||||
5. 生成正确且高效的只读 SQL，通过 `uv run scripts/query.py --sql "SQL语句" --timeout 60` 执行查询并获取结果
|
||||
6. 使用 Charts MCP 生成图表
|
||||
7. 将图表以 markdown 图片格式嵌入报表
|
||||
|
||||
## 环境变量缺失处理
|
||||
|
||||
脚本只读取 Agent 沙盒中的环境变量，不读取后端 `.env` 或 Docker Compose 变量。必填变量包括：
|
||||
|
||||
- `MYSQL_HOST`
|
||||
- `MYSQL_USER`
|
||||
- `MYSQL_PASSWORD`
|
||||
- `MYSQL_DATABASE`
|
||||
|
||||
可选变量包括：
|
||||
|
||||
- `MYSQL_PORT`：默认 `3306`
|
||||
- `MYSQL_DATABASE_DESCRIPTION`：数据库业务说明，用于辅助理解表和指标含义
|
||||
|
||||
如果执行脚本时出现 `MySQL configuration missing required key`，不要继续猜测连接信息或编造报表。应明确告诉用户：需要在个人设置中的「沙盒环境变量」里配置缺失的 `MYSQL_*` 变量；保存后仅对新建沙盒生效，需要重新发起任务或新建会话后再执行。
|
||||
|
||||
## 关键约束
|
||||
|
||||
- 生成的 SQL 查询必须正确且高效，避免全表扫描
|
||||
- MySQL 操作必须通过本技能 `scripts/` 下的 CLI 脚本执行，不要调用平台内置 MySQL tools
|
||||
- 不要在报表或错误说明中输出 `MYSQL_PASSWORD` 等敏感环境变量的值，只能说明缺少哪些变量名
|
||||
- 图表生成工具的返回结果不会默认渲染，必须在最终报表中以 `` 格式嵌入
|
||||
- 只返回报表相关的结论，不要返回原始 SQL 查询语句
|
||||
|
||||
## 允许的工具
|
||||
|
||||
- terminal：执行 `scripts/list_tables.py`、`scripts/describe_table.py`、`scripts/query.py`
|
||||
- Charts MCP：生成可视化图表
|
||||
- 网络检索工具：必要时补充背景信息
|
||||
@@ -0,0 +1,177 @@
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "pymysql>=1.1.0",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pymysql
|
||||
from pymysql import MySQLError
|
||||
from pymysql.cursors import DictCursor
|
||||
|
||||
|
||||
class MySQLConnectionError(Exception):
|
||||
"""MySQL 连接异常"""
|
||||
|
||||
|
||||
class MySQLSecurityChecker:
|
||||
"""MySQL 安全检查器"""
|
||||
|
||||
@classmethod
|
||||
def validate_table_name(cls, table_name: str) -> bool:
|
||||
"""验证表名的安全性"""
|
||||
if not table_name:
|
||||
return False
|
||||
|
||||
return bool(re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", table_name))
|
||||
|
||||
|
||||
def load_mysql_config() -> dict[str, Any]:
|
||||
config: dict[str, Any] = {
|
||||
"host": os.getenv("MYSQL_HOST"),
|
||||
"user": os.getenv("MYSQL_USER"),
|
||||
"password": os.getenv("MYSQL_PASSWORD"),
|
||||
"database": os.getenv("MYSQL_DATABASE"),
|
||||
"port": int(os.getenv("MYSQL_PORT") or "3306"),
|
||||
"charset": "utf8mb4",
|
||||
"description": os.getenv("MYSQL_DATABASE_DESCRIPTION") or "默认 MySQL 数据库",
|
||||
}
|
||||
|
||||
required_keys = ["host", "user", "password", "database"]
|
||||
for key in required_keys:
|
||||
if not config[key]:
|
||||
raise MySQLConnectionError(
|
||||
f"MySQL configuration missing required key: {key}, please check your environment variables."
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def create_connection(config: dict[str, Any]) -> pymysql.Connection:
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return pymysql.connect(
|
||||
host=config["host"],
|
||||
user=config["user"],
|
||||
password=config["password"],
|
||||
database=config["database"],
|
||||
port=config["port"],
|
||||
charset=config.get("charset", "utf8mb4"),
|
||||
cursorclass=DictCursor,
|
||||
connect_timeout=10,
|
||||
read_timeout=60,
|
||||
write_timeout=30,
|
||||
autocommit=True,
|
||||
)
|
||||
except MySQLError as exc:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(2**attempt)
|
||||
continue
|
||||
raise ConnectionError(f"MySQL connection failed: {exc}") from exc
|
||||
|
||||
raise ConnectionError("MySQL connection failed")
|
||||
|
||||
|
||||
def describe_table(table_name: str) -> str:
|
||||
if not MySQLSecurityChecker.validate_table_name(table_name):
|
||||
raise ValueError("表名包含非法字符，请检查表名")
|
||||
|
||||
config = load_mysql_config()
|
||||
connection = create_connection(config)
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(f"DESCRIBE `{table_name}`")
|
||||
columns = cursor.fetchall()
|
||||
|
||||
if not columns:
|
||||
return f"表 {table_name} 不存在或没有字段"
|
||||
|
||||
column_comments: dict[str, str] = {}
|
||||
try:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT COLUMN_NAME, COLUMN_COMMENT
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_NAME = %s AND TABLE_SCHEMA = %s
|
||||
""",
|
||||
(table_name, config["database"]),
|
||||
)
|
||||
comment_rows = cursor.fetchall()
|
||||
for row in comment_rows:
|
||||
column_name = row.get("COLUMN_NAME")
|
||||
if column_name:
|
||||
column_comments[column_name] = row.get("COLUMN_COMMENT") or ""
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
result = f"表 `{table_name}` 的结构:\n\n"
|
||||
result += "字段名\t\t类型\t\tNULL\t键\t默认值\t\t额外\t备注\n"
|
||||
result += "-" * 80 + "\n"
|
||||
|
||||
for col in columns:
|
||||
field = col["Field"] or ""
|
||||
type_str = col["Type"] or ""
|
||||
null_str = col["Null"] or ""
|
||||
key_str = col["Key"] or ""
|
||||
default_str = col.get("Default") or ""
|
||||
extra_str = col.get("Extra") or ""
|
||||
comment_str = column_comments.get(field, "")
|
||||
|
||||
result += (
|
||||
f"{field:<16}\t{type_str:<16}\t{null_str:<8}\t{key_str:<4}\t"
|
||||
f"{default_str:<16}\t{extra_str:<16}\t{comment_str}\n"
|
||||
)
|
||||
|
||||
try:
|
||||
cursor.execute(f"SHOW INDEX FROM `{table_name}`")
|
||||
indexes = cursor.fetchall()
|
||||
|
||||
if indexes:
|
||||
result += "\n索引信息:\n"
|
||||
index_dict: dict[str, list[str]] = {}
|
||||
for idx in indexes:
|
||||
key_name = idx["Key_name"]
|
||||
if key_name not in index_dict:
|
||||
index_dict[key_name] = []
|
||||
index_dict[key_name].append(idx["Column_name"])
|
||||
|
||||
for key_name, index_columns in index_dict.items():
|
||||
result += f"- {key_name}: {', '.join(index_columns)}\n"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="描述 MySQL 表结构")
|
||||
parser.add_argument("--table", required=True, help="要查询结构的表名")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
print(describe_table(args.table))
|
||||
return 0
|
||||
except ValueError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 1
|
||||
except Exception as exc:
|
||||
print(f"获取表 {args.table} 结构失败: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,112 @@
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "pymysql>=1.1.0",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pymysql
|
||||
from pymysql import MySQLError
|
||||
from pymysql.cursors import DictCursor
|
||||
|
||||
|
||||
class MySQLConnectionError(Exception):
|
||||
"""MySQL 连接异常"""
|
||||
|
||||
|
||||
def load_mysql_config() -> dict[str, Any]:
|
||||
config: dict[str, Any] = {
|
||||
"host": os.getenv("MYSQL_HOST"),
|
||||
"user": os.getenv("MYSQL_USER"),
|
||||
"password": os.getenv("MYSQL_PASSWORD"),
|
||||
"database": os.getenv("MYSQL_DATABASE"),
|
||||
"port": int(os.getenv("MYSQL_PORT") or "3306"),
|
||||
"charset": "utf8mb4",
|
||||
"description": os.getenv("MYSQL_DATABASE_DESCRIPTION") or "默认 MySQL 数据库",
|
||||
}
|
||||
|
||||
required_keys = ["host", "user", "password", "database"]
|
||||
for key in required_keys:
|
||||
if not config[key]:
|
||||
raise MySQLConnectionError(
|
||||
f"MySQL configuration missing required key: {key}, please check your environment variables."
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def create_connection(config: dict[str, Any]) -> pymysql.Connection:
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return pymysql.connect(
|
||||
host=config["host"],
|
||||
user=config["user"],
|
||||
password=config["password"],
|
||||
database=config["database"],
|
||||
port=config["port"],
|
||||
charset=config.get("charset", "utf8mb4"),
|
||||
cursorclass=DictCursor,
|
||||
connect_timeout=10,
|
||||
read_timeout=60,
|
||||
write_timeout=30,
|
||||
autocommit=True,
|
||||
)
|
||||
except MySQLError as exc:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(2**attempt)
|
||||
continue
|
||||
raise ConnectionError(f"MySQL connection failed: {exc}") from exc
|
||||
|
||||
raise ConnectionError("MySQL connection failed")
|
||||
|
||||
|
||||
def list_tables() -> str:
|
||||
config = load_mysql_config()
|
||||
connection = create_connection(config)
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SHOW TABLES")
|
||||
tables = cursor.fetchall()
|
||||
|
||||
if not tables:
|
||||
return "数据库中没有找到任何表"
|
||||
|
||||
table_names = []
|
||||
for table in tables:
|
||||
table_name = list(table.values())[0]
|
||||
table_names.append(table_name)
|
||||
|
||||
all_table_names = "\n".join(table_names)
|
||||
result = f"数据库中的表:\n{all_table_names}"
|
||||
if db_note := config.get("description"):
|
||||
result = f"数据库说明: {db_note}\n\n" + result
|
||||
return result
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="列出当前 MySQL 数据库中的所有表")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parse_args()
|
||||
try:
|
||||
print(list_tables())
|
||||
return 0
|
||||
except Exception as exc:
|
||||
print(f"获取表名失败: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,299 @@
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "pymysql>=1.1.0",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pymysql
|
||||
from pymysql import MySQLError
|
||||
from pymysql.cursors import DictCursor
|
||||
|
||||
|
||||
class MySQLConnectionError(Exception):
|
||||
"""MySQL 连接异常"""
|
||||
|
||||
|
||||
class QueryTimeoutError(Exception):
|
||||
"""查询超时异常"""
|
||||
|
||||
|
||||
class MySQLSecurityChecker:
|
||||
"""MySQL 安全检查器"""
|
||||
|
||||
ALLOWED_OPERATIONS = {"SELECT", "SHOW", "DESCRIBE", "EXPLAIN"}
|
||||
|
||||
DANGEROUS_KEYWORDS = {
|
||||
"DROP",
|
||||
"DELETE",
|
||||
"UPDATE",
|
||||
"INSERT",
|
||||
"CREATE",
|
||||
"ALTER",
|
||||
"TRUNCATE",
|
||||
"REPLACE",
|
||||
"LOAD",
|
||||
"GRANT",
|
||||
"REVOKE",
|
||||
"SET",
|
||||
"COMMIT",
|
||||
"ROLLBACK",
|
||||
"UNLOCK",
|
||||
"KILL",
|
||||
"SHUTDOWN",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def validate_sql(cls, sql: str) -> bool:
|
||||
"""验证SQL语句的安全性"""
|
||||
if not sql:
|
||||
return False
|
||||
|
||||
sql_clean = re.sub(r"--.*$", "", sql, flags=re.MULTILINE)
|
||||
sql_clean = re.sub(r"/\*.*?\*/", "", sql_clean, flags=re.DOTALL)
|
||||
sql_upper = sql_clean.strip().upper()
|
||||
sql_without_trailing_semicolon = sql_upper.rstrip()
|
||||
if sql_without_trailing_semicolon.endswith(";"):
|
||||
sql_without_trailing_semicolon = sql_without_trailing_semicolon[:-1].rstrip()
|
||||
if ";" in sql_without_trailing_semicolon:
|
||||
return False
|
||||
|
||||
if not any(sql_upper.startswith(op) for op in cls.ALLOWED_OPERATIONS):
|
||||
return False
|
||||
|
||||
first_word_match = re.match(r"^\s*(\w+)", sql_upper)
|
||||
first_word = first_word_match.group(1) if first_word_match else ""
|
||||
if first_word in cls.DANGEROUS_KEYWORDS:
|
||||
return False
|
||||
|
||||
sql_injection_patterns = [
|
||||
r"\bor\s+1\s*=\s*1\b",
|
||||
r"\bunion\s+select\b",
|
||||
r"\bexec\s*\(",
|
||||
r"\bxp_cmdshell\b",
|
||||
r"\bsleep\s*\(",
|
||||
r"\bbenchmark\s*\(",
|
||||
r"\bwaitfor\s+delay\b",
|
||||
]
|
||||
sql_injection_patterns.extend(rf"\b;\s*{re.escape(keyword)}\b" for keyword in cls.DANGEROUS_KEYWORDS)
|
||||
|
||||
for pattern in sql_injection_patterns:
|
||||
if re.search(pattern, sql_upper, re.IGNORECASE):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def validate_timeout(cls, timeout: int) -> bool:
|
||||
"""验证timeout参数"""
|
||||
return isinstance(timeout, int) and 1 <= timeout <= 600
|
||||
|
||||
|
||||
def load_mysql_config() -> dict[str, Any]:
|
||||
config: dict[str, Any] = {
|
||||
"host": os.getenv("MYSQL_HOST"),
|
||||
"user": os.getenv("MYSQL_USER"),
|
||||
"password": os.getenv("MYSQL_PASSWORD"),
|
||||
"database": os.getenv("MYSQL_DATABASE"),
|
||||
"port": int(os.getenv("MYSQL_PORT") or "3306"),
|
||||
"charset": "utf8mb4",
|
||||
"description": os.getenv("MYSQL_DATABASE_DESCRIPTION") or "默认 MySQL 数据库",
|
||||
}
|
||||
|
||||
required_keys = ["host", "user", "password", "database"]
|
||||
for key in required_keys:
|
||||
if not config[key]:
|
||||
raise MySQLConnectionError(
|
||||
f"MySQL configuration missing required key: {key}, please check your environment variables."
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def create_connection(config: dict[str, Any]) -> pymysql.Connection:
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return pymysql.connect(
|
||||
host=config["host"],
|
||||
user=config["user"],
|
||||
password=config["password"],
|
||||
database=config["database"],
|
||||
port=config["port"],
|
||||
charset=config.get("charset", "utf8mb4"),
|
||||
cursorclass=DictCursor,
|
||||
connect_timeout=10,
|
||||
read_timeout=60,
|
||||
write_timeout=30,
|
||||
autocommit=True,
|
||||
)
|
||||
except MySQLError as exc:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(2**attempt)
|
||||
continue
|
||||
raise ConnectionError(f"MySQL connection failed: {exc}") from exc
|
||||
|
||||
raise ConnectionError("MySQL connection failed")
|
||||
|
||||
|
||||
def execute_query_with_timeout(
|
||||
connection: pymysql.Connection, sql: str, params: tuple | None = None, timeout: int = 10
|
||||
):
|
||||
"""使用线程池实现超时控制，避免信号导致的生成器问题"""
|
||||
|
||||
def query_worker():
|
||||
cursor = connection.cursor(DictCursor)
|
||||
try:
|
||||
if params is None:
|
||||
cursor.execute(sql)
|
||||
else:
|
||||
cursor.execute(sql, params)
|
||||
return cursor.fetchall()
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||||
future = executor.submit(query_worker)
|
||||
try:
|
||||
return future.result(timeout=timeout)
|
||||
except concurrent.futures.TimeoutError as exc:
|
||||
future.cancel()
|
||||
try:
|
||||
connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
raise QueryTimeoutError(f"Query timeout after {timeout} seconds") from exc
|
||||
finally:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
|
||||
def limit_result_size(result: list, max_chars: int = 10000) -> list:
|
||||
"""限制结果大小"""
|
||||
if not result:
|
||||
return result
|
||||
|
||||
result_str = str(result)
|
||||
if len(result_str) > max_chars:
|
||||
limited_result = []
|
||||
current_chars = 0
|
||||
for row in result:
|
||||
row_str = str(row)
|
||||
if current_chars + len(row_str) > max_chars:
|
||||
break
|
||||
limited_result.append(row)
|
||||
current_chars += len(row_str)
|
||||
return limited_result
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def format_query_result(result: list[dict[str, Any]]) -> str:
|
||||
if not result:
|
||||
return "查询执行成功，但没有返回任何结果"
|
||||
|
||||
limited_result = limit_result_size(result, max_chars=10000)
|
||||
|
||||
if len(limited_result) < len(result):
|
||||
warning = f"\n\n⚠️ 警告: 查询结果过大，只显示了前 {len(limited_result)} 行（共 {len(result)} 行）。\n"
|
||||
warning += "建议使用更精确的查询条件或使用LIMIT子句来减少返回的数据量。"
|
||||
else:
|
||||
warning = ""
|
||||
|
||||
if limited_result:
|
||||
columns = list(limited_result[0].keys())
|
||||
|
||||
col_widths = {}
|
||||
for col in columns:
|
||||
col_widths[col] = max(len(str(col)), max(len(str(row.get(col, ""))) for row in limited_result))
|
||||
col_widths[col] = min(col_widths[col], 50)
|
||||
|
||||
header = "| " + " | ".join(f"{col:<{col_widths[col]}}" for col in columns) + " |"
|
||||
separator = "|" + "|".join("-" * (col_widths[col] + 2) for col in columns) + "|"
|
||||
|
||||
rows = []
|
||||
for row in limited_result:
|
||||
row_str = "| " + " | ".join(f"{str(row.get(col, '')):<{col_widths[col]}}" for col in columns) + " |"
|
||||
rows.append(row_str)
|
||||
|
||||
result_str = f"查询结果（共 {len(limited_result)} 行）:\n\n"
|
||||
result_str += header + "\n" + separator + "\n"
|
||||
result_str += "\n".join(rows[:50])
|
||||
|
||||
if len(rows) > 50:
|
||||
result_str += f"\n\n... 还有 {len(rows) - 50} 行未显示 ..."
|
||||
|
||||
result_str += warning
|
||||
return result_str
|
||||
|
||||
return "查询执行成功，但返回数据为空"
|
||||
|
||||
|
||||
def run_query(sql: str, timeout: int) -> str:
|
||||
if not MySQLSecurityChecker.validate_sql(sql):
|
||||
raise ValueError("SQL语句包含不安全的操作或可能的注入攻击，请检查SQL语句")
|
||||
|
||||
if not MySQLSecurityChecker.validate_timeout(timeout):
|
||||
raise ValueError("timeout参数必须在1-600之间")
|
||||
|
||||
config = load_mysql_config()
|
||||
connection = create_connection(config)
|
||||
try:
|
||||
result = execute_query_with_timeout(connection, sql, timeout=timeout or 60)
|
||||
return format_query_result(result)
|
||||
finally:
|
||||
if connection.open:
|
||||
connection.close()
|
||||
|
||||
|
||||
def build_query_error(exc: Exception, sql: str) -> str:
|
||||
error_msg = f"SQL查询执行失败: {exc}\n\n{sql}"
|
||||
|
||||
if "timeout" in str(exc).lower():
|
||||
error_msg += "\n\n💡 建议：查询超时了，请尝试以下方法：\n"
|
||||
error_msg += "1. 减少查询的数据量（使用WHERE条件过滤）\n"
|
||||
error_msg += "2. 使用LIMIT子句限制返回行数\n"
|
||||
error_msg += "3. 增加timeout参数值（最大600秒）"
|
||||
elif "table" in str(exc).lower() and "doesn't exist" in str(exc).lower():
|
||||
error_msg += "\n\n💡 建议：表不存在，请使用 scripts/list_tables.py 查看可用的表名"
|
||||
elif "column" in str(exc).lower() and "doesn't exist" in str(exc).lower():
|
||||
error_msg += "\n\n💡 建议：列不存在，请使用 scripts/describe_table.py 查看表结构"
|
||||
elif "not enough arguments for format string" in str(exc).lower():
|
||||
error_msg += (
|
||||
"\n\n💡 建议：SQL 中的百分号 (%) 被当作参数占位符使用。"
|
||||
" 如需匹配包含百分号的文本，请将百分号写成双百分号 (%%) 或使用参数化查询。"
|
||||
)
|
||||
|
||||
return error_msg
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="执行只读 MySQL SQL 查询")
|
||||
parser.add_argument("--sql", required=True, help="要执行的SQL查询语句")
|
||||
parser.add_argument("--timeout", type=int, default=60, help="查询超时时间（秒），默认60秒，最大600秒")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
print(run_query(args.sql, args.timeout))
|
||||
return 0
|
||||
except ValueError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 1
|
||||
except Exception as exc:
|
||||
print(build_query_error(exc, args.sql), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,380 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from yuxi.agents.skills.service import import_skill_dir, is_valid_skill_slug
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from yuxi.storage.postgres.models_business import Skill
|
||||
|
||||
ANSI_ESCAPE_RE = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
|
||||
CONTROL_SEQUENCE_RE = re.compile(r"\x1B\][^\x07]*(?:\x07|\x1B\\)|\x1B[\(\)][A-Za-z0-9]")
|
||||
CLI_TIMEOUT_SECONDS = 300
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RemoteSkillsBatchPreparation:
|
||||
temp_home: str | None
|
||||
results: list[dict]
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
if self.temp_home:
|
||||
await asyncio.to_thread(shutil.rmtree, self.temp_home, ignore_errors=True)
|
||||
|
||||
|
||||
def _normalize_source(source: str) -> str:
|
||||
value = str(source or "").strip()
|
||||
if not value:
|
||||
raise ValueError("source 不能为空")
|
||||
if any(ch in value for ch in ("\n", "\r", "\x00")):
|
||||
raise ValueError("source 包含非法字符")
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_skill_name(skill: str) -> str:
|
||||
value = str(skill or "").strip()
|
||||
if not is_valid_skill_slug(value):
|
||||
raise ValueError("skill 名称不合法")
|
||||
return value
|
||||
|
||||
|
||||
def _clean_cli_output(output: str) -> list[str]:
|
||||
cleaned = ANSI_ESCAPE_RE.sub("", output or "")
|
||||
cleaned = CONTROL_SEQUENCE_RE.sub("", cleaned)
|
||||
cleaned = cleaned.replace("\r", "\n")
|
||||
normalized_lines: list[str] = []
|
||||
for line in cleaned.splitlines():
|
||||
stripped = line.strip()
|
||||
stripped = re.sub(r"^[│┌└◇◒◐◓◑■●]+\s*", "", stripped)
|
||||
normalized_lines.append(stripped.strip())
|
||||
return normalized_lines
|
||||
|
||||
|
||||
def _parse_available_skills(output: str) -> list[dict[str, str]]:
|
||||
lines = _clean_cli_output(output)
|
||||
items: list[dict[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
collecting = False
|
||||
|
||||
for idx, line in enumerate(lines):
|
||||
if not collecting:
|
||||
if "Available Skills" in line:
|
||||
collecting = True
|
||||
continue
|
||||
|
||||
if not line:
|
||||
continue
|
||||
if "Use --skill " in line:
|
||||
break
|
||||
if not is_valid_skill_slug(line):
|
||||
continue
|
||||
if line in seen:
|
||||
continue
|
||||
|
||||
description = ""
|
||||
next_index = idx + 1
|
||||
while next_index < len(lines):
|
||||
next_line = lines[next_index]
|
||||
next_index += 1
|
||||
if not next_line:
|
||||
continue
|
||||
if "Use --skill " in next_line:
|
||||
break
|
||||
if is_valid_skill_slug(next_line):
|
||||
break
|
||||
if next_line and next_line[0].isalpha():
|
||||
description = next_line
|
||||
else:
|
||||
continue
|
||||
break
|
||||
|
||||
seen.add(line)
|
||||
items.append({"name": line, "description": description})
|
||||
|
||||
return items
|
||||
|
||||
|
||||
async def _run_skills_cli(
|
||||
args: list[str],
|
||||
*,
|
||||
env: dict[str, str],
|
||||
cwd: str,
|
||||
) -> str:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=CLI_TIMEOUT_SECONDS)
|
||||
except TimeoutError:
|
||||
process.kill()
|
||||
await process.communicate()
|
||||
raise ValueError("skills CLI 执行超时") from None
|
||||
|
||||
output = (stdout or b"").decode("utf-8", errors="replace")
|
||||
error_output = (stderr or b"").decode("utf-8", errors="replace")
|
||||
combined = "\n".join(part for part in [output.strip(), error_output.strip()] if part)
|
||||
if process.returncode != 0:
|
||||
cleaned_lines = _clean_cli_output(combined)
|
||||
error_msg = "\n".join(line for line in cleaned_lines if line)[:500]
|
||||
raise ValueError(error_msg or "skills CLI 执行失败")
|
||||
return combined
|
||||
|
||||
|
||||
def _create_isolated_workdir() -> tuple[str, dict[str, str], str]:
|
||||
temp_home = tempfile.mkdtemp(prefix=".remote-skills-")
|
||||
env = os.environ.copy()
|
||||
env["HOME"] = temp_home
|
||||
workdir = str(Path(temp_home) / "workspace")
|
||||
Path(workdir).mkdir(parents=True, exist_ok=True)
|
||||
return temp_home, env, workdir
|
||||
|
||||
|
||||
async def list_remote_skills(source: str) -> list[dict[str, str]]:
|
||||
normalized_source = _normalize_source(source)
|
||||
|
||||
temp_home, env, workdir = _create_isolated_workdir()
|
||||
try:
|
||||
output = await _run_skills_cli(
|
||||
["npx", "-y", "skills", "add", normalized_source, "--list"],
|
||||
env=env,
|
||||
cwd=workdir,
|
||||
)
|
||||
finally:
|
||||
await asyncio.to_thread(shutil.rmtree, temp_home, ignore_errors=True)
|
||||
|
||||
skills = _parse_available_skills(output)
|
||||
if not skills:
|
||||
raise ValueError("未发现可安装的 skills")
|
||||
return skills
|
||||
|
||||
|
||||
async def install_remote_skill(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source: str,
|
||||
skill: str,
|
||||
created_by: str | None,
|
||||
) -> Skill:
|
||||
normalized_source = _normalize_source(source)
|
||||
normalized_skill = _normalize_skill_name(skill)
|
||||
|
||||
temp_home, env, workdir = _create_isolated_workdir()
|
||||
try:
|
||||
available_skills = _parse_available_skills(
|
||||
await _run_skills_cli(
|
||||
["npx", "-y", "skills", "add", normalized_source, "--list"],
|
||||
env=env,
|
||||
cwd=workdir,
|
||||
)
|
||||
)
|
||||
available_names = {item["name"] for item in available_skills}
|
||||
if normalized_skill not in available_names:
|
||||
raise ValueError(f"远程仓库中不存在 skill: {normalized_skill}")
|
||||
|
||||
await _run_skills_cli(
|
||||
[
|
||||
"npx",
|
||||
"-y",
|
||||
"skills",
|
||||
"add",
|
||||
normalized_source,
|
||||
"--skill",
|
||||
normalized_skill,
|
||||
"-g",
|
||||
"-y",
|
||||
"--copy",
|
||||
],
|
||||
env=env,
|
||||
cwd=workdir,
|
||||
)
|
||||
|
||||
skills_dir = Path(temp_home).resolve() / ".agents" / "skills"
|
||||
installed_dir = _find_skill_dir(skills_dir, normalized_skill)
|
||||
if installed_dir is None:
|
||||
raise ValueError("skills CLI 未生成预期的技能目录")
|
||||
|
||||
return await import_skill_dir(
|
||||
db,
|
||||
source_dir=installed_dir,
|
||||
created_by=created_by,
|
||||
)
|
||||
finally:
|
||||
await asyncio.to_thread(shutil.rmtree, temp_home, ignore_errors=True)
|
||||
|
||||
|
||||
async def install_remote_skills_batch(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source: str,
|
||||
skills: list[str],
|
||||
created_by: str | None,
|
||||
) -> list[dict]:
|
||||
"""批量从同一个远程仓库安装多个 skills（仅一次克隆）。
|
||||
|
||||
Args:
|
||||
db: 数据库会话。
|
||||
source: 远程仓库来源，如 ``owner/repo`` 或 GitHub URL。
|
||||
skills: 需要安装的 skill 名称列表。
|
||||
created_by: 操作者标识。
|
||||
|
||||
Returns:
|
||||
每个 skill 的安装结果列表，顺序与请求一致: ``[{slug, success, error?}, ...]``
|
||||
"""
|
||||
preparation = await prepare_remote_skills_batch(source=source, skills=skills)
|
||||
try:
|
||||
results = preparation.results
|
||||
for index, result in enumerate(results):
|
||||
if not result.get("success"):
|
||||
continue
|
||||
|
||||
source_dir = result.get("source_dir")
|
||||
try:
|
||||
item = await import_skill_dir(
|
||||
db,
|
||||
source_dir=source_dir,
|
||||
created_by=created_by,
|
||||
)
|
||||
results[index] = {"slug": item.slug, "success": True}
|
||||
except Exception as e:
|
||||
if hasattr(db, "rollback"):
|
||||
await db.rollback()
|
||||
results[index] = {"slug": result["slug"], "success": False, "error": str(e)}
|
||||
|
||||
return results
|
||||
finally:
|
||||
await preparation.cleanup()
|
||||
|
||||
|
||||
async def prepare_remote_skills_batch(
|
||||
*,
|
||||
source: str,
|
||||
skills: list[str],
|
||||
) -> RemoteSkillsBatchPreparation:
|
||||
"""批量从远程仓库拉取 skill 目录，但不写数据库。"""
|
||||
normalized_source = _normalize_source(source)
|
||||
if not skills:
|
||||
raise ValueError("skills 列表不能为空")
|
||||
|
||||
# 预分配结果数组（按请求顺序），校验非法名并记录失败
|
||||
results: list[dict] = [{"slug": "", "success": False, "error": "unset"} for _ in range(len(skills))]
|
||||
normalized_skills: list[str] = []
|
||||
valid_indices: list[int] = []
|
||||
for i, skill in enumerate(skills):
|
||||
try:
|
||||
normalized_skills.append(_normalize_skill_name(skill))
|
||||
valid_indices.append(i)
|
||||
except ValueError as e:
|
||||
results[i] = {"slug": skill, "success": False, "error": str(e)}
|
||||
|
||||
if not normalized_skills:
|
||||
return RemoteSkillsBatchPreparation(temp_home=None, results=results)
|
||||
|
||||
temp_home, env, workdir = _create_isolated_workdir()
|
||||
try:
|
||||
skill_args: list[str] = []
|
||||
for name in normalized_skills:
|
||||
skill_args.extend(["--skill", name])
|
||||
|
||||
cli_failed = False
|
||||
try:
|
||||
await _run_skills_cli(
|
||||
[
|
||||
"npx",
|
||||
"-y",
|
||||
"skills",
|
||||
"add",
|
||||
normalized_source,
|
||||
*skill_args,
|
||||
"-g",
|
||||
"-y",
|
||||
"--copy",
|
||||
],
|
||||
env=env,
|
||||
cwd=workdir,
|
||||
)
|
||||
except ValueError:
|
||||
# CLI 对不匹配的 skill 会退出码非零，但已安装的目录仍在
|
||||
cli_failed = True
|
||||
|
||||
skills_dir = Path(temp_home).resolve() / ".agents" / "skills"
|
||||
for original_index, name in zip(valid_indices, normalized_skills):
|
||||
installed_dir = _find_skill_dir(skills_dir, name)
|
||||
if installed_dir is None:
|
||||
error_msg = "CLI 安装失败" if cli_failed else "skills CLI 未生成预期的技能目录"
|
||||
results[original_index] = {"slug": name, "success": False, "error": error_msg}
|
||||
else:
|
||||
results[original_index] = {"slug": name, "success": True, "source_dir": installed_dir}
|
||||
|
||||
return RemoteSkillsBatchPreparation(temp_home=temp_home, results=results)
|
||||
except Exception:
|
||||
await asyncio.to_thread(shutil.rmtree, temp_home, ignore_errors=True)
|
||||
raise
|
||||
|
||||
|
||||
def _find_skill_dir(skills_dir: Path, name: str) -> Path | None:
|
||||
"""在 skills 安装目录下按名称查找 skill 子目录。"""
|
||||
if not skills_dir.is_dir():
|
||||
return None
|
||||
for candidate in skills_dir.iterdir():
|
||||
if candidate.name == name and candidate.is_dir():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _parse_search_skills(output: str) -> list[dict[str, str]]:
|
||||
"""解析 npx skills find 命令的输出。"""
|
||||
lines = _clean_cli_output(output)
|
||||
results: list[dict[str, str]] = []
|
||||
# 匹配形如 "owner/repo@skill-name [installs]"
|
||||
# 例如：vercel-labs/agent-skills@web-design-guidelines 339.3K installs
|
||||
pattern = re.compile(r"^([a-zA-Z0-9_\-\.]+/[a-zA-Z0-9_\-\.]+)\@([a-zA-Z0-9_\-\.]+)(?:\s+(.*))?$")
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
match = pattern.match(line)
|
||||
if match:
|
||||
source, name, extra = match.groups()
|
||||
installs = extra.strip() if extra else ""
|
||||
results.append(
|
||||
{
|
||||
"source": source,
|
||||
"name": name,
|
||||
"installs": installs,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def search_remote_skills(query: str) -> list[dict[str, str]]:
|
||||
"""使用 npx skills find <query> 搜索远程 skills。"""
|
||||
query_val = str(query or "").strip()
|
||||
if not query_val:
|
||||
return []
|
||||
if any(ch in query_val for ch in ("\n", "\r", "\x00")):
|
||||
raise ValueError("搜索关键字包含非法字符")
|
||||
|
||||
temp_home, env, workdir = _create_isolated_workdir()
|
||||
try:
|
||||
output = await _run_skills_cli(
|
||||
["npx", "-y", "skills", "find", query_val],
|
||||
env=env,
|
||||
cwd=workdir,
|
||||
)
|
||||
finally:
|
||||
await asyncio.to_thread(shutil.rmtree, temp_home, ignore_errors=True)
|
||||
|
||||
return _parse_search_skills(output)
|
||||
@@ -0,0 +1,154 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from yuxi.storage.postgres.models_business import Skill
|
||||
from yuxi.utils.datetime_utils import utc_now_naive
|
||||
|
||||
|
||||
class SkillRepository:
|
||||
def __init__(self, db_session: AsyncSession):
|
||||
self.db = db_session
|
||||
|
||||
async def list_all(self) -> list[Skill]:
|
||||
result = await self.db.execute(select(Skill).order_by(Skill.updated_at.desc(), Skill.id.desc()))
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def list_enabled(self) -> list[Skill]:
|
||||
result = await self.db.execute(
|
||||
select(Skill).where(Skill.enabled.is_(True)).order_by(Skill.updated_at.desc(), Skill.id.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def list_by_slugs(self, slugs: list[str]) -> list[Skill]:
|
||||
normalized = [slug for slug in dict.fromkeys(slugs) if isinstance(slug, str) and slug]
|
||||
if not normalized:
|
||||
return []
|
||||
result = await self.db.execute(select(Skill).where(Skill.slug.in_(normalized)))
|
||||
items = list(result.scalars().all())
|
||||
item_map = {item.slug: item for item in items}
|
||||
return [item_map[slug] for slug in normalized if slug in item_map]
|
||||
|
||||
async def get_by_slug(self, slug: str, *, for_update: bool = False) -> Skill | None:
|
||||
stmt = select(Skill).where(Skill.slug == slug)
|
||||
if for_update:
|
||||
stmt = stmt.with_for_update()
|
||||
result = await self.db.execute(stmt)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def exists_slug(self, slug: str) -> bool:
|
||||
return (await self.get_by_slug(slug)) is not None
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
slug: str,
|
||||
name: str,
|
||||
description: str,
|
||||
source_type: str,
|
||||
tool_dependencies: list[str] | None,
|
||||
mcp_dependencies: list[str] | None,
|
||||
skill_dependencies: list[str] | None,
|
||||
dir_path: str,
|
||||
share_config: dict,
|
||||
enabled: bool = True,
|
||||
version: str | None = None,
|
||||
content_hash: str | None = None,
|
||||
created_by: str | None,
|
||||
) -> Skill:
|
||||
now = utc_now_naive()
|
||||
item = Skill(
|
||||
slug=slug,
|
||||
name=name,
|
||||
description=description,
|
||||
source_type=source_type,
|
||||
tool_dependencies=tool_dependencies or [],
|
||||
mcp_dependencies=mcp_dependencies or [],
|
||||
skill_dependencies=skill_dependencies or [],
|
||||
dir_path=dir_path,
|
||||
version=version,
|
||||
content_hash=content_hash,
|
||||
share_config=share_config,
|
||||
enabled=enabled,
|
||||
created_by=created_by,
|
||||
updated_by=created_by,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
self.db.add(item)
|
||||
await self.db.commit()
|
||||
await self.db.refresh(item)
|
||||
return item
|
||||
|
||||
async def update_builtin_install(
|
||||
self,
|
||||
item: Skill,
|
||||
*,
|
||||
version: str,
|
||||
content_hash: str,
|
||||
updated_by: str | None,
|
||||
) -> Skill:
|
||||
item.version = version
|
||||
item.content_hash = content_hash
|
||||
item.source_type = "builtin"
|
||||
item.share_config = {"access_level": "global", "department_ids": [], "user_uids": []}
|
||||
item.updated_by = updated_by
|
||||
item.updated_at = utc_now_naive()
|
||||
await self.db.commit()
|
||||
await self.db.refresh(item)
|
||||
return item
|
||||
|
||||
async def update_dependencies(
|
||||
self,
|
||||
item: Skill,
|
||||
*,
|
||||
tool_dependencies: list[str],
|
||||
mcp_dependencies: list[str],
|
||||
skill_dependencies: list[str],
|
||||
updated_by: str | None,
|
||||
) -> Skill:
|
||||
item.tool_dependencies = tool_dependencies
|
||||
item.mcp_dependencies = mcp_dependencies
|
||||
item.skill_dependencies = skill_dependencies
|
||||
item.updated_by = updated_by
|
||||
item.updated_at = utc_now_naive()
|
||||
await self.db.commit()
|
||||
await self.db.refresh(item)
|
||||
return item
|
||||
|
||||
async def update_metadata(
|
||||
self,
|
||||
item: Skill,
|
||||
*,
|
||||
name: str,
|
||||
description: str,
|
||||
updated_by: str | None,
|
||||
) -> Skill:
|
||||
item.name = name
|
||||
item.description = description
|
||||
item.updated_by = updated_by
|
||||
item.updated_at = utc_now_naive()
|
||||
await self.db.commit()
|
||||
await self.db.refresh(item)
|
||||
return item
|
||||
|
||||
async def update_share_config(self, item: Skill, *, share_config: dict, updated_by: str | None) -> Skill:
|
||||
item.share_config = share_config
|
||||
item.updated_by = updated_by
|
||||
item.updated_at = utc_now_naive()
|
||||
await self.db.commit()
|
||||
await self.db.refresh(item)
|
||||
return item
|
||||
|
||||
async def update_enabled(self, item: Skill, *, enabled: bool, updated_by: str | None) -> Skill:
|
||||
item.enabled = enabled
|
||||
item.updated_by = updated_by
|
||||
item.updated_at = utc_now_naive()
|
||||
await self.db.commit()
|
||||
await self.db.refresh(item)
|
||||
return item
|
||||
|
||||
async def delete(self, item: Skill) -> None:
|
||||
await self.db.delete(item)
|
||||
await self.db.commit()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
"""Define the state structures for the agent."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, TypedDict
|
||||
|
||||
from langchain.agents import AgentState
|
||||
|
||||
|
||||
def merge_artifacts(existing: list[str] | None, new: list[str] | None) -> list[str]:
|
||||
"""Merge artifact file paths while preserving order and removing duplicates."""
|
||||
if existing is None:
|
||||
return new or []
|
||||
if new is None:
|
||||
return existing
|
||||
return list(dict.fromkeys(existing + new))
|
||||
|
||||
|
||||
class BaseState(AgentState):
|
||||
"""Shared state fields for Yuxi agents."""
|
||||
|
||||
artifacts: Annotated[list[str], merge_artifacts]
|
||||
|
||||
|
||||
class AgentStatePayload(TypedDict):
|
||||
"""Serialized agent state payload consumed by the frontend."""
|
||||
|
||||
todos: list
|
||||
files: dict
|
||||
artifacts: list[str]
|
||||
subagent_runs: list[dict]
|
||||
token_usage: dict | None
|
||||
@@ -0,0 +1,25 @@
|
||||
# toolkits 包
|
||||
# 触发各模块的 @tool 装饰器执行，自动注册工具
|
||||
from . import buildin, debug
|
||||
|
||||
# 工具获取函数
|
||||
from .kbs import get_common_kb_tools
|
||||
from .registry import (
|
||||
ToolExtraMetadata,
|
||||
get_all_extra_metadata,
|
||||
get_all_tool_instances,
|
||||
get_extra_metadata,
|
||||
tool,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"get_extra_metadata",
|
||||
"get_all_extra_metadata",
|
||||
"get_all_tool_instances",
|
||||
"ToolExtraMetadata",
|
||||
"tool",
|
||||
"get_common_kb_tools",
|
||||
# 触发各模块的 @tool 装饰器执行，自动注册工具
|
||||
"buildin",
|
||||
"debug",
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
# buildin 工具包
|
||||
from .install_skill import install_skill
|
||||
from .tools import ask_user_question, ocr_parse_file, present_artifacts
|
||||
|
||||
__all__ = [
|
||||
"ask_user_question",
|
||||
"install_skill",
|
||||
"ocr_parse_file",
|
||||
"present_artifacts",
|
||||
]
|
||||
@@ -0,0 +1,302 @@
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Annotated
|
||||
|
||||
from langchain.tools import InjectedToolCallId
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.prebuilt.tool_node import ToolRuntime
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from yuxi.agents.toolkits.registry import tool
|
||||
from yuxi.repositories.agent_repository import AgentRepository
|
||||
from yuxi.repositories.conversation_repository import ConversationRepository
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
SANDBOX_PATH_HINT = (
|
||||
"请使用 /home/gem/user-data/workspace/...、/home/gem/user-data/uploads/... 或 /home/gem/user-data/outputs/..."
|
||||
)
|
||||
MAX_SANDBOX_SKILL_FILES = 1000
|
||||
|
||||
|
||||
class InstallSkillInput(BaseModel):
|
||||
source: str = Field(
|
||||
description="Skill 来源，支持两种格式:\n"
|
||||
"1. Sandbox 路径: /home/gem/user-data/workspace/...、"
|
||||
"/home/gem/user-data/uploads/... 或 /home/gem/user-data/outputs/...（/ 开头）\n"
|
||||
"2. Git 仓库: owner/repo 或完整 GitHub URL"
|
||||
)
|
||||
skill_names: list[str] | None = Field(
|
||||
default=None, description="Git 安装时指定要安装的 skill slug 列表（至少一个）。Sandbox 路径安装时忽略此参数。"
|
||||
)
|
||||
|
||||
|
||||
def _personal_skill_share_config(uid: str) -> dict:
|
||||
return {"access_level": "user", "department_ids": [], "user_uids": [str(uid)]}
|
||||
|
||||
|
||||
def _collect_sandbox_file_paths(backend, remote_dir: str, file_paths: list[str] | None = None) -> list[str]:
|
||||
file_paths = file_paths if file_paths is not None else []
|
||||
result = backend.ls(remote_dir)
|
||||
if result.error:
|
||||
raise ValueError(result.error)
|
||||
entries = result.entries or []
|
||||
for entry in entries:
|
||||
path = entry["path"]
|
||||
if entry.get("is_dir"):
|
||||
_collect_sandbox_file_paths(backend, path, file_paths)
|
||||
else:
|
||||
if len(file_paths) >= MAX_SANDBOX_SKILL_FILES:
|
||||
raise ValueError(f"Skill 目录文件数超过限制（最多 {MAX_SANDBOX_SKILL_FILES} 个文件）")
|
||||
file_paths.append(path)
|
||||
return file_paths
|
||||
|
||||
|
||||
def _download_skill_dir(backend, remote_dir: str, local_dir: Path) -> None:
|
||||
"""递归通过沙盒 API 下载 skill 目录到本地。"""
|
||||
remote_root = PurePosixPath(remote_dir.rstrip("/"))
|
||||
file_paths = _collect_sandbox_file_paths(backend, remote_dir)
|
||||
if not file_paths:
|
||||
raise ValueError(f"沙盒路径 {remote_dir} 中未发现可下载文件")
|
||||
|
||||
responses = backend.download_files(file_paths)
|
||||
if len(responses) != len(file_paths):
|
||||
raise ValueError("沙盒文件下载结果数量不匹配")
|
||||
|
||||
for expected_path, response in zip(file_paths, responses):
|
||||
error = getattr(response, "error", None)
|
||||
content = getattr(response, "content", None)
|
||||
if error or content is None:
|
||||
raise ValueError(f"下载沙盒文件失败: {expected_path} ({error or 'empty_content'})")
|
||||
|
||||
pure_path = PurePosixPath(expected_path)
|
||||
try:
|
||||
relative_path = pure_path.relative_to(remote_root)
|
||||
except ValueError:
|
||||
relative_path = PurePosixPath(pure_path.name)
|
||||
|
||||
target_path = local_dir / Path(relative_path.as_posix())
|
||||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
target_path.write_bytes(content)
|
||||
|
||||
|
||||
def _prepare_skill_from_sandbox(sandbox_path: str, thread_id: str, uid: str, staging_root: Path) -> tuple[Path, str]:
|
||||
"""从 Sandbox 路径准备 skill 目录。返回 (本地目录, 原始 skill name)。"""
|
||||
from yuxi.agents.backends.sandbox import ProvisionerSandboxBackend, resolve_virtual_path
|
||||
from yuxi.agents.skills.service import (
|
||||
_parse_skill_markdown,
|
||||
is_valid_skill_slug,
|
||||
)
|
||||
|
||||
slug = PurePosixPath(sandbox_path.rstrip("/")).name
|
||||
if not is_valid_skill_slug(slug):
|
||||
raise ValueError(f"slug '{slug}' 不合法（仅允许小写字母、数字和连字符）")
|
||||
|
||||
if not sandbox_path.startswith("/home/gem/user-data/"):
|
||||
raise ValueError(f"不支持的沙盒路径: {sandbox_path}。{SANDBOX_PATH_HINT}")
|
||||
|
||||
staging = staging_root / slug
|
||||
|
||||
# 优先尝试共享卷路径（性能更好，无需走沙盒 API）。
|
||||
try:
|
||||
local_path = resolve_virtual_path(thread_id, sandbox_path, uid=uid)
|
||||
if (local_path / "SKILL.md").exists():
|
||||
shutil.copytree(local_path, staging)
|
||||
else:
|
||||
raise FileNotFoundError(f"{local_path} 中未找到 SKILL.md")
|
||||
except FileNotFoundError:
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
backend = ProvisionerSandboxBackend(thread_id=thread_id, uid=uid)
|
||||
_download_skill_dir(backend, sandbox_path, staging)
|
||||
if not (staging / "SKILL.md").exists():
|
||||
raise ValueError(f"沙盒路径 {sandbox_path} 中未找到 SKILL.md")
|
||||
|
||||
content = (staging / "SKILL.md").read_text(encoding="utf-8")
|
||||
parsed_name, _, _ = _parse_skill_markdown(content)
|
||||
return staging, parsed_name
|
||||
|
||||
|
||||
async def _enable_skills_in_current_config(db, thread_id: str, uid: str, skill_slugs: list[str]) -> bool:
|
||||
"""在当前会话绑定且当前用户拥有的 Agent 配置中启用新安装的 skill。"""
|
||||
conv_repo = ConversationRepository(db)
|
||||
conv = await conv_repo.get_conversation_by_thread_id(thread_id)
|
||||
if not conv or str(conv.uid) != str(uid):
|
||||
return False
|
||||
|
||||
agent_repo = AgentRepository(db)
|
||||
agent = await agent_repo.get_by_slug(conv.agent_id)
|
||||
if not agent or agent.created_by != str(uid):
|
||||
return False
|
||||
|
||||
config_json = dict(agent.config_json or {})
|
||||
context = dict(config_json.get("context") or {})
|
||||
skills = [item for item in context.get("skills") or [] if isinstance(item, str) and item.strip()]
|
||||
seen = set(skills)
|
||||
for slug in skill_slugs:
|
||||
if slug not in seen:
|
||||
skills.append(slug)
|
||||
seen.add(slug)
|
||||
context["skills"] = skills
|
||||
config_json["context"] = context
|
||||
await agent_repo.update(agent, config_json=config_json, updated_by=str(uid))
|
||||
return True
|
||||
|
||||
|
||||
async def _run_install_task(
|
||||
source: str,
|
||||
runtime: ToolRuntime,
|
||||
tool_call_id: str,
|
||||
skill_names: list[str] | None = None,
|
||||
) -> Command:
|
||||
"""执行异步安装任务的核心逻辑。"""
|
||||
runtime_context = getattr(runtime, "context", None)
|
||||
if getattr(runtime_context, "is_subagent_runtime", False):
|
||||
return Command(
|
||||
update={
|
||||
"messages": [
|
||||
ToolMessage(
|
||||
content="错误：install_skill 只能在主智能体中使用，子智能体无法安装 Skill",
|
||||
tool_call_id=tool_call_id,
|
||||
)
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
source = str(source or "").strip()
|
||||
uid = getattr(runtime_context, "uid", None)
|
||||
thread_id = getattr(runtime_context, "thread_id", None)
|
||||
|
||||
logger.info(f"install_skill called with uid={uid}, thread_id={thread_id}, source={source}")
|
||||
|
||||
if not uid or not thread_id:
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(content="错误：无法获取当前会话信息", tool_call_id=tool_call_id)]}
|
||||
)
|
||||
if not source:
|
||||
return Command(
|
||||
update={"messages": [ToolMessage(content="错误：Skill 来源不能为空", tool_call_id=tool_call_id)]}
|
||||
)
|
||||
|
||||
personal_share_config = _personal_skill_share_config(uid)
|
||||
|
||||
try:
|
||||
from yuxi.agents.skills.service import (
|
||||
import_skill_dir,
|
||||
normalize_string_list,
|
||||
sync_thread_readable_skills,
|
||||
)
|
||||
|
||||
installed_slugs: list[str] = []
|
||||
failed_items: list[dict] = []
|
||||
slug_warnings: list[str] = []
|
||||
config_success = True
|
||||
|
||||
if source.startswith("/"):
|
||||
with tempfile.TemporaryDirectory(prefix=".skill-install-") as tmp:
|
||||
source_dir, parsed_name = _prepare_skill_from_sandbox(source, thread_id, uid, Path(tmp))
|
||||
async with pg_manager.get_async_session_context() as db:
|
||||
item = await import_skill_dir(
|
||||
db,
|
||||
source_dir=source_dir,
|
||||
created_by=uid,
|
||||
share_config=personal_share_config,
|
||||
)
|
||||
installed_slugs = [item.slug]
|
||||
if item.slug != parsed_name:
|
||||
slug_warnings.append(f"⚠️ 技能 slug '{item.slug}' 已存在，已自动重命名安装")
|
||||
config_success = await _enable_skills_in_current_config(db, thread_id, uid, installed_slugs)
|
||||
else:
|
||||
_skill_names = skill_names or []
|
||||
if not _skill_names:
|
||||
return Command(
|
||||
update={
|
||||
"messages": [
|
||||
ToolMessage(
|
||||
content="❌ 错误: 从 Git 安装时必须通过 skill_names 指定技能名称",
|
||||
tool_call_id=tool_call_id,
|
||||
)
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
from yuxi.agents.skills.remote_install import prepare_remote_skills_batch
|
||||
|
||||
preparation = await prepare_remote_skills_batch(source=source, skills=_skill_names)
|
||||
try:
|
||||
async with pg_manager.get_async_session_context() as db:
|
||||
for result in preparation.results:
|
||||
if not result.get("success"):
|
||||
failed_items.append(result)
|
||||
continue
|
||||
|
||||
try:
|
||||
item = await import_skill_dir(
|
||||
db,
|
||||
source_dir=result["source_dir"],
|
||||
created_by=uid,
|
||||
share_config=personal_share_config,
|
||||
)
|
||||
installed_slugs.append(item.slug)
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
failed_items.append({"slug": result["slug"], "success": False, "error": str(e)})
|
||||
|
||||
config_success = True
|
||||
if installed_slugs:
|
||||
config_success = await _enable_skills_in_current_config(db, thread_id, uid, installed_slugs)
|
||||
finally:
|
||||
preparation.cleanup()
|
||||
|
||||
current_skills = normalize_string_list(getattr(runtime_context, "skills", None))
|
||||
sync_thread_readable_skills(thread_id, normalize_string_list(current_skills + installed_slugs))
|
||||
|
||||
lines = []
|
||||
if installed_slugs:
|
||||
lines.append(f"✅ 成功安装并激活技能: {', '.join(installed_slugs)}")
|
||||
for warning in slug_warnings:
|
||||
lines.append(warning)
|
||||
if failed_items:
|
||||
for item in failed_items:
|
||||
lines.append(f"❌ 安装失败 ({item['slug']}): {item.get('error', '未知错误')}")
|
||||
if not config_success:
|
||||
lines.append("⚠️ 已添加这个 Skill 拓展（仅当前会话生效，未写入当前 Agent 配置）")
|
||||
if not installed_slugs and not failed_items:
|
||||
lines.append("ℹ️ 未发现需要安装的技能")
|
||||
|
||||
return Command(
|
||||
update={
|
||||
"activated_skills": installed_slugs,
|
||||
"messages": [ToolMessage(content="\n".join(lines), tool_call_id=tool_call_id)],
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("install_skill 异常")
|
||||
return Command(
|
||||
update={
|
||||
"messages": [
|
||||
ToolMessage(
|
||||
content=f"❌ 安装异常: {str(e)}",
|
||||
tool_call_id=tool_call_id,
|
||||
)
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@tool(
|
||||
category="buildin",
|
||||
tags=["skill", "安装"],
|
||||
display_name="安装技能",
|
||||
args_schema=InstallSkillInput,
|
||||
)
|
||||
async def install_skill(
|
||||
source: str,
|
||||
skill_names: list[str] | None = None,
|
||||
runtime: ToolRuntime = None,
|
||||
tool_call_id: Annotated[str, InjectedToolCallId] = "",
|
||||
) -> Command:
|
||||
"""安装新的 Skill 到当前用户的私有空间，并在当前主智能体会话中激活。"""
|
||||
return await _run_install_task(source, runtime, tool_call_id, skill_names)
|
||||
@@ -0,0 +1,388 @@
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
from langchain.tools import InjectedToolCallId
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.prebuilt.tool_node import ToolRuntime
|
||||
from langgraph.types import Command, interrupt
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from yuxi.agents.toolkits.registry import ToolExtraMetadata, _all_tool_instances, _extra_registry, tool
|
||||
from yuxi.utils import logger
|
||||
from yuxi.utils.paths import (
|
||||
CONVERSATION_HISTORY_DIR_NAME,
|
||||
LARGE_TOOL_RESULTS_DIR_NAME,
|
||||
OUTPUTS_DIR_NAME,
|
||||
UPLOADS_DIR_NAME,
|
||||
VIRTUAL_PATH_OUTPUTS,
|
||||
WORKSPACE_DIR_NAME,
|
||||
)
|
||||
from yuxi.utils.question_utils import normalize_questions
|
||||
|
||||
# Lazy initialization for TavilySearch (only when API key is available)
|
||||
_tavily_search_instance = None
|
||||
|
||||
_PRESENT_ARTIFACTS_INTERNAL_DIR_NAMES = frozenset(
|
||||
{CONVERSATION_HISTORY_DIR_NAME, LARGE_TOOL_RESULTS_DIR_NAME, "large_tool_history"}
|
||||
)
|
||||
_OCR_PARSE_ALLOWED_DIRS = frozenset({WORKSPACE_DIR_NAME, UPLOADS_DIR_NAME, OUTPUTS_DIR_NAME})
|
||||
_OCR_OUTPUT_DIR_NAME = "ocr"
|
||||
_OCR_PREVIEW_LIMIT = 1200
|
||||
_SAFE_OUTPUT_STEM_RE = re.compile(r"[^A-Za-z0-9._\-\u4e00-\u9fff]+")
|
||||
|
||||
|
||||
def _create_tavily_search():
|
||||
"""Create and register TavilySearch tool with metadata."""
|
||||
global _tavily_search_instance
|
||||
if _tavily_search_instance is None:
|
||||
from langchain_tavily import TavilySearch
|
||||
|
||||
_tavily_search_instance = TavilySearch()
|
||||
|
||||
return _tavily_search_instance
|
||||
|
||||
|
||||
# 注册 TavilySearch 工具（延迟初始化）
|
||||
def _register_tavily_tool():
|
||||
"""Register TavilySearch tool with extra metadata."""
|
||||
tavily_instance = _create_tavily_search()
|
||||
# 手动注册到全局注册表
|
||||
_extra_registry["tavily_search"] = ToolExtraMetadata(
|
||||
category="buildin",
|
||||
tags=["搜索"],
|
||||
display_name="Tavily 网页搜索",
|
||||
)
|
||||
# 添加到工具实例列表
|
||||
_all_tool_instances.append(tavily_instance)
|
||||
|
||||
|
||||
# 模块加载时注册
|
||||
if os.getenv("TAVILY_API_KEY"):
|
||||
try:
|
||||
_register_tavily_tool()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to register TavilySearch tool: {e}")
|
||||
|
||||
|
||||
class PresentArtifactsInput(BaseModel):
|
||||
"""Expose artifact files to the frontend after the agent finishes."""
|
||||
|
||||
filepaths: list[str] = Field(
|
||||
description=f"需要展示给用户的文件绝对路径列表，只允许位于 {VIRTUAL_PATH_OUTPUTS} 下，且不能是内部运行文件"
|
||||
)
|
||||
|
||||
|
||||
def _normalize_presented_artifact_path(filepath: str, runtime: ToolRuntime) -> str:
|
||||
from yuxi.agents.backends.sandbox.paths import (
|
||||
VIRTUAL_PATH_PREFIX,
|
||||
ensure_thread_dirs,
|
||||
resolve_virtual_path,
|
||||
sandbox_outputs_dir,
|
||||
)
|
||||
|
||||
outputs_virtual_prefix = f"{VIRTUAL_PATH_PREFIX}/outputs"
|
||||
runtime_context = runtime.context
|
||||
thread_id = getattr(runtime_context, "file_thread_id", None) or getattr(runtime_context, "thread_id", None)
|
||||
if not thread_id:
|
||||
raise ValueError("当前运行时缺少 thread_id")
|
||||
uid = getattr(runtime_context, "uid", None)
|
||||
if not uid:
|
||||
raise ValueError("当前运行时缺少 uid")
|
||||
|
||||
ensure_thread_dirs(thread_id, str(uid))
|
||||
outputs_dir = sandbox_outputs_dir(thread_id).resolve()
|
||||
normalized_input = str(filepath or "").strip()
|
||||
if not normalized_input:
|
||||
raise ValueError("文件路径不能为空")
|
||||
|
||||
stripped = normalized_input.lstrip("/")
|
||||
virtual_prefix = VIRTUAL_PATH_PREFIX.lstrip("/")
|
||||
if stripped == virtual_prefix or stripped.startswith(f"{virtual_prefix}/"):
|
||||
actual_path = resolve_virtual_path(thread_id, normalized_input, uid=str(uid))
|
||||
else:
|
||||
actual_path = Path(normalized_input).expanduser().resolve()
|
||||
|
||||
if not actual_path.exists() or not actual_path.is_file():
|
||||
raise ValueError(f"文件不存在或不是普通文件: {normalized_input}")
|
||||
|
||||
try:
|
||||
relative_path = actual_path.relative_to(outputs_dir)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"只允许展示 {outputs_virtual_prefix}/ 下的文件: {normalized_input}") from exc
|
||||
|
||||
if relative_path.parts and relative_path.parts[0] in _PRESENT_ARTIFACTS_INTERNAL_DIR_NAMES:
|
||||
raise ValueError(f"不允许展示工具调用阶段文件: {outputs_virtual_prefix}/{relative_path.as_posix()}")
|
||||
|
||||
return f"{outputs_virtual_prefix}/{relative_path.as_posix()}"
|
||||
|
||||
|
||||
PRESENT_ARTIFACTS_DESCRIPTION = f"""
|
||||
将已经生成好的结果文件展示给用户。
|
||||
|
||||
使用场景：
|
||||
1. 你已经在 `{VIRTUAL_PATH_OUTPUTS}` 下写好了最终结果文件
|
||||
2. 你希望前端在对话结束后显示这些结果文件卡片
|
||||
3. 这些文件需要支持下载或预览
|
||||
|
||||
注意事项：
|
||||
1. 只能传入 `{VIRTUAL_PATH_OUTPUTS}` 下的文件
|
||||
2. 不要传入中间过程文件，只有真正需要给用户看的结果文件才调用
|
||||
3. 不要传入工具调用阶段文件，例如：
|
||||
- `{VIRTUAL_PATH_OUTPUTS}/{LARGE_TOOL_RESULTS_DIR_NAME}`
|
||||
- `{VIRTUAL_PATH_OUTPUTS}/{CONVERSATION_HISTORY_DIR_NAME}`
|
||||
4. 可以一次传多个文件
|
||||
"""
|
||||
|
||||
|
||||
@tool(
|
||||
category="buildin",
|
||||
tags=["文件", "交付物"],
|
||||
display_name="展示交付物",
|
||||
description=PRESENT_ARTIFACTS_DESCRIPTION,
|
||||
args_schema=PresentArtifactsInput,
|
||||
)
|
||||
def present_artifacts(
|
||||
filepaths: list[str],
|
||||
runtime: ToolRuntime,
|
||||
tool_call_id: Annotated[str, InjectedToolCallId],
|
||||
) -> Command:
|
||||
"""登记当前线程 outputs 目录下的交付物文件，使前端在对话结束后展示给用户。"""
|
||||
try:
|
||||
normalized_paths = [_normalize_presented_artifact_path(filepath, runtime) for filepath in filepaths]
|
||||
except ValueError as exc:
|
||||
return Command(update={"messages": [ToolMessage(content=f"Error: {exc}", tool_call_id=tool_call_id)]})
|
||||
|
||||
return Command(
|
||||
update={
|
||||
"artifacts": normalized_paths,
|
||||
"messages": [ToolMessage(content="已将交付物展示给用户", tool_call_id=tool_call_id)],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class OcrParseFileInput(BaseModel):
|
||||
"""Parse a sandbox file with OCR and save the Markdown result."""
|
||||
|
||||
file_path: str = Field(description="需要 OCR 解析的沙盒虚拟路径，必须位于 /home/gem/user-data 下")
|
||||
ocr_engine: str | None = Field(default=None, description="可选 OCR 引擎；省略时使用系统默认 OCR 引擎")
|
||||
|
||||
|
||||
OCR_PARSE_FILE_DESCRIPTION = f"""
|
||||
将沙盒中的 PDF 或图片文件解析为 Markdown 文本，并把结果保存为文件。
|
||||
|
||||
使用场景：
|
||||
1. 用户上传了 PDF/图片附件，需要提取其中的文字内容
|
||||
2. 工作区、uploads 或 outputs 下已有文件，需要转成可读取的 Markdown
|
||||
3. 解析结果较长，后续应使用 read_file 读取保存后的 Markdown 文件
|
||||
|
||||
注意事项：
|
||||
1. file_path 必须是 /home/gem/user-data 下的虚拟路径
|
||||
2. 只允许读取 workspace、uploads、outputs 下的普通文件
|
||||
3. 解析结果会写入 {VIRTUAL_PATH_OUTPUTS}/{_OCR_OUTPUT_DIR_NAME}/
|
||||
4. 工具只返回结果文件路径和短预览，不直接返回完整 OCR 文本
|
||||
5. 如需在前端展示结果文件，请再调用 present_artifacts
|
||||
"""
|
||||
|
||||
|
||||
@tool(
|
||||
category="buildin",
|
||||
tags=["文件", "OCR"],
|
||||
display_name="OCR 解析文件",
|
||||
description=OCR_PARSE_FILE_DESCRIPTION,
|
||||
args_schema=OcrParseFileInput,
|
||||
)
|
||||
async def ocr_parse_file(file_path: str, runtime: ToolRuntime, ocr_engine: str | None = None) -> dict:
|
||||
"""Parse a sandbox file with OCR, persist Markdown output, and return only a short result summary."""
|
||||
from yuxi.agents.backends.sandbox.paths import virtual_path_for_thread_file
|
||||
from yuxi.knowledge.parser import Parser
|
||||
|
||||
file_thread_id, uid, actual_path = _resolve_ocr_source_path(file_path, runtime)
|
||||
engine = _resolve_ocr_engine(ocr_engine)
|
||||
markdown = await Parser.aparse(str(actual_path), params={"ocr_engine": engine})
|
||||
|
||||
output_path = _next_ocr_output_path(file_thread_id, actual_path)
|
||||
output_path.write_text(markdown, encoding="utf-8")
|
||||
parsed_path = virtual_path_for_thread_file(file_thread_id, output_path, uid=uid)
|
||||
source_virtual_path = virtual_path_for_thread_file(file_thread_id, actual_path, uid=uid)
|
||||
preview, truncated = _ocr_preview(markdown)
|
||||
|
||||
return {
|
||||
"source_path": source_virtual_path,
|
||||
"parsed_path": parsed_path,
|
||||
"ocr_engine": engine,
|
||||
"char_count": len(markdown),
|
||||
"preview": preview,
|
||||
"truncated": truncated,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_ocr_source_path(file_path: str, runtime: ToolRuntime) -> tuple[str, str, Path]:
|
||||
"""Resolve a sandbox virtual path to a host file inside the Agent-visible user-data roots."""
|
||||
from yuxi.agents.backends.sandbox.paths import get_virtual_path_prefix, resolve_virtual_path
|
||||
|
||||
file_thread_id, uid = _resolve_runtime_file_scope(runtime)
|
||||
|
||||
normalized_input = str(file_path or "").strip()
|
||||
if not normalized_input:
|
||||
raise ValueError("文件路径不能为空")
|
||||
|
||||
virtual_prefix = get_virtual_path_prefix().rstrip("/")
|
||||
clean_virtual_path = "/" + normalized_input.lstrip("/")
|
||||
if clean_virtual_path != virtual_prefix and not clean_virtual_path.startswith(f"{virtual_prefix}/"):
|
||||
raise ValueError(f"只允许解析 {virtual_prefix} 下的沙盒虚拟路径")
|
||||
|
||||
relative_path = clean_virtual_path[len(virtual_prefix) :].lstrip("/")
|
||||
namespace = Path(relative_path).parts[0] if relative_path else ""
|
||||
if namespace not in _OCR_PARSE_ALLOWED_DIRS:
|
||||
allowed = ", ".join(f"{virtual_prefix}/{item}" for item in sorted(_OCR_PARSE_ALLOWED_DIRS))
|
||||
raise ValueError(f"只允许解析 {allowed} 下的文件")
|
||||
|
||||
try:
|
||||
actual_path = resolve_virtual_path(file_thread_id, clean_virtual_path, uid=uid)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"只允许解析 {virtual_prefix} 下的沙盒虚拟路径") from exc
|
||||
if not actual_path.exists():
|
||||
raise ValueError(f"文件不存在: {clean_virtual_path}")
|
||||
if not actual_path.is_file():
|
||||
raise ValueError(f"路径不是普通文件: {clean_virtual_path}")
|
||||
|
||||
return file_thread_id, uid, actual_path
|
||||
|
||||
|
||||
def _resolve_runtime_file_scope(runtime: ToolRuntime) -> tuple[str, str]:
|
||||
"""Read the thread and user scope needed for sandbox path mapping from ToolRuntime."""
|
||||
thread_id = _runtime_scope_value(runtime, "file_thread_id") or _runtime_scope_value(runtime, "thread_id")
|
||||
uid = _runtime_scope_value(runtime, "uid")
|
||||
if not thread_id:
|
||||
raise ValueError("当前运行时缺少 thread_id")
|
||||
if not uid:
|
||||
raise ValueError("当前运行时缺少 uid")
|
||||
return thread_id, uid
|
||||
|
||||
|
||||
def _runtime_scope_value(runtime: ToolRuntime, key: str) -> str | None:
|
||||
"""Look up a runtime scope value from LangGraph config, context, or state."""
|
||||
config = getattr(runtime, "config", None)
|
||||
configurable = config.get("configurable", {}) if isinstance(config, dict) else {}
|
||||
sources = (
|
||||
configurable if isinstance(configurable, dict) else {},
|
||||
getattr(runtime, "context", None),
|
||||
getattr(runtime, "state", None) if isinstance(getattr(runtime, "state", None), dict) else {},
|
||||
)
|
||||
for source in sources:
|
||||
value = source.get(key) if isinstance(source, dict) else getattr(source, key, None)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_ocr_engine(ocr_engine: str | None) -> str:
|
||||
"""Validate the requested OCR engine, falling back to the system default when omitted."""
|
||||
from yuxi import config
|
||||
from yuxi.knowledge.parser.factory import DocumentProcessorFactory
|
||||
|
||||
engine = str(ocr_engine or config.default_ocr_engine).strip() or config.default_ocr_engine
|
||||
allowed = {"disable", *DocumentProcessorFactory.get_available_processors()}
|
||||
if engine not in allowed:
|
||||
raise ValueError(f"不支持的 OCR 引擎: {engine}")
|
||||
return engine
|
||||
|
||||
|
||||
def _next_ocr_output_path(thread_id: str, source_path: Path) -> Path:
|
||||
"""Choose a non-conflicting Markdown output path under the thread outputs/ocr directory."""
|
||||
from yuxi.agents.backends.sandbox.paths import sandbox_outputs_dir
|
||||
|
||||
output_dir = sandbox_outputs_dir(thread_id) / _OCR_OUTPUT_DIR_NAME
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
base_name = _safe_ocr_output_stem(source_path)
|
||||
candidate = output_dir / f"{base_name}.md"
|
||||
index = 1
|
||||
while candidate.exists():
|
||||
candidate = output_dir / f"{base_name}-{index}.md"
|
||||
index += 1
|
||||
return candidate
|
||||
|
||||
|
||||
def _safe_ocr_output_stem(source_path: Path) -> str:
|
||||
"""Build a filesystem-friendly output filename stem from the source file name."""
|
||||
stem = source_path.stem.strip() or "ocr_result"
|
||||
safe_stem = _SAFE_OUTPUT_STEM_RE.sub("_", stem).strip("._-")
|
||||
return safe_stem or "ocr_result"
|
||||
|
||||
|
||||
def _ocr_preview(markdown: str) -> tuple[str, bool]:
|
||||
"""Return the short preview included in the tool result and whether it was truncated."""
|
||||
if len(markdown) <= _OCR_PREVIEW_LIMIT:
|
||||
return markdown, False
|
||||
return markdown[:_OCR_PREVIEW_LIMIT].rstrip(), True
|
||||
|
||||
|
||||
ASK_USER_QUESTION_DESCRIPTION = """
|
||||
在执行过程中，当你需要用户做决定或补充需求时，使用这个工具向用户提问。
|
||||
|
||||
适用场景：
|
||||
1. 收集用户偏好或需求（例如风格、范围、优先级）
|
||||
2. 澄清模糊指令（存在多种合理解释时）
|
||||
3. 在实现过程中让用户选择方案方向
|
||||
4. 在有明显权衡时让用户做取舍
|
||||
|
||||
使用规范：
|
||||
1. questions 提供 1-5 个问题，每项包含：question、options、multi_select、allow_other
|
||||
2. 每个问题的 options 提供 2-5 个有区分度的选项，每项包含 label 和 value
|
||||
3. 若有推荐选项：把推荐项放在第一位，并在 label 末尾加 "(Recommended)"
|
||||
4. 若需要多选：将该问题的 multi_select 设为 true
|
||||
5. allow_other 通常保持 true，用户可通过 Other 输入自定义答案
|
||||
|
||||
注意事项：
|
||||
1. 不要用这个工具询问“是否继续执行”“计划是否准备好”这类流程控制问题
|
||||
2. 不要在信息已充分、无需用户决策时滥用该工具
|
||||
3. 先基于现有上下文自行决策，只有关键不确定性时才提问
|
||||
|
||||
返回结果：
|
||||
answer 为 object，格式为 {question_id: answer}。
|
||||
其中 answer 可能是 string（单选）、list（多选）或 object（Other 文本）。
|
||||
"""
|
||||
|
||||
|
||||
@tool(
|
||||
category="buildin",
|
||||
tags=["交互"],
|
||||
display_name="向用户提问",
|
||||
description=ASK_USER_QUESTION_DESCRIPTION,
|
||||
)
|
||||
def ask_user_question(
|
||||
questions: Annotated[
|
||||
list[dict] | str | None,
|
||||
"问题列表，每项格式 {question, options, multi_select, allow_other, question_id(optional)}",
|
||||
] = None,
|
||||
) -> dict:
|
||||
"""向用户发起问题并等待回答。"""
|
||||
# 解析 questions 参数：如果是字符串，尝试解析为 JSON
|
||||
if isinstance(questions, str):
|
||||
try:
|
||||
import json
|
||||
|
||||
questions = json.loads(questions)
|
||||
logger.debug(f"Parsed string questions to list: {questions}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse questions string: {e}, using None")
|
||||
questions = None
|
||||
|
||||
normalized_questions = normalize_questions(questions or [])
|
||||
|
||||
if not normalized_questions:
|
||||
raise ValueError("questions 至少需要包含一个有效问题")
|
||||
|
||||
interrupt_payload = {
|
||||
"questions": normalized_questions,
|
||||
"source": "ask_user_question",
|
||||
}
|
||||
answer = interrupt(interrupt_payload)
|
||||
|
||||
return {
|
||||
"questions": normalized_questions,
|
||||
"answer": answer,
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# debug 工具包
|
||||
|
||||
__all__ = []
|
||||
@@ -0,0 +1,7 @@
|
||||
from .tools import (
|
||||
find_kb_document,
|
||||
get_common_kb_tools,
|
||||
open_kb_document,
|
||||
)
|
||||
|
||||
__all__ = ["find_kb_document", "get_common_kb_tools", "open_kb_document"]
|
||||
@@ -0,0 +1,473 @@
|
||||
"""知识库工具模块"""
|
||||
|
||||
import inspect
|
||||
from typing import Any
|
||||
|
||||
from langgraph.prebuilt.tool_node import ToolRuntime
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from yuxi.agents.toolkits.registry import tool
|
||||
from yuxi.knowledge.base import KnowledgeBase
|
||||
from yuxi.knowledge.schemas import (
|
||||
FindInputSchema,
|
||||
FindOutputSchema,
|
||||
OpenInputSchema,
|
||||
OpenOutputSchema,
|
||||
SearchInputSchema,
|
||||
SearchOutputSchema,
|
||||
)
|
||||
from yuxi.utils import logger
|
||||
|
||||
# ========== 通用知识库工具函数 ==========
|
||||
|
||||
|
||||
def _get_knowledge_base():
|
||||
from yuxi import knowledge_base
|
||||
|
||||
return knowledge_base
|
||||
|
||||
|
||||
class ListKBsInput(BaseModel):
|
||||
"""列出用户可访问的知识库输入模型"""
|
||||
|
||||
# Langchain 的 runtime 注入机制要求必须有参数
|
||||
dummy: str = Field(default="", description="Dummy parameter - ignore") # Add this
|
||||
|
||||
|
||||
@tool(category="knowledge", tags=["知识库"], args_schema=ListKBsInput)
|
||||
async def list_kbs(dummy: str, runtime: ToolRuntime) -> str: # Now has 2 params
|
||||
"""列出当前用户可访问的知识库列表
|
||||
|
||||
返回用户基于权限可访问的知识库名称列表。这个列表是根据用户的角色和部门信息过滤后的结果，
|
||||
但不包括用户在当前对话中未启用的知识库。
|
||||
|
||||
Returns:
|
||||
用户可访问的知识库名称列表（字符串格式）
|
||||
"""
|
||||
# 从 runtime.context 获取用户信息
|
||||
runtime_context = runtime.context
|
||||
uid = getattr(runtime_context, "uid", None)
|
||||
if not uid:
|
||||
return "无法获取用户信息"
|
||||
|
||||
# 打印 runtime—context 中的所有信息以进行调试
|
||||
logger.debug(f"Runtime context: {runtime_context.__dict__}")
|
||||
|
||||
enabled_kb_names = getattr(runtime_context, "knowledges", None)
|
||||
|
||||
try:
|
||||
from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context
|
||||
|
||||
available_kbs = await resolve_visible_knowledge_bases_for_context(runtime_context)
|
||||
except Exception as e:
|
||||
logger.error(f"获取用户知识库列表失败: {e}")
|
||||
return f"获取知识库列表失败: {str(e)}"
|
||||
|
||||
all_kb_names = [kb["name"] for kb in available_kbs]
|
||||
|
||||
logger.debug(f"用户 {uid} 可访问的知识库列表: {all_kb_names}")
|
||||
logger.debug(f"用户 {uid} 当前对话启用的知识库列表: {enabled_kb_names}")
|
||||
|
||||
if not available_kbs:
|
||||
return "当前没有可访问的知识库"
|
||||
|
||||
# 格式化输出（包含名称和描述）
|
||||
kb_list = []
|
||||
for kb in available_kbs:
|
||||
name = kb.get("name", "")
|
||||
desc = kb.get("description") or "无描述"
|
||||
kb_list.append({"kb_id": kb.get("kb_id"), "name": name, "description": desc})
|
||||
|
||||
return kb_list
|
||||
|
||||
|
||||
class GetMindmapInput(BaseModel):
|
||||
"""获取思维导图输入模型"""
|
||||
|
||||
kb_name: str = Field(description="知识库名称，用于指定要获取思维导图的知识库")
|
||||
|
||||
|
||||
@tool(category="knowledge", tags=["知识库"], args_schema=GetMindmapInput)
|
||||
async def get_mindmap(kb_name: str, runtime: ToolRuntime) -> str:
|
||||
"""获取指定知识库的思维导图结构
|
||||
|
||||
当用户想要了解知识库的整体结构、文件分类、知识架构时使用此工具。
|
||||
返回知识库的思维导图层级结构。
|
||||
|
||||
Args:
|
||||
kb_name: 知识库名称
|
||||
|
||||
Returns:
|
||||
知识库的思维导图结构（文本格式）
|
||||
"""
|
||||
if not kb_name:
|
||||
return "请提供知识库名称"
|
||||
|
||||
# 获取所有检索器
|
||||
knowledge_base = _get_knowledge_base()
|
||||
retrievers = knowledge_base.get_retrievers()
|
||||
|
||||
# 查找对应的知识库
|
||||
target_kb_id = None
|
||||
target_info = None
|
||||
for kb_id, info in retrievers.items():
|
||||
if info["name"] == kb_name:
|
||||
target_kb_id = kb_id
|
||||
target_info = info
|
||||
break
|
||||
|
||||
if not target_kb_id:
|
||||
return f"知识库 '{kb_name}' 不存在"
|
||||
|
||||
try:
|
||||
from yuxi.repositories.knowledge_base_repository import KnowledgeBaseRepository
|
||||
|
||||
kb_repo = KnowledgeBaseRepository()
|
||||
kb = await kb_repo.get_by_kb_id(target_kb_id)
|
||||
|
||||
if kb is None:
|
||||
return f"知识库 {target_info['name']} 不存在"
|
||||
|
||||
mindmap_data = kb.mindmap
|
||||
|
||||
if not mindmap_data:
|
||||
return f"知识库 {target_info['name']} 还没有生成思维导图。"
|
||||
|
||||
# 将思维导图数据转换为文本格式
|
||||
def mindmap_to_text(node, level=0):
|
||||
"""递归将思维导图JSON转换为层级文本"""
|
||||
indent = " " * level
|
||||
text = f"{indent}- {node.get('content', '')}\n"
|
||||
for child in node.get("children", []):
|
||||
text += mindmap_to_text(child, level + 1)
|
||||
return text
|
||||
|
||||
mindmap_text = f"知识库 {target_info['name']} 的思维导图结构：\n\n"
|
||||
mindmap_text += mindmap_to_text(mindmap_data)
|
||||
|
||||
return mindmap_text
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取思维导图失败: {e}")
|
||||
return f"获取思维导图失败: {str(e)}"
|
||||
|
||||
|
||||
QueryKBInput = SearchInputSchema
|
||||
OpenKBDocumentInput = OpenInputSchema
|
||||
FindKBDocumentInput = FindInputSchema
|
||||
|
||||
|
||||
async def _resolve_visible_knowledge_bases_for_query(runtime: ToolRuntime | None) -> list[dict[str, Any]]:
|
||||
if runtime is None:
|
||||
return []
|
||||
|
||||
context = getattr(runtime, "context", None)
|
||||
if context is None:
|
||||
return []
|
||||
|
||||
visible_kbs = getattr(context, "_visible_knowledge_bases", None)
|
||||
if isinstance(visible_kbs, list):
|
||||
return visible_kbs
|
||||
|
||||
try:
|
||||
from yuxi.agents.backends.knowledge_base_backend import resolve_visible_knowledge_bases_for_context
|
||||
|
||||
return await resolve_visible_knowledge_bases_for_context(context)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(f"解析会话可见知识库失败: {exc}")
|
||||
return []
|
||||
|
||||
|
||||
def _find_query_target(
|
||||
*,
|
||||
kb_id: str,
|
||||
retrievers: dict[str, Any],
|
||||
visible_kbs: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, Any] | None, str | None, str | None]:
|
||||
if not visible_kbs:
|
||||
return None, None, "无法获取当前会话可访问的知识库"
|
||||
|
||||
normalized_kb_id = str(kb_id or "").strip()
|
||||
visible_kb_ids = {str(kb.get("kb_id") or "").strip() for kb in visible_kbs}
|
||||
if normalized_kb_id not in visible_kb_ids:
|
||||
return None, None, f"知识库资源 '{normalized_kb_id}' 不存在或当前会话未启用"
|
||||
|
||||
target_info = retrievers.get(normalized_kb_id)
|
||||
if target_info is None:
|
||||
return None, None, f"知识库资源 '{normalized_kb_id}' 不存在"
|
||||
return target_info, normalized_kb_id, None
|
||||
|
||||
|
||||
async def _build_query_output(target_kb_id: str, result: Any) -> Any:
|
||||
if isinstance(result, dict) and result.get("kb_id") == target_kb_id and isinstance(result.get("results"), list):
|
||||
return SearchOutputSchema(**result).model_dump()
|
||||
return KnowledgeBase.build_search_output(target_kb_id, result)
|
||||
|
||||
|
||||
@tool(category="knowledge", tags=["知识库"], args_schema=QueryKBInput)
|
||||
async def query_kb(kb_id: str, query_text: str, file_name: str | None = None, runtime: ToolRuntime = None) -> Any:
|
||||
"""在指定知识库中检索内容
|
||||
|
||||
当用户需要查询具体内容时使用此工具。kb_id 是知识库资源 ID，也就是 kb_id；返回结果中的
|
||||
file_id 可继续用于 find_kb_document 或 open_kb_document。
|
||||
"""
|
||||
if not kb_id:
|
||||
return "请提供 kb_id"
|
||||
if not query_text:
|
||||
return "请提供查询内容"
|
||||
|
||||
knowledge_base = _get_knowledge_base()
|
||||
retrievers = knowledge_base.get_retrievers()
|
||||
visible_kbs = await _resolve_visible_knowledge_bases_for_query(runtime)
|
||||
target_info, target_kb_id, target_error = _find_query_target(
|
||||
kb_id=kb_id,
|
||||
retrievers=retrievers,
|
||||
visible_kbs=visible_kbs,
|
||||
)
|
||||
if target_error:
|
||||
return target_error
|
||||
|
||||
try:
|
||||
retriever = target_info["retriever"]
|
||||
kwargs = {}
|
||||
if file_name:
|
||||
kwargs["file_name"] = file_name
|
||||
|
||||
if inspect.iscoroutinefunction(retriever):
|
||||
result = await retriever(query_text, **kwargs)
|
||||
else:
|
||||
result = retriever(query_text, **kwargs)
|
||||
|
||||
return await _build_query_output(target_kb_id, result)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"检索失败: {e}")
|
||||
return f"检索失败: {str(e)}"
|
||||
|
||||
|
||||
@tool(category="knowledge", tags=["知识库"], args_schema=OpenKBDocumentInput)
|
||||
async def open_kb_document(
|
||||
kb_id: str,
|
||||
file_id: str,
|
||||
line: int | None = None,
|
||||
offset: int | None = None,
|
||||
window_size: int = 1800,
|
||||
runtime: ToolRuntime = None,
|
||||
) -> dict[str, Any] | str:
|
||||
"""按行窗口打开知识库文档原文
|
||||
|
||||
当 query_kb 返回的片段不足以回答问题，或需要查看某个文档的上下文时使用。
|
||||
kb_id 是知识库资源 ID，也就是 kb_id；file_id 是知识库文件 ID。
|
||||
"""
|
||||
normalized_kb_id = str(kb_id or "").strip()
|
||||
normalized_file_id = str(file_id or "").strip()
|
||||
if not normalized_kb_id:
|
||||
return "请提供 kb_id"
|
||||
if not normalized_file_id:
|
||||
return "请提供 file_id"
|
||||
|
||||
visible_kbs = await _resolve_visible_knowledge_bases_for_query(runtime)
|
||||
if not visible_kbs:
|
||||
return "无法获取当前会话可访问的知识库"
|
||||
|
||||
visible_kb_ids = {str(kb.get("kb_id") or "").strip() for kb in visible_kbs}
|
||||
if normalized_kb_id not in visible_kb_ids:
|
||||
return f"知识库资源 '{normalized_kb_id}' 不存在或当前会话未启用"
|
||||
|
||||
knowledge_base = _get_knowledge_base()
|
||||
retrievers = knowledge_base.get_retrievers()
|
||||
target_info = retrievers.get(normalized_kb_id)
|
||||
if target_info is None:
|
||||
return f"知识库资源 '{normalized_kb_id}' 不存在"
|
||||
|
||||
metadata = target_info.get("metadata") if isinstance(target_info, dict) else None
|
||||
kb_type = str((metadata or {}).get("kb_type") or "").strip().lower()
|
||||
if kb_type == "dify":
|
||||
return "Dify 知识库为外部只读检索源，当前不支持通过 Open 打开全文"
|
||||
|
||||
try:
|
||||
start_offset = int(line) - 1 if line is not None else int(offset or 0)
|
||||
window = await knowledge_base.open_file_content(
|
||||
normalized_kb_id,
|
||||
normalized_file_id,
|
||||
offset=start_offset,
|
||||
limit=window_size,
|
||||
)
|
||||
return OpenOutputSchema(kb_id=normalized_kb_id, file_id=normalized_file_id, **window).model_dump()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"打开知识库文档失败: {e}")
|
||||
return f"打开知识库文档失败: {str(e)}"
|
||||
|
||||
|
||||
@tool(category="knowledge", tags=["知识库"], args_schema=FindKBDocumentInput)
|
||||
async def find_kb_document(
|
||||
kb_id: str,
|
||||
file_id: str,
|
||||
patterns: list[str],
|
||||
use_regex: bool = False,
|
||||
case_sensitive: bool = False,
|
||||
max_windows: int = 5,
|
||||
window_size: int = 80,
|
||||
runtime: ToolRuntime = None,
|
||||
) -> dict[str, Any] | str:
|
||||
"""在已知知识库文件内做关键词或正则定位。
|
||||
|
||||
当 query_kb 已找到候选文件，但需要在该文件内定位术语、指标、章节或实体时使用。
|
||||
"""
|
||||
normalized_kb_id = str(kb_id or "").strip()
|
||||
normalized_file_id = str(file_id or "").strip()
|
||||
if not normalized_kb_id:
|
||||
return "请提供 kb_id"
|
||||
if not normalized_file_id:
|
||||
return "请提供 file_id"
|
||||
if not patterns:
|
||||
return "请提供 patterns"
|
||||
|
||||
visible_kbs = await _resolve_visible_knowledge_bases_for_query(runtime)
|
||||
if not visible_kbs:
|
||||
return "无法获取当前会话可访问的知识库"
|
||||
|
||||
visible_kb_ids = {str(kb.get("kb_id") or "").strip() for kb in visible_kbs}
|
||||
if normalized_kb_id not in visible_kb_ids:
|
||||
return f"知识库资源 '{normalized_kb_id}' 不存在或当前会话未启用"
|
||||
|
||||
knowledge_base = _get_knowledge_base()
|
||||
retrievers = knowledge_base.get_retrievers()
|
||||
target_info = retrievers.get(normalized_kb_id)
|
||||
if target_info is None:
|
||||
return f"知识库资源 '{normalized_kb_id}' 不存在"
|
||||
|
||||
metadata = target_info.get("metadata") if isinstance(target_info, dict) else None
|
||||
kb_type = str((metadata or {}).get("kb_type") or "").strip().lower()
|
||||
if kb_type == "dify":
|
||||
return "Dify 知识库为外部只读检索源，当前不支持通过 Find 检索全文"
|
||||
|
||||
try:
|
||||
result = await knowledge_base.find_file_content(
|
||||
normalized_kb_id,
|
||||
normalized_file_id,
|
||||
patterns,
|
||||
use_regex=use_regex,
|
||||
case_sensitive=case_sensitive,
|
||||
max_windows=max_windows,
|
||||
window_size=window_size,
|
||||
)
|
||||
return FindOutputSchema(kb_id=normalized_kb_id, file_id=normalized_file_id, **result).model_dump()
|
||||
except Exception as e:
|
||||
logger.error(f"知识库文档内检索失败: {e}")
|
||||
return f"知识库文档内检索失败: {str(e)}"
|
||||
|
||||
|
||||
# 单个知识库一次最多扫描的文件数（与仓储层 list_by_kb_id_after 的硬上限保持一致），
|
||||
# 用于在内存中做文件名过滤与准确计数，避免按 limit+offset 截断导致 total/has_more 失真。
|
||||
_KB_FILE_SCAN_LIMIT = 5000
|
||||
|
||||
|
||||
class SearchFileInput(BaseModel):
|
||||
"""搜索文件输入模型"""
|
||||
|
||||
kb_name: str | None = Field(default=None, description="知识库名称，为空时搜索所有知识库")
|
||||
query: str | None = Field(default=None, description="搜索关键词，为空时返回所有文件")
|
||||
offset: int = Field(default=0, ge=0, description="偏移量，从 0 开始")
|
||||
limit: int = Field(default=300, ge=1, le=5000, description="返回数量限制，默认 300")
|
||||
|
||||
|
||||
@tool(category="knowledge", tags=["知识库"], args_schema=SearchFileInput)
|
||||
async def search_file(
|
||||
kb_name: str | None = None,
|
||||
query: str | None = None,
|
||||
offset: int = 0,
|
||||
limit: int = 300,
|
||||
runtime: ToolRuntime = None,
|
||||
) -> dict[str, Any] | str:
|
||||
"""搜索知识库中的文件
|
||||
|
||||
当用户需要查找特定文件时使用此工具。可以指定知识库名称和搜索关键词。
|
||||
如果不指定知识库，将搜索所有可访问的知识库。
|
||||
如果不指定搜索关键词，将返回所有文件。
|
||||
|
||||
Args:
|
||||
kb_name: 知识库名称，为空时搜索所有知识库
|
||||
query: 搜索关键词，为空时返回所有文件
|
||||
offset: 偏移量，从 0 开始
|
||||
limit: 返回数量限制，默认 300
|
||||
|
||||
Returns:
|
||||
匹配的文件列表和分页信息
|
||||
"""
|
||||
if not kb_name and not query:
|
||||
return "请提供知识库名称或搜索关键词，不能同时为空"
|
||||
|
||||
visible_kbs = await _resolve_visible_knowledge_bases_for_query(runtime)
|
||||
if not visible_kbs:
|
||||
return "无法获取当前会话可访问的知识库"
|
||||
|
||||
if kb_name:
|
||||
target_kbs = [kb for kb in visible_kbs if kb.get("name") == kb_name]
|
||||
if not target_kbs:
|
||||
return f"知识库 '{kb_name}' 不存在或当前会话未启用"
|
||||
else:
|
||||
target_kbs = visible_kbs
|
||||
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
|
||||
repo = KnowledgeFileRepository()
|
||||
|
||||
all_files = []
|
||||
for kb in target_kbs:
|
||||
kb_id = kb.get("kb_id")
|
||||
if not kb_id:
|
||||
continue
|
||||
|
||||
files = await repo.list_by_kb_id_after(
|
||||
kb_id=kb_id,
|
||||
limit=_KB_FILE_SCAN_LIMIT,
|
||||
files_only=True,
|
||||
)
|
||||
|
||||
if query:
|
||||
query_lower = query.lower()
|
||||
files = [f for f in files if query_lower in f.filename.lower()]
|
||||
|
||||
for f in files:
|
||||
all_files.append(
|
||||
{
|
||||
"kb_id": kb_id,
|
||||
"kb_name": kb.get("name"),
|
||||
"file_id": f.file_id,
|
||||
"filename": f.filename,
|
||||
"file_type": f.file_type,
|
||||
"status": f.status,
|
||||
"created_at": str(f.created_at) if f.created_at else None,
|
||||
"updated_at": str(f.updated_at) if f.updated_at else None,
|
||||
"file_size": f.file_size,
|
||||
}
|
||||
)
|
||||
|
||||
all_files.sort(key=lambda x: x.get("updated_at") or "", reverse=True)
|
||||
|
||||
total = len(all_files)
|
||||
paginated_files = all_files[offset : offset + limit]
|
||||
|
||||
return {
|
||||
"files": paginated_files,
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"has_more": offset + limit < total,
|
||||
}
|
||||
|
||||
|
||||
def get_common_kb_tools() -> list:
|
||||
"""获取通用知识库工具列表
|
||||
|
||||
返回 6 个通用工具：
|
||||
- list_kbs: 列出用户可访问的知识库
|
||||
- get_mindmap: 获取指定知识库的思维导图
|
||||
- query_kb: 在指定知识库中检索
|
||||
- find_kb_document: 在指定文件内定位关键词或正则模式
|
||||
- open_kb_document: 按 file_id 分段打开知识库文档
|
||||
- search_file: 搜索知识库中的文件
|
||||
"""
|
||||
return [list_kbs, get_mindmap, query_kb, find_kb_document, open_kb_document, search_file]
|
||||
@@ -0,0 +1,89 @@
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolExtraMetadata:
|
||||
"""附加元数据（用装饰器注册）"""
|
||||
|
||||
category: str = "" # 分类: buildin, knowledge, mysql, subagents, debug
|
||||
tags: list[str] = field(default_factory=list)
|
||||
display_name: str = "" # 显示名称（给人看的名字）
|
||||
icon: str = ""
|
||||
config_guide: str = "" # 配置说明（给人看的使用前配置提示）
|
||||
|
||||
|
||||
# 全局注册表: tool_name -> ToolExtraMetadata
|
||||
_extra_registry: dict[str, ToolExtraMetadata] = {}
|
||||
|
||||
# 全局工具实例列表（由 @tool 装饰器自动收集）
|
||||
_all_tool_instances: list = []
|
||||
|
||||
|
||||
def get_extra_metadata(tool_name: str) -> ToolExtraMetadata | None:
|
||||
"""获取工具附加元数据"""
|
||||
return _extra_registry.get(tool_name)
|
||||
|
||||
|
||||
def get_all_extra_metadata() -> dict[str, ToolExtraMetadata]:
|
||||
"""获取所有附加元数据"""
|
||||
return _extra_registry.copy()
|
||||
|
||||
|
||||
def get_all_tool_instances() -> list:
|
||||
"""获取所有工具实例（由 @tool 装饰器自动收集）"""
|
||||
return _all_tool_instances
|
||||
|
||||
|
||||
# 基于 langchain.tool 的拓展装饰器
|
||||
def tool(
|
||||
category: str = "",
|
||||
tags: list[str] = None,
|
||||
display_name: str = "",
|
||||
icon: str = "",
|
||||
config_guide: str = "",
|
||||
name_or_callable: str | Callable | None = None,
|
||||
description: str | None = None,
|
||||
args_schema: type | None = None,
|
||||
return_direct: bool = False,
|
||||
):
|
||||
"""基于 langchain.tool 的拓展装饰器，同时注册元数据
|
||||
|
||||
使用方式:
|
||||
@tool(category="buildin", tags=["计算"], display_name="计算器")
|
||||
def calculator(a: float, b: float, operation: str) -> float:
|
||||
...
|
||||
|
||||
或者保留原有的 name_or_callable 和 description 参数来自定义工具名称和说明。
|
||||
"""
|
||||
from langchain.tools import tool as langchain_tool
|
||||
|
||||
# 先应用 langchain tool 装饰器
|
||||
langchain_decorator = langchain_tool(
|
||||
name_or_callable=name_or_callable,
|
||||
description=description,
|
||||
args_schema=args_schema,
|
||||
return_direct=return_direct,
|
||||
)
|
||||
|
||||
def decorator(func: Callable) -> Callable:
|
||||
# 应用 langchain 装饰器
|
||||
tool_obj = langchain_decorator(func)
|
||||
|
||||
# 注册附加元数据
|
||||
tool_name = tool_obj.name
|
||||
_extra_registry[tool_name] = ToolExtraMetadata(
|
||||
category=category,
|
||||
tags=tags or [],
|
||||
display_name=display_name,
|
||||
icon=icon,
|
||||
config_guide=config_guide,
|
||||
)
|
||||
|
||||
# 自动收集工具实例
|
||||
tool_obj.handle_tool_error = True
|
||||
_all_tool_instances.append(tool_obj)
|
||||
|
||||
return tool_obj
|
||||
|
||||
return decorator
|
||||
@@ -0,0 +1,144 @@
|
||||
from typing import Any
|
||||
|
||||
from yuxi.utils import logger
|
||||
|
||||
# 工具元数据缓存
|
||||
_metadata_cache: list[dict] = []
|
||||
|
||||
|
||||
def _extract_tool_info(tool_obj) -> dict:
|
||||
"""从 tool_obj 提取基础信息"""
|
||||
metadata = getattr(tool_obj, "metadata", {}) or {}
|
||||
info = {
|
||||
"slug": tool_obj.name,
|
||||
"name": metadata.get("name", tool_obj.name), # 显示名称优先从 metadata 获取
|
||||
"description": tool_obj.description,
|
||||
"metadata": metadata,
|
||||
"args": [],
|
||||
}
|
||||
|
||||
if hasattr(tool_obj, "args_schema") and tool_obj.args_schema:
|
||||
schema = tool_obj.args_schema
|
||||
if hasattr(schema, "schema"):
|
||||
schema = schema.schema()
|
||||
for arg_name, arg_info in schema.get("properties", {}).items():
|
||||
info["args"].append(
|
||||
{
|
||||
"name": arg_name,
|
||||
"type": arg_info.get("type", ""),
|
||||
"description": arg_info.get("description", ""),
|
||||
}
|
||||
)
|
||||
return info
|
||||
|
||||
|
||||
def _ensure_metadata_loaded():
|
||||
"""延迟加载工具元数据（首次调用时自动触发）"""
|
||||
global _metadata_cache
|
||||
|
||||
if _metadata_cache: # 已加载
|
||||
return
|
||||
|
||||
from yuxi.agents.toolkits.registry import (
|
||||
get_all_extra_metadata,
|
||||
get_all_tool_instances,
|
||||
)
|
||||
|
||||
# 获取所有工具实例
|
||||
all_tools = get_all_tool_instances()
|
||||
extra_meta = get_all_extra_metadata()
|
||||
|
||||
for tool in all_tools:
|
||||
tool_name = tool.name
|
||||
runtime_info = _extract_tool_info(tool)
|
||||
|
||||
# 合并附加元数据
|
||||
if tool_name in extra_meta:
|
||||
extra = extra_meta[tool_name]
|
||||
runtime_info["category"] = extra.category
|
||||
runtime_info["tags"] = extra.tags
|
||||
runtime_info["config_guide"] = extra.config_guide
|
||||
# display_name 优先级高于 tool.name
|
||||
if extra.display_name:
|
||||
runtime_info["name"] = extra.display_name
|
||||
else:
|
||||
# 未注册，设为默认分类
|
||||
runtime_info["category"] = "buildin"
|
||||
runtime_info["tags"] = []
|
||||
runtime_info["config_guide"] = ""
|
||||
|
||||
_metadata_cache.append(runtime_info)
|
||||
|
||||
logger.info(f"Tool service loaded {len(_metadata_cache)} tools (lazy load)")
|
||||
|
||||
|
||||
def get_tool_metadata(category: str = None) -> list[dict]:
|
||||
"""获取工具元数据列表（延迟加载）"""
|
||||
_ensure_metadata_loaded()
|
||||
|
||||
if category:
|
||||
return [t for t in _metadata_cache if t.get("category") == category]
|
||||
return _metadata_cache
|
||||
|
||||
|
||||
def get_tool_instances_by_category(category: str) -> list[Any]:
|
||||
from yuxi.agents.toolkits.registry import get_all_extra_metadata, get_all_tool_instances
|
||||
|
||||
extra_meta = get_all_extra_metadata()
|
||||
tools = []
|
||||
for tool in get_all_tool_instances():
|
||||
tool_meta = extra_meta.get(tool.name)
|
||||
tool_category = tool_meta.category if tool_meta else "buildin"
|
||||
if tool_category == category:
|
||||
tools.append(tool)
|
||||
return tools
|
||||
|
||||
|
||||
async def resolve_configured_runtime_tools(context) -> list[Any]:
|
||||
from yuxi.agents.mcp.service import get_enabled_mcp_tools
|
||||
|
||||
selected_tools = []
|
||||
selected_tool_names: set[str] = set()
|
||||
buildin_tools = {tool.name: tool for tool in get_tool_instances_by_category("buildin")}
|
||||
|
||||
for tool_name in getattr(context, "tools", None) or []:
|
||||
if not isinstance(tool_name, str) or tool_name in selected_tool_names:
|
||||
continue
|
||||
tool = buildin_tools.get(tool_name)
|
||||
if tool is None:
|
||||
logger.warning(f"Configured buildin tool not found, skip: {tool_name}")
|
||||
continue
|
||||
selected_tools.append(tool)
|
||||
selected_tool_names.add(tool_name)
|
||||
|
||||
selected_mcp_servers: set[str] = set()
|
||||
for server_name in getattr(context, "mcps", None) or []:
|
||||
if not isinstance(server_name, str) or server_name in selected_mcp_servers:
|
||||
continue
|
||||
selected_mcp_servers.add(server_name)
|
||||
try:
|
||||
mcp_tools = await get_enabled_mcp_tools(server_name)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load configured MCP tools '{server_name}': {e}")
|
||||
continue
|
||||
if not mcp_tools:
|
||||
logger.warning(f"Configured MCP unavailable, skip: {server_name}")
|
||||
continue
|
||||
for tool in mcp_tools:
|
||||
if tool.name in selected_tool_names:
|
||||
continue
|
||||
selected_tools.append(tool)
|
||||
selected_tool_names.add(tool.name)
|
||||
|
||||
# Skill 依赖的本地工具：必须随基础工具一起注册进 create_agent 的 ToolNode 才可执行，
|
||||
# 否则 Skill 激活后模型虽能发起调用，执行器仍报 "not a valid tool"。
|
||||
# 默认绑定给模型的可见性由 SkillsMiddleware 按 Skill 激活状态门控（保持按需加载）。
|
||||
from yuxi.agents.middlewares.skills import resolve_skill_gated_tools
|
||||
|
||||
for tool in resolve_skill_gated_tools(context):
|
||||
if tool.name in selected_tool_names:
|
||||
continue
|
||||
selected_tools.append(tool)
|
||||
selected_tool_names.add(tool.name)
|
||||
|
||||
return selected_tools
|
||||
@@ -0,0 +1,55 @@
|
||||
import traceback
|
||||
from typing import Any
|
||||
|
||||
from yuxi.utils import logger
|
||||
|
||||
|
||||
def get_tool_info(tools) -> list[dict[str, Any]]:
|
||||
"""获取所有工具的信息（用于前端展示）"""
|
||||
tools_info = []
|
||||
|
||||
try:
|
||||
# 获取注册的工具信息
|
||||
for tool_obj in tools:
|
||||
try:
|
||||
metadata = getattr(tool_obj, "metadata", {}) or {}
|
||||
info = {
|
||||
"id": tool_obj.name,
|
||||
"name": metadata.get("name", tool_obj.name),
|
||||
"description": tool_obj.description,
|
||||
"metadata": metadata,
|
||||
"args": [],
|
||||
# "is_async": is_async # Include async information
|
||||
}
|
||||
|
||||
if hasattr(tool_obj, "args_schema") and tool_obj.args_schema:
|
||||
if isinstance(tool_obj.args_schema, dict):
|
||||
schema = tool_obj.args_schema
|
||||
else:
|
||||
schema = tool_obj.args_schema.schema()
|
||||
|
||||
for arg_name, arg_info in schema.get("properties", {}).items():
|
||||
info["args"].append(
|
||||
{
|
||||
"name": arg_name,
|
||||
"type": arg_info.get("type", ""),
|
||||
"description": arg_info.get("description", ""),
|
||||
}
|
||||
)
|
||||
|
||||
tools_info.append(info)
|
||||
# logger.debug(f"Successfully processed tool info for {tool_obj.name}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to process tool {getattr(tool_obj, 'name', 'unknown')}: {e}\n{traceback.format_exc()}. "
|
||||
f"Details: {dict(tool_obj.__dict__)}"
|
||||
)
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get tools info: {e}\n{traceback.format_exc()}")
|
||||
return []
|
||||
|
||||
logger.info(f"Successfully extracted info for {len(tools_info)} tools")
|
||||
return tools_info
|
||||
@@ -0,0 +1,4 @@
|
||||
from .app import config
|
||||
from .user import UserConfig, UserConfigSchema
|
||||
|
||||
__all__ = ["UserConfig", "UserConfigSchema", "config"]
|
||||
@@ -0,0 +1,211 @@
|
||||
"""应用配置模块。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import tomli
|
||||
import tomli_w
|
||||
from pydantic import BaseModel, Field, PrivateAttr
|
||||
|
||||
from yuxi.config import cache as runtime_cache
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
READONLY_CONFIG_FIELDS = frozenset({"save_dir"})
|
||||
DEFAULT_OCR_ENGINE = "rapid_ocr"
|
||||
|
||||
|
||||
def _get_available_ocr_engines() -> set[str]:
|
||||
from yuxi.knowledge.parser.factory import DocumentProcessorFactory
|
||||
|
||||
return {"disable", *DocumentProcessorFactory.get_available_processors()}
|
||||
|
||||
|
||||
def _normalize_default_ocr_engine(value: Any) -> str:
|
||||
engine = str(value or "").strip() or DEFAULT_OCR_ENGINE
|
||||
if engine not in _get_available_ocr_engines():
|
||||
raise ValueError(f"不支持的默认 OCR 引擎: {engine}")
|
||||
return engine
|
||||
|
||||
|
||||
class Config(BaseModel):
|
||||
"""应用配置类。
|
||||
|
||||
`save_dir` 只在启动时决定配置文件位置，运行时不可修改。管理员保存配置时先写
|
||||
`base.toml`，再把可运行时同步的字段写入 Redis 快照（`yuxi:runtime_config`）。
|
||||
其他进程通过 `start_runtime_sync()` 启动的后台线程周期性拉取该快照刷新内存值。
|
||||
"""
|
||||
|
||||
save_dir: str = Field(default="saves", description="保存目录", exclude=True)
|
||||
enable_content_guard: bool = Field(default=False, description="是否启用内容审查")
|
||||
enable_content_guard_llm: bool = Field(default=False, description="是否启用LLM内容审查")
|
||||
default_model: str = Field(
|
||||
default="siliconflow-cn:Pro/MiniMaxAI/MiniMax-M2.5",
|
||||
description="默认对话模型",
|
||||
)
|
||||
fast_model: str = Field(
|
||||
default="siliconflow-cn:Pro/MiniMaxAI/MiniMax-M2.5",
|
||||
description="快速响应模型",
|
||||
)
|
||||
embed_model: str = Field(
|
||||
default="siliconflow-cn:Pro/BAAI/bge-m3",
|
||||
description="默认 Embedding 模型",
|
||||
)
|
||||
reranker: str = Field(
|
||||
default="siliconflow-cn:Pro/BAAI/bge-reranker-v2-m3",
|
||||
description="默认 Re-Ranker 模型",
|
||||
)
|
||||
content_guard_llm_model: str = Field(
|
||||
default="siliconflow-cn:Pro/MiniMaxAI/MiniMax-M2.5",
|
||||
description="内容审查LLM模型",
|
||||
)
|
||||
default_ocr_engine: str = Field(default=DEFAULT_OCR_ENGINE, description="默认 OCR 解析引擎")
|
||||
|
||||
sandbox_provider: str = Field(default="provisioner", description="沙箱提供者")
|
||||
sandbox_provisioner_url: str = Field(default="http://sandbox-provisioner:8002", description="沙箱服务地址")
|
||||
sandbox_virtual_path_prefix: str = Field(default="/home/gem/user-data", description="沙箱用户目录前缀")
|
||||
sandbox_exec_timeout_seconds: int = Field(default=180, description="沙箱执行超时时间（秒）")
|
||||
sandbox_max_output_bytes: int = Field(default=262144, description="沙箱最大输出字节数")
|
||||
sandbox_keepalive_interval_seconds: int = Field(default=30, description="沙箱保活间隔")
|
||||
|
||||
_config_file: Path | None = PrivateAttr(default=None)
|
||||
_runtime_sync_thread: Any = PrivateAttr(default=None)
|
||||
|
||||
model_config = {"arbitrary_types_allowed": True, "extra": "allow"}
|
||||
|
||||
def __init__(self, **data):
|
||||
super().__init__(**data)
|
||||
self._setup_paths()
|
||||
self._load_user_config()
|
||||
self._handle_environment()
|
||||
|
||||
def _setup_paths(self) -> None:
|
||||
self._config_file = Path(self.save_dir) / "config" / "base.toml"
|
||||
self._config_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _load_user_config(self) -> None:
|
||||
if not self._config_file or not self._config_file.exists():
|
||||
logger.info(f"Config file not found, using defaults: {self._config_file}")
|
||||
return
|
||||
|
||||
logger.info(f"Loading config from {self._config_file}")
|
||||
try:
|
||||
with open(self._config_file, "rb") as f:
|
||||
user_config = tomli.load(f)
|
||||
|
||||
for key, value in user_config.items():
|
||||
if key in READONLY_CONFIG_FIELDS:
|
||||
logger.warning(f"Readonly config key ignored: {key}")
|
||||
elif key in type(self).model_fields:
|
||||
try:
|
||||
setattr(self, key, self._normalize_config_value(key, value))
|
||||
except ValueError as exc:
|
||||
logger.warning(f"Invalid config key ignored: {key} ({exc})")
|
||||
else:
|
||||
logger.warning(f"Unknown config key: {key}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load config from {self._config_file}: {e}")
|
||||
|
||||
def _handle_environment(self) -> None:
|
||||
self.sandbox_provider = (os.getenv("SANDBOX_PROVIDER") or self.sandbox_provider or "provisioner").strip()
|
||||
self.sandbox_provisioner_url = (
|
||||
os.getenv("SANDBOX_PROVISIONER_URL") or self.sandbox_provisioner_url or "http://sandbox-provisioner:8002"
|
||||
).strip()
|
||||
self.sandbox_virtual_path_prefix = (
|
||||
os.getenv("SANDBOX_VIRTUAL_PATH_PREFIX") or self.sandbox_virtual_path_prefix or "/home/gem/user-data"
|
||||
).strip()
|
||||
self.sandbox_exec_timeout_seconds = int(
|
||||
os.getenv("SANDBOX_EXEC_TIMEOUT_SECONDS") or self.sandbox_exec_timeout_seconds or 180
|
||||
)
|
||||
self.sandbox_max_output_bytes = int(
|
||||
os.getenv("SANDBOX_MAX_OUTPUT_BYTES") or self.sandbox_max_output_bytes or 262144
|
||||
)
|
||||
self.sandbox_keepalive_interval_seconds = int(
|
||||
os.getenv("SANDBOX_KEEPALIVE_INTERVAL_SECONDS") or self.sandbox_keepalive_interval_seconds or 30
|
||||
)
|
||||
|
||||
if self.sandbox_provider.lower() != "provisioner":
|
||||
raise ValueError("Only sandbox_provider=provisioner is supported.")
|
||||
if not self.sandbox_provisioner_url:
|
||||
raise ValueError("SANDBOX_PROVISIONER_URL is required when sandbox provider is provisioner.")
|
||||
if not self.sandbox_virtual_path_prefix.startswith("/"):
|
||||
self.sandbox_virtual_path_prefix = f"/{self.sandbox_virtual_path_prefix}"
|
||||
|
||||
def start_runtime_sync(self, interval: float = runtime_cache.RUNTIME_CONFIG_SYNC_INTERVAL_SECONDS) -> None:
|
||||
"""启动后台线程周期性从 Redis 同步运行时配置。多次调用仅启动一次。"""
|
||||
self._runtime_sync_thread = runtime_cache.start_runtime_sync(
|
||||
self,
|
||||
self._runtime_sync_thread,
|
||||
interval=interval,
|
||||
)
|
||||
|
||||
def refresh(self) -> None:
|
||||
"""从 Redis 快照刷新公开配置字段到内存；Redis 不可用或无快照时保持当前值。"""
|
||||
runtime_cache.refresh_runtime_config(self)
|
||||
|
||||
def save(self) -> None:
|
||||
if not self._config_file:
|
||||
logger.warning("Config file path not set")
|
||||
return
|
||||
|
||||
logger.info(f"Saving config to {self._config_file}")
|
||||
user_modified = {}
|
||||
for field_name, field_info in type(self).model_fields.items():
|
||||
if field_info.exclude:
|
||||
continue
|
||||
current_value = getattr(self, field_name)
|
||||
if current_value != field_info.default:
|
||||
user_modified[field_name] = current_value
|
||||
|
||||
try:
|
||||
with open(self._config_file, "wb") as f:
|
||||
tomli_w.dump(user_modified, f)
|
||||
logger.info(f"Config saved to {self._config_file}")
|
||||
runtime_cache.save_runtime_config(self)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save config to {self._config_file}: {e}")
|
||||
|
||||
def dump_config(self) -> dict[str, Any]:
|
||||
config_dict = self.model_dump()
|
||||
fields_info = {}
|
||||
for field_name, field_info in Config.model_fields.items():
|
||||
if field_info.exclude:
|
||||
continue
|
||||
fields_info[field_name] = {
|
||||
"des": field_info.description,
|
||||
"default": field_info.default,
|
||||
"type": field_info.annotation.__name__
|
||||
if hasattr(field_info.annotation, "__name__")
|
||||
else str(field_info.annotation),
|
||||
"exclude": field_info.exclude if hasattr(field_info, "exclude") else False,
|
||||
}
|
||||
config_dict["_config_items"] = fields_info
|
||||
return config_dict
|
||||
|
||||
def update(self, other: dict[str, Any]) -> None:
|
||||
for key, value in other.items():
|
||||
if self.can_update(key):
|
||||
self.set_value(key, value)
|
||||
elif key in READONLY_CONFIG_FIELDS:
|
||||
logger.warning(f"Readonly config key ignored: {key}")
|
||||
else:
|
||||
logger.warning(f"Unknown config key: {key}")
|
||||
|
||||
def can_update(self, key: object) -> bool:
|
||||
return isinstance(key, str) and key in type(self).model_fields and key not in READONLY_CONFIG_FIELDS
|
||||
|
||||
def set_value(self, key: str, value: Any) -> None:
|
||||
if not self.can_update(key):
|
||||
raise ValueError(f"配置项不可修改: {key}")
|
||||
setattr(self, key, self._normalize_config_value(key, value))
|
||||
|
||||
def _normalize_config_value(self, key: str, value: Any) -> Any:
|
||||
if key == "default_ocr_engine":
|
||||
return _normalize_default_ocr_engine(value)
|
||||
return value
|
||||
|
||||
|
||||
config = Config()
|
||||
@@ -0,0 +1,99 @@
|
||||
"""运行时配置 Redis 快照同步。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from yuxi.storage.redis import RedisConfig, sync_redis_client
|
||||
from yuxi.utils.logging_config import logger
|
||||
|
||||
RUNTIME_CONFIG_REDIS_KEY = "yuxi:runtime_config"
|
||||
RUNTIME_CONFIG_SYNC_INTERVAL_SECONDS = 5.0
|
||||
_RUNTIME_CONFIG_REDIS_TIMEOUT_SECONDS = 0.2
|
||||
|
||||
|
||||
def _runtime_config_redis_config() -> RedisConfig:
|
||||
return RedisConfig.from_env(
|
||||
decode_responses=True,
|
||||
socket_timeout=_RUNTIME_CONFIG_REDIS_TIMEOUT_SECONDS,
|
||||
socket_connect_timeout=_RUNTIME_CONFIG_REDIS_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _runtime_fields(config: Any) -> Iterator[str]:
|
||||
for field_name, field_info in type(config).model_fields.items():
|
||||
if field_info.exclude:
|
||||
continue
|
||||
yield field_name
|
||||
|
||||
|
||||
def _runtime_snapshot(config: Any) -> dict[str, Any]:
|
||||
return {field_name: getattr(config, field_name) for field_name in _runtime_fields(config)}
|
||||
|
||||
|
||||
def _load_snapshot() -> dict[str, Any] | None:
|
||||
try:
|
||||
with sync_redis_client(_runtime_config_redis_config()) as redis_client:
|
||||
raw = redis_client.get(RUNTIME_CONFIG_REDIS_KEY)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load runtime config from Redis: {e}")
|
||||
return None
|
||||
|
||||
if not raw:
|
||||
return None
|
||||
|
||||
try:
|
||||
snapshot = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"Failed to decode runtime config snapshot: {e}")
|
||||
return None
|
||||
|
||||
return snapshot if isinstance(snapshot, dict) else None
|
||||
|
||||
|
||||
def save_runtime_config(config: Any) -> None:
|
||||
try:
|
||||
with sync_redis_client(_runtime_config_redis_config()) as redis_client:
|
||||
redis_client.set(
|
||||
RUNTIME_CONFIG_REDIS_KEY,
|
||||
json.dumps(_runtime_snapshot(config), ensure_ascii=False),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save runtime config to Redis: {e}")
|
||||
|
||||
|
||||
def refresh_runtime_config(config: Any) -> None:
|
||||
snapshot = _load_snapshot()
|
||||
if snapshot is None:
|
||||
return
|
||||
|
||||
for field_name in _runtime_fields(config):
|
||||
if field_name in snapshot:
|
||||
setattr(config, field_name, snapshot[field_name])
|
||||
|
||||
|
||||
def start_runtime_sync(
|
||||
config: Any,
|
||||
current_thread: threading.Thread | None,
|
||||
*,
|
||||
interval: float = RUNTIME_CONFIG_SYNC_INTERVAL_SECONDS,
|
||||
) -> threading.Thread:
|
||||
if current_thread is not None:
|
||||
return current_thread
|
||||
|
||||
def runtime_sync_loop() -> None:
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
try:
|
||||
refresh_runtime_config(config)
|
||||
except Exception as e:
|
||||
logger.warning(f"Runtime config sync iteration failed: {e}")
|
||||
|
||||
thread = threading.Thread(target=runtime_sync_loop, name="config-runtime-sync", daemon=True)
|
||||
thread.start()
|
||||
logger.info(f"Runtime config sync thread started (interval={interval}s)")
|
||||
return thread
|
||||
@@ -0,0 +1,129 @@
|
||||
# ==========================================================
|
||||
# 有害内容过滤关键词列表
|
||||
# 说明:
|
||||
# 1. 本列表仅为基础示例，请根据您的具体应用场景进行修改和扩展。
|
||||
# 2. 很多词语具有多义性，请务必结合语境进行判断，以防误伤。
|
||||
# 3. 网络用语和暗语变化迅速，建议定期更新此列表。
|
||||
# ==========================================================
|
||||
|
||||
# ----------------------------------------
|
||||
# 1. 仇恨言论 (Hate Speech)
|
||||
# ----------------------------------------
|
||||
|
||||
# ** 种族与民族歧视 **
|
||||
尼哥
|
||||
黑鬼
|
||||
白皮猪
|
||||
棒子
|
||||
阿三
|
||||
|
||||
# ** 宗教歧视 **
|
||||
异教徒
|
||||
邪教
|
||||
|
||||
# ** 性别与性取向歧视 **
|
||||
死娘炮
|
||||
男人婆
|
||||
同性恋不得好死
|
||||
恐同
|
||||
|
||||
|
||||
# ----------------------------------------
|
||||
# 2. 暴力与极端内容 (Violence & Extremism)
|
||||
# ----------------------------------------
|
||||
|
||||
# ** 煽动暴力 **
|
||||
杀全家
|
||||
弄死他
|
||||
打断腿
|
||||
血洗
|
||||
人肉他
|
||||
|
||||
# ** 恐怖主义与极端思想 **
|
||||
圣战
|
||||
ISIS
|
||||
东突
|
||||
|
||||
# ** 血腥与残忍 **
|
||||
虐杀
|
||||
活剐
|
||||
分尸
|
||||
电锯惊魂
|
||||
|
||||
# ** 武器与爆炸物 **
|
||||
土制炸弹
|
||||
买枪
|
||||
军火
|
||||
C4炸药
|
||||
炸药爆
|
||||
制作炸药
|
||||
制作武器
|
||||
炸弹
|
||||
|
||||
# ----------------------------------------
|
||||
# 3. 非法活动与违禁品 (Illegal Activities)
|
||||
# ----------------------------------------
|
||||
|
||||
# ** 毒品 **
|
||||
冰毒
|
||||
K粉
|
||||
海洛因
|
||||
吸毒
|
||||
贩毒
|
||||
摇头丸
|
||||
毒品走私
|
||||
|
||||
|
||||
# ** 赌博 **
|
||||
线上赌场
|
||||
澳门首家
|
||||
性感荷官
|
||||
百家乐
|
||||
六合彩
|
||||
|
||||
# ** 诈骗 **
|
||||
杀猪盘
|
||||
刷单兼职
|
||||
网贷陷阱
|
||||
冒充公检法
|
||||
|
||||
# ** 违禁品交易 **
|
||||
出售个人信息
|
||||
办假证
|
||||
针孔摄像头
|
||||
窃听器
|
||||
|
||||
|
||||
# ----------------------------------------
|
||||
# 4. 色情与成人内容 (Pornography)
|
||||
# ----------------------------------------
|
||||
AV女优
|
||||
草榴
|
||||
91大神
|
||||
黄片
|
||||
色情网站
|
||||
约炮
|
||||
裸聊
|
||||
福利姬
|
||||
|
||||
# ----------------------------------------
|
||||
# 5. 自残与危险行为 (Self-harm)
|
||||
# ----------------------------------------
|
||||
自杀教程
|
||||
割腕
|
||||
烧炭
|
||||
无痛自杀
|
||||
一起死
|
||||
|
||||
# ----------------------------------------
|
||||
# 6. 网络欺凌与骚扰 (Cyberbullying)
|
||||
# ----------------------------------------
|
||||
biss
|
||||
nmsl
|
||||
开盒
|
||||
挂人
|
||||
孤儿
|
||||
|
||||
# ==========================================================
|
||||
# 列表结束
|
||||
# ==========================================================
|
||||
@@ -0,0 +1,27 @@
|
||||
# 单位信息配置文件
|
||||
# 用于配置网站的基本信息、品牌信息等
|
||||
|
||||
# 组织信息
|
||||
organization:
|
||||
name: "江南语析" # 完整组织名称
|
||||
logo: "/favicon.svg" # Logo文件路径（放在 web/public 目录下）
|
||||
avatar: "/avatar.jpg" # 头像文件路径（放在 web/public 目录下）
|
||||
login_bg: "/login-bg.jpg" # 登录背景图片路径（放在 web/public 目录下）
|
||||
|
||||
# 项目信息
|
||||
branding:
|
||||
name: "Yuxi"
|
||||
title: "让智能体可构建、可编排、可落地" # 系统标题
|
||||
subtitle: "开源智能体平台套件，融合 RAG 与知识图谱" # 副标题（subtitles 为空时使用）
|
||||
subtitles:
|
||||
- "开源智能体平台套件，融合 RAG 与知识图谱"
|
||||
- "统一编排 Agent、知识库、图谱与工具链"
|
||||
- "让智能体可构建，让知识可连接，让决策可验证"
|
||||
- "让智能体可落地，让流程可编排，让协作可扩展"
|
||||
- "让数据可沉淀，让能力可复用，让系统可进化"
|
||||
|
||||
# 页脚信息
|
||||
footer:
|
||||
copyright: "© 江南语析 2026 v{{YUXI_VERSION}}"
|
||||
user_agreement_url: "/protocols/user-agreement.template.html"
|
||||
privacy_policy_url: "/protocols/privacy-policy.template.html"
|
||||
@@ -0,0 +1,80 @@
|
||||
"""用户级配置模块。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from yuxi.storage.postgres.models_business import UserConfig as UserConfigRecord
|
||||
from yuxi.utils.datetime_utils import format_utc_datetime, utc_now_naive
|
||||
|
||||
|
||||
class UserConfigSchema(BaseModel):
|
||||
"""用户专属配置 schema。"""
|
||||
|
||||
enable_memory: bool = Field(default=False, description="是否启用 Memory")
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class UserConfig:
|
||||
"""用户级配置访问器。每次加载都从 PostgreSQL 查询，不做进程缓存。"""
|
||||
|
||||
def __init__(self, uid: str, schema: UserConfigSchema | None = None, updated_at: datetime | None = None):
|
||||
self.uid = uid
|
||||
self.schema = schema or UserConfigSchema()
|
||||
self.updated_at = updated_at
|
||||
|
||||
@classmethod
|
||||
async def load(cls, db: AsyncSession, uid: str) -> UserConfig:
|
||||
result = await db.execute(select(UserConfigRecord).filter(UserConfigRecord.uid == uid))
|
||||
record = result.scalar_one_or_none()
|
||||
if record is None:
|
||||
return cls(uid=uid)
|
||||
return cls(
|
||||
uid=uid,
|
||||
schema=UserConfigSchema(enable_memory=bool(record.enable_memory)),
|
||||
updated_at=record.updated_at,
|
||||
)
|
||||
|
||||
async def save(self, db: AsyncSession) -> UserConfig:
|
||||
now = utc_now_naive()
|
||||
result = await db.execute(
|
||||
update(UserConfigRecord)
|
||||
.where(UserConfigRecord.uid == self.uid)
|
||||
.values(enable_memory=self.schema.enable_memory, updated_at=now)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
db.add(
|
||||
UserConfigRecord(
|
||||
uid=self.uid,
|
||||
enable_memory=self.schema.enable_memory,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
try:
|
||||
await db.commit()
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
now = utc_now_naive()
|
||||
retry_result = await db.execute(
|
||||
update(UserConfigRecord)
|
||||
.where(UserConfigRecord.uid == self.uid)
|
||||
.values(enable_memory=self.schema.enable_memory, updated_at=now)
|
||||
)
|
||||
if retry_result.rowcount == 0:
|
||||
raise
|
||||
await db.commit()
|
||||
return type(self)(uid=self.uid, schema=self.schema, updated_at=now)
|
||||
|
||||
def dump_config(self) -> dict[str, str | bool | None]:
|
||||
return {
|
||||
"uid": self.uid,
|
||||
"enable_memory": self.schema.enable_memory,
|
||||
"updated_at": format_utc_datetime(self.updated_at),
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import os
|
||||
|
||||
from ..config import config
|
||||
from .factory import KnowledgeBaseFactory
|
||||
from .implementations.dify import DifyKB
|
||||
from .implementations.milvus import MilvusKB
|
||||
from .implementations.notion import NotionKB
|
||||
from .manager import KnowledgeBaseManager
|
||||
|
||||
_LITE_MODE = os.environ.get("LITE_MODE", "").lower() in ("true", "1")
|
||||
_SKIP_APP_INIT = os.environ.get("YUXI_SKIP_APP_INIT") == "1"
|
||||
|
||||
if not _LITE_MODE:
|
||||
# 注册知识库类型
|
||||
KnowledgeBaseFactory.register(MilvusKB)
|
||||
|
||||
KnowledgeBaseFactory.register(DifyKB)
|
||||
KnowledgeBaseFactory.register(NotionKB)
|
||||
|
||||
# 创建知识库管理器
|
||||
work_dir = os.path.join(config.save_dir, "knowledge_base_data")
|
||||
knowledge_base = KnowledgeBaseManager(work_dir)
|
||||
|
||||
__all__ = ["knowledge_base"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
__all__ = []
|
||||
@@ -0,0 +1,3 @@
|
||||
from yuxi.knowledge.chunking.ragflow_like.dispatcher import chunk_file, chunk_markdown
|
||||
|
||||
__all__ = ["chunk_file", "chunk_markdown"]
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa, semantic, separator
|
||||
from yuxi.knowledge.chunking.ragflow_like.presets import map_to_internal_parser_id, normalize_chunk_preset_id
|
||||
|
||||
|
||||
def _build_chunk_records(
|
||||
text_chunks: list[str], file_id: str, filename: str, source_text: str | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
records: list[dict[str, Any]] = []
|
||||
search_from = 0
|
||||
|
||||
for idx, chunk_content in enumerate(text_chunks):
|
||||
text = (chunk_content or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
start_char_pos = None
|
||||
end_char_pos = None
|
||||
if source_text:
|
||||
found_at = source_text.find(text, search_from)
|
||||
if found_at >= 0:
|
||||
start_char_pos = found_at
|
||||
end_char_pos = found_at + len(text)
|
||||
search_from = end_char_pos
|
||||
|
||||
records.append(
|
||||
{
|
||||
"id": f"{file_id}_chunk_{idx}",
|
||||
"content": text,
|
||||
"file_id": file_id,
|
||||
"filename": filename,
|
||||
"chunk_index": idx,
|
||||
"source": filename,
|
||||
"chunk_id": f"{file_id}_chunk_{idx}",
|
||||
"start_char_pos": start_char_pos,
|
||||
"end_char_pos": end_char_pos,
|
||||
"start_token_pos": None,
|
||||
"end_token_pos": None,
|
||||
"extraction_result": None,
|
||||
}
|
||||
)
|
||||
|
||||
return records
|
||||
|
||||
|
||||
def _dispatch_markdown_parser(
|
||||
preset_id: str, filename: str, markdown_content: str, parser_config: dict[str, Any]
|
||||
) -> list[str]:
|
||||
parser_id = map_to_internal_parser_id(preset_id)
|
||||
|
||||
if parser_id == "naive":
|
||||
return general.chunk_markdown(markdown_content, parser_config)
|
||||
if parser_id == "qa":
|
||||
return qa.chunk_markdown(filename, markdown_content, parser_config)
|
||||
if parser_id == "book":
|
||||
return book.chunk_markdown(markdown_content, parser_config)
|
||||
if parser_id == "laws":
|
||||
return laws.chunk_markdown(filename, markdown_content, parser_config)
|
||||
if parser_id == "semantic":
|
||||
return semantic.chunk_markdown(markdown_content, parser_config)
|
||||
if parser_id == "separator":
|
||||
return separator.chunk_markdown(markdown_content, parser_config)
|
||||
|
||||
return general.chunk_markdown(markdown_content, parser_config)
|
||||
|
||||
|
||||
def chunk_markdown(
|
||||
markdown_content: str, file_id: str, filename: str, processing_params: dict[str, Any]
|
||||
) -> list[dict[str, Any]]:
|
||||
params = dict(processing_params or {})
|
||||
preset_id = normalize_chunk_preset_id(params.get("chunk_preset_id"))
|
||||
parser_config = params.get("chunk_parser_config") if isinstance(params.get("chunk_parser_config"), dict) else {}
|
||||
|
||||
text_chunks = _dispatch_markdown_parser(preset_id, filename, markdown_content, parser_config)
|
||||
return _build_chunk_records(text_chunks, file_id, filename, markdown_content)
|
||||
|
||||
|
||||
def chunk_file(
|
||||
file_content: str, file_id: str, filename: str, processing_params: dict[str, Any]
|
||||
) -> list[dict[str, Any]]:
|
||||
# 当前链路中入库前均已转换为 markdown，因此与 chunk_markdown 保持同实现。
|
||||
return chunk_markdown(file_content, file_id, filename, processing_params)
|
||||
@@ -0,0 +1,583 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
BULLET_PATTERN = [
|
||||
[
|
||||
r"第[零一二三四五六七八九十百0-9]+(分?编|部分)",
|
||||
r"第[零一二三四五六七八九十百0-9]+章",
|
||||
r"第[零一二三四五六七八九十百0-9]+节",
|
||||
r"第[零一二三四五六七八九十百0-9]+条",
|
||||
r"[\(（][零一二三四五六七八九十百]+[\)）]",
|
||||
],
|
||||
[
|
||||
r"第[0-9]+章",
|
||||
r"第[0-9]+节",
|
||||
r"[0-9]{,2}[\. 、]",
|
||||
r"[0-9]{,2}\.[0-9]{,2}[^a-zA-Z/%~-]",
|
||||
r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}",
|
||||
r"[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}\.[0-9]{,2}",
|
||||
],
|
||||
[
|
||||
r"第[零一二三四五六七八九十百0-9]+章",
|
||||
r"第[零一二三四五六七八九十百0-9]+节",
|
||||
r"[零一二三四五六七八九十百]+[ 、]",
|
||||
r"[\(（][零一二三四五六七八九十百]+[\)）]",
|
||||
r"[\(（][0-9]{,2}[\)）]",
|
||||
],
|
||||
[
|
||||
r"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)",
|
||||
r"Chapter (I+V?|VI*|XI|IX|X)",
|
||||
r"Section [0-9]+",
|
||||
r"Article [0-9]+",
|
||||
],
|
||||
[
|
||||
r"^#[^#]",
|
||||
r"^##[^#]",
|
||||
r"^###.*",
|
||||
r"^####.*",
|
||||
r"^#####.*",
|
||||
r"^######.*",
|
||||
],
|
||||
]
|
||||
|
||||
MARKDOWN_BULLET_GROUP_INDEX = 4
|
||||
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
"""近似 token 计数，避免引入额外依赖。"""
|
||||
if not text:
|
||||
return 0
|
||||
# 英文单词 + 数字 + CJK 单字
|
||||
parts = re.findall(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]", text)
|
||||
return max(1, len(parts)) if text.strip() else 0
|
||||
|
||||
|
||||
def hard_split_by_token_limit(text: str, chunk_token_num: int, hard_limit_token_num: int | None = None) -> list[str]:
|
||||
"""将文本按 token 上限硬切，用于 naive_merge 之后的兜底保护。
|
||||
|
||||
hard_limit_token_num 只在调用方显式传入时生效，用于允许略超目标长度的块
|
||||
保持完整；默认保持严格不超过 chunk_token_num 的历史行为。
|
||||
"""
|
||||
token_iter = list(re.finditer(r"[A-Za-z0-9_]+|[一-鿿]", text or ""))
|
||||
if not token_iter:
|
||||
cleaned = (text or "").strip()
|
||||
return [cleaned] if cleaned else []
|
||||
|
||||
max_tokens = max(int(chunk_token_num or 0), 1)
|
||||
hard_limit = None
|
||||
if hard_limit_token_num is not None:
|
||||
hard_limit = max(int(hard_limit_token_num or 0), max_tokens)
|
||||
if len(token_iter) <= hard_limit:
|
||||
cleaned = (text or "").strip()
|
||||
return [cleaned] if cleaned else []
|
||||
|
||||
spans: list[tuple[int, int]] = []
|
||||
start = 0
|
||||
index = 0
|
||||
|
||||
while index < len(token_iter):
|
||||
next_index = min(index + max_tokens, len(token_iter))
|
||||
if next_index < len(token_iter):
|
||||
end = token_iter[next_index].start()
|
||||
else:
|
||||
end = len(text)
|
||||
if text[start:end].strip():
|
||||
spans.append((start, end))
|
||||
start = end
|
||||
index = next_index
|
||||
|
||||
tail = text[start:].strip()
|
||||
if tail:
|
||||
spans.append((start, len(text)))
|
||||
|
||||
if hard_limit is not None and len(spans) >= 2:
|
||||
prev_start, _ = spans[-2]
|
||||
_, tail_end = spans[-1]
|
||||
candidate = text[prev_start:tail_end].strip()
|
||||
if count_tokens(candidate) <= hard_limit:
|
||||
spans[-2] = (prev_start, tail_end)
|
||||
spans.pop()
|
||||
|
||||
return [text[start:end].strip() for start, end in spans if text[start:end].strip()]
|
||||
|
||||
|
||||
def random_choices(arr: list[str], k: int) -> list[str]:
|
||||
if not arr:
|
||||
return []
|
||||
return random.choices(arr, k=min(len(arr), k))
|
||||
|
||||
|
||||
def is_english(texts: str | list[str]) -> bool:
|
||||
if not texts:
|
||||
return False
|
||||
|
||||
patt = re.compile(r"[`a-zA-Z0-9\s.,':;/\"?<>!\(\)\-]+")
|
||||
if isinstance(texts, str):
|
||||
seq = [texts]
|
||||
else:
|
||||
seq = [t for t in texts if isinstance(t, str) and t.strip()]
|
||||
|
||||
if not seq:
|
||||
return False
|
||||
|
||||
hits = sum(1 for t in seq if patt.fullmatch(t.strip()))
|
||||
return (hits / len(seq)) > 0.8
|
||||
|
||||
|
||||
def not_bullet(line: str) -> bool:
|
||||
patt = [
|
||||
r"0",
|
||||
r"[0-9]+ +[0-9~个只-]",
|
||||
r"[0-9]+\.{2,}",
|
||||
]
|
||||
return any(re.match(p, line) for p in patt)
|
||||
|
||||
|
||||
def is_probable_heading_line(line: str) -> bool:
|
||||
text = (line or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
if re.match(r"^#{1,6}\s+\S", text):
|
||||
return True
|
||||
|
||||
# 表格/HTML 残留通常不是标题。
|
||||
if re.search(r"</?(table|tr|td|th|caption|tbody|thead)[^>]*>", text, flags=re.IGNORECASE):
|
||||
return False
|
||||
|
||||
# 超长行基本是正文或条款，不是章节标题。
|
||||
if len(text) > 96:
|
||||
return False
|
||||
if count_tokens(text) > 72:
|
||||
return False
|
||||
|
||||
# 标题前段通常不会出现明显句号/逗号；出现则大概率是正文。
|
||||
if re.search(r"[，。；！？!?:：]", text[:24]):
|
||||
return False
|
||||
|
||||
if text.endswith(("。", "；", "！", "!", "？", "?")) and len(text) > 20:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _is_mid_sentence_bullet(line: str) -> bool:
|
||||
text = (line or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
if re.match(r"^#{1,6}\s+\S", text):
|
||||
return False
|
||||
|
||||
marker = re.search(
|
||||
r"([一二三四五六七八九十百]+、|[\(（][一二三四五六七八九十百]+[\)）]|[0-9]{1,2}[\.、])",
|
||||
text,
|
||||
)
|
||||
if not marker:
|
||||
return False
|
||||
|
||||
if marker.start() == 0:
|
||||
return False
|
||||
|
||||
prev = text[marker.start() - 1]
|
||||
return prev not in {"#", "\n"}
|
||||
|
||||
|
||||
def bullets_category(sections: list[str]) -> int:
|
||||
hits: list[float] = [0.0] * len(BULLET_PATTERN)
|
||||
|
||||
def bullet_weight(group_idx: int, line: str) -> float:
|
||||
# 对 markdown 标题候选增加权重，避免“正文里的 一、/（一）”压过真正的 # 标题层级。
|
||||
if group_idx != MARKDOWN_BULLET_GROUP_INDEX:
|
||||
return 1.0
|
||||
|
||||
heading = line.strip()
|
||||
if not re.match(r"^#{1,6}\s+\S", heading):
|
||||
return 1.0
|
||||
|
||||
level = len(heading) - len(heading.lstrip("#"))
|
||||
if level <= 2:
|
||||
return 4.0
|
||||
if level <= 4:
|
||||
return 3.0
|
||||
return 2.0
|
||||
|
||||
for i, pro in enumerate(BULLET_PATTERN):
|
||||
for sec in sections:
|
||||
sec = sec.strip()
|
||||
for p in pro:
|
||||
if re.match(p, sec) and not not_bullet(sec):
|
||||
w = bullet_weight(i, sec)
|
||||
if _is_mid_sentence_bullet(sec):
|
||||
w *= 0.1
|
||||
if i != MARKDOWN_BULLET_GROUP_INDEX and not is_probable_heading_line(sec):
|
||||
w *= 0.2
|
||||
hits[i] += w
|
||||
break
|
||||
maximum = 0
|
||||
res = -1
|
||||
for i, hit in enumerate(hits):
|
||||
if hit <= maximum:
|
||||
continue
|
||||
res = i
|
||||
maximum = hit
|
||||
return res
|
||||
|
||||
|
||||
def _get_text(section: str | tuple[str, str]) -> str:
|
||||
if isinstance(section, str):
|
||||
return section.strip()
|
||||
return (section[0] or "").strip()
|
||||
|
||||
|
||||
def remove_contents_table(sections: list[str] | list[tuple[str, str]], eng: bool = False) -> None:
|
||||
i = 0
|
||||
while i < len(sections):
|
||||
line = re.sub(r"( | |\u3000)+", "", _get_text(sections[i]).split("@@")[0], flags=re.IGNORECASE)
|
||||
if not re.match(r"(contents|目录|目次|tableofcontents|致谢|acknowledge)$", line, flags=re.IGNORECASE):
|
||||
i += 1
|
||||
continue
|
||||
|
||||
sections.pop(i)
|
||||
if i >= len(sections):
|
||||
break
|
||||
|
||||
prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2])
|
||||
while not prefix and i < len(sections):
|
||||
sections.pop(i)
|
||||
if i >= len(sections):
|
||||
break
|
||||
prefix = _get_text(sections[i])[:3] if not eng else " ".join(_get_text(sections[i]).split()[:2])
|
||||
|
||||
if i >= len(sections) or not prefix:
|
||||
break
|
||||
|
||||
sections.pop(i)
|
||||
if i >= len(sections):
|
||||
break
|
||||
|
||||
for j in range(i, min(i + 128, len(sections))):
|
||||
if not re.match(re.escape(prefix), _get_text(sections[j])):
|
||||
continue
|
||||
for _ in range(i, j):
|
||||
sections.pop(i)
|
||||
break
|
||||
|
||||
|
||||
def make_colon_as_title(sections: list[str] | list[tuple[str, str]]) -> list[str] | list[tuple[str, str]]:
|
||||
if not sections:
|
||||
return sections
|
||||
if isinstance(sections[0], str):
|
||||
return sections
|
||||
|
||||
i = 0
|
||||
while i < len(sections):
|
||||
text, layout = sections[i]
|
||||
i += 1
|
||||
text = text.split("@")[0].strip()
|
||||
if not text or text[-1] not in ":：":
|
||||
continue
|
||||
|
||||
rev = text[::-1]
|
||||
arr = re.split(r"([。？！!?;；]| \.)", rev)
|
||||
if len(arr) < 2 or len(arr[1]) < 32:
|
||||
continue
|
||||
|
||||
sections.insert(i - 1, (arr[0][::-1], "title"))
|
||||
i += 1
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def not_title(text: str) -> bool:
|
||||
if re.match(r"第[零一二三四五六七八九十百0-9]+条", text):
|
||||
return False
|
||||
if len(text.split()) > 12 or (" " not in text and len(text) >= 32):
|
||||
return True
|
||||
return bool(re.search(r"[,;，。；！!]", text))
|
||||
|
||||
|
||||
def tree_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[str]:
|
||||
if not sections or bull < 0:
|
||||
return [s if isinstance(s, str) else s[0] for s in sections]
|
||||
|
||||
if isinstance(sections[0], str):
|
||||
typed_sections: list[tuple[str, str]] = [(s, "") for s in sections]
|
||||
else:
|
||||
typed_sections = sections # type: ignore[assignment]
|
||||
|
||||
typed_sections = [
|
||||
(t, o)
|
||||
for t, o in typed_sections
|
||||
if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip())
|
||||
]
|
||||
|
||||
def get_level(section: tuple[str, str]) -> tuple[int, str]:
|
||||
text, layout = section
|
||||
text = re.sub(r"\u3000", " ", text).strip()
|
||||
|
||||
for i, patt in enumerate(BULLET_PATTERN[bull]):
|
||||
if re.match(patt, text) and is_probable_heading_line(text):
|
||||
return i + 1, text
|
||||
|
||||
if re.search(r"(title|head)", layout) and not not_title(text):
|
||||
return len(BULLET_PATTERN[bull]) + 1, text
|
||||
|
||||
return len(BULLET_PATTERN[bull]) + 2, text
|
||||
|
||||
lines: list[tuple[int, str]] = []
|
||||
level_set: set[int] = set()
|
||||
for section in typed_sections:
|
||||
level, text = get_level(section)
|
||||
if not text.strip("\n"):
|
||||
continue
|
||||
lines.append((level, text))
|
||||
level_set.add(level)
|
||||
|
||||
if not lines:
|
||||
return []
|
||||
|
||||
sorted_levels = sorted(level_set)
|
||||
target_level = sorted_levels[depth - 1] if depth <= len(sorted_levels) else sorted_levels[-1]
|
||||
|
||||
max_body_level = len(BULLET_PATTERN[bull]) + 2
|
||||
if target_level == max_body_level:
|
||||
target_level = sorted_levels[-2] if len(sorted_levels) > 1 else sorted_levels[0]
|
||||
|
||||
root = Node(level=0, depth=target_level, texts=[])
|
||||
root.build_tree(lines)
|
||||
return [item for item in root.get_tree() if item]
|
||||
|
||||
|
||||
def hierarchical_merge(bull: int, sections: list[str] | list[tuple[str, str]], depth: int) -> list[list[str]]:
|
||||
if not sections or bull < 0:
|
||||
return []
|
||||
|
||||
if isinstance(sections[0], str):
|
||||
typed_sections: list[tuple[str, str]] = [(s, "") for s in sections]
|
||||
else:
|
||||
typed_sections = sections # type: ignore[assignment]
|
||||
|
||||
typed_sections = [
|
||||
(t, o)
|
||||
for t, o in typed_sections
|
||||
if t and len(t.split("@")[0].strip()) > 1 and not re.match(r"[0-9]+$", t.split("@")[0].strip())
|
||||
]
|
||||
|
||||
bullets_size = len(BULLET_PATTERN[bull])
|
||||
levels: list[list[int]] = [[] for _ in range(bullets_size + 2)]
|
||||
|
||||
for i, (text, layout) in enumerate(typed_sections):
|
||||
for j, patt in enumerate(BULLET_PATTERN[bull]):
|
||||
if re.match(patt, text.strip()) and is_probable_heading_line(text):
|
||||
levels[j].append(i)
|
||||
break
|
||||
else:
|
||||
if re.search(r"(title|head)", layout) and not not_title(text):
|
||||
levels[bullets_size].append(i)
|
||||
else:
|
||||
levels[bullets_size + 1].append(i)
|
||||
|
||||
pure_sections = [t for t, _ in typed_sections]
|
||||
|
||||
def binary_search(arr: list[int], target: int) -> int:
|
||||
if not arr:
|
||||
return -1
|
||||
if target > arr[-1]:
|
||||
return len(arr) - 1
|
||||
if target < arr[0]:
|
||||
return -1
|
||||
|
||||
s, e = 0, len(arr)
|
||||
while e - s > 1:
|
||||
mid = (e + s) // 2
|
||||
if target > arr[mid]:
|
||||
s = mid
|
||||
elif target < arr[mid]:
|
||||
e = mid
|
||||
else:
|
||||
return mid
|
||||
return s
|
||||
|
||||
cks: list[list[int]] = []
|
||||
readed = [False] * len(pure_sections)
|
||||
levels = list(reversed(levels))
|
||||
for i, arr in enumerate(levels[:depth]):
|
||||
for j in arr:
|
||||
if readed[j]:
|
||||
continue
|
||||
readed[j] = True
|
||||
cks.append([j])
|
||||
if i + 1 == len(levels) - 1:
|
||||
continue
|
||||
|
||||
for ii in range(i + 1, len(levels)):
|
||||
jj = binary_search(levels[ii], j)
|
||||
if jj < 0:
|
||||
continue
|
||||
if levels[ii][jj] > cks[-1][-1]:
|
||||
cks[-1].pop(-1)
|
||||
cks[-1].append(levels[ii][jj])
|
||||
|
||||
for ii in cks[-1]:
|
||||
readed[ii] = True
|
||||
|
||||
if not cks:
|
||||
return []
|
||||
|
||||
for i in range(len(cks)):
|
||||
cks[i] = [pure_sections[j] for j in reversed(cks[i])]
|
||||
|
||||
res: list[list[str]] = [[]]
|
||||
num = [0]
|
||||
for ck in cks:
|
||||
if len(ck) == 1:
|
||||
n = count_tokens(re.sub(r"@@[0-9]+.*", "", ck[0]))
|
||||
if n + num[-1] < 218:
|
||||
res[-1].append(ck[0])
|
||||
num[-1] += n
|
||||
continue
|
||||
res.append(ck)
|
||||
num.append(n)
|
||||
continue
|
||||
res.append(ck)
|
||||
num.append(218)
|
||||
|
||||
return [chunk for chunk in res if chunk]
|
||||
|
||||
|
||||
def _remove_pdf_tags(text: str) -> str:
|
||||
return re.sub(r"@@[0-9-]+\t[0-9.\t]+##", "", text or "")
|
||||
|
||||
|
||||
def _extract_custom_delimiters(delimiter: str) -> list[str]:
|
||||
return [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter or "")]
|
||||
|
||||
|
||||
def naive_merge(
|
||||
sections: str | list[str] | list[tuple[str, str]],
|
||||
chunk_token_num: int = 128,
|
||||
delimiter: str = "\n。；！？",
|
||||
overlapped_percent: int = 0,
|
||||
) -> list[str]:
|
||||
if not sections:
|
||||
return []
|
||||
|
||||
if isinstance(sections, str):
|
||||
typed_sections: list[tuple[str, str]] = [(sections, "")]
|
||||
elif isinstance(sections[0], str):
|
||||
typed_sections = [(s, "") for s in sections] # type: ignore[index]
|
||||
else:
|
||||
typed_sections = sections # type: ignore[assignment]
|
||||
|
||||
chunk_token_num = max(int(chunk_token_num or 0), 0)
|
||||
overlap = max(0, min(int(overlapped_percent or 0), 99))
|
||||
|
||||
custom_delimiters = _extract_custom_delimiters(delimiter)
|
||||
if custom_delimiters:
|
||||
pattern = "|".join(re.escape(t) for t in sorted(set(custom_delimiters), key=len, reverse=True))
|
||||
chunks: list[str] = []
|
||||
for sec, pos in typed_sections:
|
||||
split_sec = re.split(rf"({pattern})", sec, flags=re.DOTALL)
|
||||
for sub in split_sec:
|
||||
if re.fullmatch(pattern, sub or ""):
|
||||
continue
|
||||
text = "\n" + sub
|
||||
local_pos = pos if count_tokens(text) >= 8 else ""
|
||||
if local_pos and local_pos not in text:
|
||||
text += local_pos
|
||||
if text.strip():
|
||||
chunks.append(text)
|
||||
return chunks
|
||||
|
||||
if chunk_token_num <= 0:
|
||||
merged = "\n".join(sec for sec, _ in typed_sections if sec and sec.strip())
|
||||
return [merged] if merged.strip() else []
|
||||
|
||||
chunks = [""]
|
||||
token_nums = [0]
|
||||
|
||||
def add_chunk(text: str, pos: str) -> None:
|
||||
tnum = count_tokens(text)
|
||||
local_pos = pos or ""
|
||||
if tnum < 8:
|
||||
local_pos = ""
|
||||
|
||||
threshold = chunk_token_num * (100 - overlap) / 100.0
|
||||
if chunks[-1] == "" or token_nums[-1] > threshold:
|
||||
if chunks:
|
||||
prev = _remove_pdf_tags(chunks[-1])
|
||||
start = int(len(prev) * (100 - overlap) / 100.0)
|
||||
text = prev[start:] + text
|
||||
if local_pos and local_pos not in text:
|
||||
text += local_pos
|
||||
chunks.append(text)
|
||||
token_nums.append(tnum)
|
||||
else:
|
||||
if local_pos and local_pos not in chunks[-1]:
|
||||
text += local_pos
|
||||
chunks[-1] += text
|
||||
token_nums[-1] += tnum
|
||||
|
||||
for sec, pos in typed_sections:
|
||||
if not sec:
|
||||
continue
|
||||
add_chunk("\n" + sec, pos)
|
||||
|
||||
return [chunk for chunk in chunks if chunk.strip()]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Node:
|
||||
level: int
|
||||
depth: int = -1
|
||||
texts: list[str] = field(default_factory=list)
|
||||
children: list[Node] = field(default_factory=list)
|
||||
|
||||
def add_child(self, child_node: Node) -> None:
|
||||
self.children.append(child_node)
|
||||
|
||||
def add_text(self, text: str) -> None:
|
||||
self.texts.append(text)
|
||||
|
||||
def build_tree(self, lines: list[tuple[int, str]]) -> Node:
|
||||
stack: list[Node] = [self]
|
||||
for level, text in lines:
|
||||
if self.depth != -1 and level > self.depth:
|
||||
stack[-1].add_text(text)
|
||||
continue
|
||||
|
||||
while len(stack) > 1 and level <= stack[-1].level:
|
||||
stack.pop()
|
||||
|
||||
node = Node(level=level, texts=[text])
|
||||
stack[-1].add_child(node)
|
||||
stack.append(node)
|
||||
|
||||
return self
|
||||
|
||||
def get_tree(self) -> list[str]:
|
||||
tree_list: list[str] = []
|
||||
self._dfs(self, tree_list, [])
|
||||
return tree_list
|
||||
|
||||
def _dfs(self, node: Node, tree_list: list[str], titles: list[str]) -> None:
|
||||
level = node.level
|
||||
texts = node.texts
|
||||
child = node.children
|
||||
|
||||
if level == 0 and texts:
|
||||
tree_list.append("\n".join(titles + texts))
|
||||
|
||||
path_titles = titles + texts if 1 <= level <= self.depth else titles
|
||||
|
||||
if level > self.depth and texts:
|
||||
tree_list.append("\n".join(path_titles + texts))
|
||||
elif not child and (1 <= level <= self.depth):
|
||||
tree_list.append("\n".join(path_titles))
|
||||
|
||||
for c in child:
|
||||
self._dfs(c, tree_list, path_titles)
|
||||
@@ -0,0 +1,3 @@
|
||||
from yuxi.knowledge.chunking.ragflow_like.parsers import book, general, laws, qa, separator
|
||||
|
||||
__all__ = ["general", "qa", "book", "laws", "separator"]
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like import nlp
|
||||
|
||||
|
||||
def _unescape_delimiter(delimiter: str) -> str:
|
||||
return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\")
|
||||
|
||||
|
||||
def _iter_sections(markdown_content: str) -> list[tuple[str, str]]:
|
||||
sections: list[tuple[str, str]] = []
|
||||
for line in (markdown_content or "").splitlines():
|
||||
text = line.strip()
|
||||
if not text:
|
||||
continue
|
||||
sections.append((text, ""))
|
||||
|
||||
if not sections and markdown_content and markdown_content.strip():
|
||||
sections.append((markdown_content.strip(), ""))
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def _ensure_chunk_token_limit(chunks: list[str], chunk_token_num: int) -> list[str]:
|
||||
max_tokens = int(chunk_token_num or 0)
|
||||
normalized = [chunk.strip() for chunk in chunks if chunk and chunk.strip()]
|
||||
if max_tokens <= 0:
|
||||
return normalized
|
||||
|
||||
protected: list[str] = []
|
||||
for chunk in normalized:
|
||||
if nlp.count_tokens(chunk) <= max_tokens:
|
||||
protected.append(chunk)
|
||||
else:
|
||||
protected.extend(nlp.hard_split_by_token_limit(chunk, max_tokens))
|
||||
return protected
|
||||
|
||||
|
||||
def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
|
||||
parser_config = parser_config or {}
|
||||
|
||||
delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n"))
|
||||
chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512)
|
||||
overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0)
|
||||
|
||||
sections = _iter_sections(markdown_content)
|
||||
if not sections:
|
||||
return []
|
||||
|
||||
section_texts = [text for text, _ in sections]
|
||||
nlp.remove_contents_table(sections, eng=nlp.is_english(nlp.random_choices(section_texts, k=200)))
|
||||
nlp.make_colon_as_title(sections)
|
||||
|
||||
bull = nlp.bullets_category([t for t in nlp.random_choices([t for t, _ in sections], k=100)])
|
||||
|
||||
if bull >= 0:
|
||||
chunks = ["\n".join(ck) for ck in nlp.hierarchical_merge(bull, sections, depth=5)]
|
||||
else:
|
||||
chunks = nlp.naive_merge(
|
||||
sections,
|
||||
chunk_token_num=chunk_token_num,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
)
|
||||
|
||||
if chunks:
|
||||
return _ensure_chunk_token_limit(chunks, chunk_token_num)
|
||||
|
||||
return _ensure_chunk_token_limit(
|
||||
nlp.naive_merge(
|
||||
sections,
|
||||
chunk_token_num=chunk_token_num,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
),
|
||||
chunk_token_num,
|
||||
)
|
||||
@@ -0,0 +1,68 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like import nlp
|
||||
|
||||
GENERAL_HARD_LIMIT_RATIO = 1.5
|
||||
|
||||
|
||||
def _unescape_delimiter(delimiter: str) -> str:
|
||||
return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\")
|
||||
|
||||
|
||||
def _iter_sections(markdown_content: str, delimiter: str) -> list[tuple[str, str]]:
|
||||
sections: list[tuple[str, str]] = []
|
||||
text = markdown_content or ""
|
||||
if delimiter and delimiter not in {"\n", "\r\n"} and "`" not in delimiter:
|
||||
for part in text.split(delimiter):
|
||||
block = part.strip()
|
||||
if block:
|
||||
sections.append((block, ""))
|
||||
else:
|
||||
for line in text.splitlines():
|
||||
block = line.strip()
|
||||
if not block:
|
||||
continue
|
||||
sections.append((block, ""))
|
||||
|
||||
if not sections and text.strip():
|
||||
sections.append((text.strip(), ""))
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def _ensure_chunk_token_limit(chunks: list[str], chunk_token_num: int) -> list[str]:
|
||||
"""对输出 chunk 做 token 上限保护：默认 512 token 时允许到 768 再硬切。"""
|
||||
max_tokens = int(chunk_token_num or 0)
|
||||
if max_tokens <= 0:
|
||||
return [c.strip() for c in chunks if c and c.strip()]
|
||||
|
||||
hard_limit = max(max_tokens, int(max_tokens * GENERAL_HARD_LIMIT_RATIO))
|
||||
protected: list[str] = []
|
||||
for chunk in chunks:
|
||||
cleaned = (chunk or "").strip()
|
||||
if not cleaned:
|
||||
continue
|
||||
if nlp.count_tokens(cleaned) <= max_tokens:
|
||||
protected.append(cleaned)
|
||||
else:
|
||||
protected.extend(nlp.hard_split_by_token_limit(cleaned, max_tokens, hard_limit_token_num=hard_limit))
|
||||
return protected
|
||||
|
||||
|
||||
def chunk_markdown(markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
|
||||
parser_config = parser_config or {}
|
||||
|
||||
delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n"))
|
||||
chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512)
|
||||
overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0)
|
||||
|
||||
sections = _iter_sections(markdown_content, delimiter)
|
||||
chunks = nlp.naive_merge(
|
||||
sections,
|
||||
chunk_token_num=chunk_token_num,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
)
|
||||
return _ensure_chunk_token_limit(chunks, chunk_token_num)
|
||||
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from yuxi.knowledge.chunking.ragflow_like import nlp
|
||||
|
||||
_ARTICLE_PATTERN = re.compile(r"^(第[零一二三四五六七八九十百千万0-9]+条)[\s :：]*(.*)$")
|
||||
|
||||
|
||||
def _unescape_delimiter(delimiter: str) -> str:
|
||||
return delimiter.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t").replace("\\\\", "\\")
|
||||
|
||||
|
||||
def _iter_lines(markdown_content: str) -> list[str]:
|
||||
return [line.strip() for line in (markdown_content or "").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _normalize_law_line(line: str) -> str:
|
||||
# 法规 markdown 常见的 #、-、** 装饰会干扰层级识别，这里先做轻量归一化。
|
||||
text = (line or "").strip()
|
||||
text = re.sub(r"^#{1,6}\s+", "", text)
|
||||
text = re.sub(r"^[-*+]\s+", "", text)
|
||||
text = text.replace("**", "").replace("__", "").replace("`", "")
|
||||
text = re.sub(r"[ \t]+", " ", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _expand_article_line(line: str) -> list[str]:
|
||||
normalized = _normalize_law_line(line)
|
||||
if not normalized:
|
||||
return []
|
||||
|
||||
matched = _ARTICLE_PATTERN.match(normalized)
|
||||
if not matched:
|
||||
return [normalized]
|
||||
|
||||
article = matched.group(1).strip()
|
||||
body = matched.group(2).strip()
|
||||
if not body:
|
||||
return [article]
|
||||
return [article, body]
|
||||
|
||||
|
||||
def _iter_law_sections(markdown_content: str) -> list[str]:
|
||||
sections: list[str] = []
|
||||
for line in _iter_lines(markdown_content):
|
||||
sections.extend(_expand_article_line(line))
|
||||
return [section for section in sections if section]
|
||||
|
||||
|
||||
def _docx_heading_tree(markdown_content: str) -> list[str]:
|
||||
lines: list[tuple[int, str]] = []
|
||||
level_set: set[int] = set()
|
||||
|
||||
for raw in (markdown_content or "").splitlines():
|
||||
text = raw.strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
heading_match = re.match(r"^(#{1,6})\s+(.*)$", text)
|
||||
if heading_match:
|
||||
level = len(heading_match.group(1))
|
||||
value = heading_match.group(2).strip()
|
||||
else:
|
||||
level = 99
|
||||
value = text
|
||||
|
||||
if not value:
|
||||
continue
|
||||
|
||||
lines.append((level, value))
|
||||
level_set.add(level)
|
||||
|
||||
if not lines:
|
||||
return []
|
||||
|
||||
sorted_levels = sorted(level_set)
|
||||
h2_level = sorted_levels[1] if len(sorted_levels) > 1 else 1
|
||||
h2_level = sorted_levels[-2] if h2_level == sorted_levels[-1] and len(sorted_levels) > 2 else h2_level
|
||||
|
||||
root = nlp.Node(level=0, depth=h2_level, texts=[])
|
||||
root.build_tree(lines)
|
||||
return [element for element in root.get_tree() if element]
|
||||
|
||||
|
||||
def _ensure_chunk_token_limit(
|
||||
chunks: list[str], chunk_token_num: int, delimiter: str, overlapped_percent: int
|
||||
) -> list[str]:
|
||||
"""
|
||||
对输出 chunk 做 token 上限保护：
|
||||
1) 先尝试按行用 naive_merge 再切一次；
|
||||
2) 仍超长时才硬切，避免 embeddings 413。
|
||||
"""
|
||||
max_tokens = int(chunk_token_num or 0)
|
||||
normalized = [chunk.strip() for chunk in chunks if chunk and chunk.strip()]
|
||||
if max_tokens <= 0:
|
||||
return normalized
|
||||
|
||||
protected: list[str] = []
|
||||
for chunk in normalized:
|
||||
if nlp.count_tokens(chunk) <= max_tokens:
|
||||
protected.append(chunk)
|
||||
continue
|
||||
|
||||
refined = nlp.naive_merge(
|
||||
[(_line, "") for _line in _iter_lines(chunk)],
|
||||
chunk_token_num=max_tokens,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
)
|
||||
if not refined:
|
||||
refined = [chunk]
|
||||
|
||||
for item in refined:
|
||||
cleaned = (item or "").strip()
|
||||
if not cleaned:
|
||||
continue
|
||||
if nlp.count_tokens(cleaned) <= max_tokens:
|
||||
protected.append(cleaned)
|
||||
else:
|
||||
sentence_refined = nlp.naive_merge(
|
||||
[(_sentence, "") for _sentence in re.split(r"(?<=[。！？；;!?])", cleaned) if _sentence.strip()],
|
||||
chunk_token_num=max_tokens,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
)
|
||||
if not sentence_refined:
|
||||
sentence_refined = [cleaned]
|
||||
|
||||
for sentence_chunk in sentence_refined:
|
||||
text = sentence_chunk.strip()
|
||||
if not text:
|
||||
continue
|
||||
if nlp.count_tokens(text) <= max_tokens:
|
||||
protected.append(text)
|
||||
else:
|
||||
protected.extend(nlp.hard_split_by_token_limit(text, max_tokens))
|
||||
|
||||
return [chunk for chunk in protected if chunk.strip()]
|
||||
|
||||
|
||||
def chunk_markdown(filename: str, markdown_content: str, parser_config: dict[str, Any] | None = None) -> list[str]:
|
||||
"""
|
||||
法规分块主流程（简化版）：
|
||||
- docx 优先尝试标题树；
|
||||
- 其余格式先做法规文本归一化，再按章/节/条树形切分（depth=3）；
|
||||
- 最后统一执行超长保护。
|
||||
"""
|
||||
parser_config = parser_config or {}
|
||||
|
||||
delimiter = _unescape_delimiter(str(parser_config.get("delimiter", "\n") or "\n"))
|
||||
chunk_token_num = int(parser_config.get("chunk_token_num", 512) or 512)
|
||||
overlapped_percent = int(parser_config.get("overlapped_percent", 0) or 0)
|
||||
|
||||
if re.search(r"\.docx$", filename or "", re.IGNORECASE):
|
||||
docx_chunks = _docx_heading_tree(markdown_content)
|
||||
if len(docx_chunks) > 1:
|
||||
return _ensure_chunk_token_limit(docx_chunks, chunk_token_num, delimiter, overlapped_percent)
|
||||
|
||||
sections = _iter_law_sections(markdown_content)
|
||||
if not sections:
|
||||
return []
|
||||
|
||||
nlp.remove_contents_table(sections, eng=nlp.is_english(sections))
|
||||
typed_sections = [(s, "") for s in sections]
|
||||
nlp.make_colon_as_title(typed_sections)
|
||||
|
||||
bull = nlp.bullets_category([s for s, _ in typed_sections])
|
||||
if bull == nlp.MARKDOWN_BULLET_GROUP_INDEX:
|
||||
# 归一化后仍命中 markdown 组时，回退到中文法规层级组，避免退化成“按章大块”。
|
||||
bull = 0
|
||||
|
||||
merged = nlp.tree_merge(bull, typed_sections, depth=3)
|
||||
if not merged:
|
||||
merged = nlp.naive_merge(
|
||||
typed_sections,
|
||||
chunk_token_num=chunk_token_num,
|
||||
delimiter=delimiter,
|
||||
overlapped_percent=overlapped_percent,
|
||||
)
|
||||
|
||||
return _ensure_chunk_token_limit(merged, chunk_token_num, delimiter, overlapped_percent)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user