chore: import zh skill eval-driven-dev
This commit is contained in:
wehub-skill-sync
@@ -0,0 +1,66 @@
|
||||
---
|
||||
|
||||
# 可运行示例：CLI 应用程序
|
||||
|
||||
**当应用程序从命令行被调用时**（例如 `python -m myapp`，使用 argparse/click 的 CLI 工具）。
|
||||
|
||||
**做法**：使用 `asyncio.create_subprocess_exec` 调用 CLI 并捕获输出。
|
||||
|
||||
```python
|
||||
# pixie_qa/run_app.py
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from pydantic import BaseModel
|
||||
import pixie
|
||||
|
||||
|
||||
class AppArgs(BaseModel):
|
||||
query: str
|
||||
|
||||
|
||||
class AppRunnable(pixie.Runnable[AppArgs]):
|
||||
"""通过子进程驱动 CLI 应用程序。"""
|
||||
|
||||
@classmethod
|
||||
def create(cls) -> "AppRunnable":
|
||||
return cls()
|
||||
|
||||
async def run(self, args: AppArgs) -> None:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "myapp", "--query", args.query,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=120)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"App 失败（退出码 {proc.returncode}）：{stderr.decode()}")
|
||||
```
|
||||
|
||||
## 当 CLI 需要修补的依赖项时
|
||||
|
||||
如果 CLI 读取外部服务，请创建一个包装入口点，在运行真实 CLI 之前修补依赖项：
|
||||
|
||||
```python
|
||||
# pixie_qa/patched_app.py
|
||||
"""在运行真实 CLI 之前修补外部依赖项的入口点。"""
|
||||
import myapp.config as config
|
||||
config.redis_url = "mock://localhost"
|
||||
|
||||
from myapp.main import main
|
||||
main()
|
||||
```
|
||||
|
||||
然后将 Runnable 指向该包装器：
|
||||
|
||||
```python
|
||||
async def run(self, args: AppArgs) -> None:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "pixie_qa.patched_app", "--query", args.query,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=120)
|
||||
```
|
||||
|
||||
**注意**：对于 CLI 应用程序，`wrap(purpose="input")` 注入仅当应用程序在同一进程中运行时才有效。如果使用子进程，则可能需要通过环境变量或配置文件传递测试数据。
|
||||
@@ -0,0 +1,125 @@
|
||||
# Runnable Example: FastAPI / Web Server
|
||||
|
||||
**当应用是 Web 服务器时**（FastAPI、Flask、Starlette），需要测试完整的 HTTP 请求管道。
|
||||
|
||||
**方法**：使用 `httpx.AsyncClient` 配合 `ASGITransport`，在进程内运行 ASGI 应用。这是最快且最可靠的方式——无需子进程，无需管理端口。
|
||||
|
||||
```python
|
||||
# pixie_qa/run_app.py
|
||||
import httpx
|
||||
from pydantic import BaseModel
|
||||
import pixie
|
||||
|
||||
|
||||
class AppArgs(BaseModel):
|
||||
user_message: str
|
||||
|
||||
|
||||
class AppRunnable(pixie.Runnable[AppArgs]):
|
||||
"""通过进程内 ASGI 传输驱动 FastAPI 应用。"""
|
||||
|
||||
_client: httpx.AsyncClient
|
||||
|
||||
@classmethod
|
||||
def create(cls) -> "AppRunnable":
|
||||
return cls()
|
||||
|
||||
async def setup(self) -> None:
|
||||
from myapp.main import app # 你的 FastAPI/Starlette 应用实例
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
self._client = httpx.AsyncClient(transport=transport, base_url="http://test")
|
||||
|
||||
async def run(self, args: AppArgs) -> None:
|
||||
await self._client.post("/chat", json={"message": args.user_message})
|
||||
|
||||
async def teardown(self) -> None:
|
||||
await self._client.aclose()
|
||||
```
|
||||
|
||||
## ASGITransport 会跳过生命周期事件
|
||||
|
||||
`httpx.ASGITransport` **不会**触发 ASGI 生命周期事件（`startup` / `shutdown`）。如果应用在其生命周期中初始化了资源（数据库连接、缓存、服务客户端），则必须在 `setup()` 中手动复制该初始化逻辑：
|
||||
|
||||
```python
|
||||
async def setup(self) -> None:
|
||||
# 手动复制应用生命周期中的操作
|
||||
from myapp.db import get_connection, init_db, seed_data
|
||||
import myapp.main as app_module
|
||||
|
||||
conn = get_connection()
|
||||
init_db(conn)
|
||||
seed_data(conn)
|
||||
app_module.db_conn = conn # 设置应用期望的模块级全局变量
|
||||
|
||||
transport = httpx.ASGITransport(app=app_module.app)
|
||||
self._client = httpx.AsyncClient(transport=transport, base_url="http://test")
|
||||
|
||||
async def teardown(self) -> None:
|
||||
await self._client.aclose()
|
||||
# 清理手动初始化的资源
|
||||
import myapp.main as app_module
|
||||
if hasattr(app_module, "db_conn") and app_module.db_conn:
|
||||
app_module.db_conn.close()
|
||||
```
|
||||
|
||||
## 共享可变状态下的并发
|
||||
|
||||
如果应用使用了共享可变状态（内存 SQLite、基于文件的数据库、全局缓存），请添加信号量来串行化访问：
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
|
||||
class AppRunnable(pixie.Runnable[AppArgs]):
|
||||
_client: httpx.AsyncClient
|
||||
_sem: asyncio.Semaphore
|
||||
|
||||
@classmethod
|
||||
def create(cls) -> "AppRunnable":
|
||||
inst = cls()
|
||||
inst._sem = asyncio.Semaphore(1)
|
||||
return inst
|
||||
|
||||
async def setup(self) -> None:
|
||||
from myapp.main import app
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
self._client = httpx.AsyncClient(transport=transport, base_url="http://test")
|
||||
|
||||
async def run(self, args: AppArgs) -> None:
|
||||
async with self._sem:
|
||||
await self._client.post("/chat", json={"message": args.user_message})
|
||||
|
||||
async def teardown(self) -> None:
|
||||
await self._client.aclose()
|
||||
```
|
||||
|
||||
仅在需要时才使用信号量——如果应用使用以唯一 ID（call_sid、session_id）为键的按会话状态，则并发调用天然隔离，无需加锁。
|
||||
|
||||
## 备选方案：外部服务器配合 httpx
|
||||
|
||||
当应用无法直接导入时（复杂的启动流程、`__main__` 中的 `uvicorn.run()`），以子进程方式启动并通过 HTTP 访问：
|
||||
|
||||
```python
|
||||
class AppRunnable(pixie.Runnable[AppArgs]):
|
||||
_client: httpx.AsyncClient
|
||||
|
||||
@classmethod
|
||||
def create(cls) -> "AppRunnable":
|
||||
return cls()
|
||||
|
||||
async def setup(self) -> None:
|
||||
# 假定服务器已在运行（通过 run-with-timeout.sh 启动）
|
||||
self._client = httpx.AsyncClient(base_url="http://localhost:8000")
|
||||
|
||||
async def run(self, args: AppArgs) -> None:
|
||||
await self._client.post("/chat", json={"message": args.user_message})
|
||||
|
||||
async def teardown(self) -> None:
|
||||
await self._client.aclose()
|
||||
```
|
||||
|
||||
在运行 `pixie trace` 或 `pixie test` 之前启动服务器：
|
||||
|
||||
```bash
|
||||
bash resources/run-with-timeout.sh 120 uv run python -m myapp.server
|
||||
sleep 3 # 等待就绪
|
||||
@@ -0,0 +1,59 @@
|
||||
# 可运行示例：独立函数（无服务器）
|
||||
|
||||
**当应用是一个纯 Python 函数或模块时**——没有 Web 框架，没有服务器，没有基础设施。
|
||||
|
||||
**方法**：直接从 `run()` 中导入并调用该函数。这是最简单的情况。
|
||||
|
||||
```python
|
||||
# pixie_qa/run_app.py
|
||||
from pydantic import BaseModel
|
||||
import pixie
|
||||
|
||||
|
||||
class AppArgs(BaseModel):
|
||||
question: str
|
||||
|
||||
|
||||
class AppRunnable(pixie.Runnable[AppArgs]):
|
||||
"""驱动一个独立函数，用于追踪和评估。"""
|
||||
|
||||
@classmethod
|
||||
def create(cls) -> "AppRunnable":
|
||||
return cls()
|
||||
|
||||
async def run(self, args: AppArgs) -> None:
|
||||
from myapp.agent import answer_question
|
||||
await answer_question(args.question)
|
||||
```
|
||||
|
||||
如果函数是同步的，用 `asyncio.to_thread` 包裹它：
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
|
||||
async def run(self, args: AppArgs) -> None:
|
||||
from myapp.agent import answer_question
|
||||
await asyncio.to_thread(answer_question, args.question)
|
||||
```
|
||||
|
||||
如果函数依赖外部服务（例如向量存储），你在步骤 2a 中添加的 `wrap(purpose="input")` 调用会自动处理——注册中心会在评估模式下注入测试数据。
|
||||
|
||||
### 何时使用 `setup()` / `teardown()`
|
||||
|
||||
大多数独立函数不需要生命周期方法。仅当函数需要共享资源（例如预加载的嵌入模型、数据库连接）时才使用它们：
|
||||
|
||||
```python
|
||||
class AppRunnable(pixie.Runnable[AppArgs]):
|
||||
_model: SomeModel
|
||||
|
||||
@classmethod
|
||||
def create(cls) -> "AppRunnable":
|
||||
return cls()
|
||||
|
||||
async def setup(self) -> None:
|
||||
from myapp.models import load_model
|
||||
self._model = load_model()
|
||||
|
||||
async def run(self, args: AppArgs) -> None:
|
||||
from myapp.agent import answer_question
|
||||
await answer_question(args.question, model=self._model)
|
||||
Reference in New Issue
Block a user