mirrors/hkuds--deeptutor
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Tests / Import Check (Python 3.13) (push) Has been cancelled
Details
Tests / Import Check (Python 3.14) (push) Has been cancelled
Details
Tests / Python Tests (Python 3.11) (push) Has been cancelled
Details
Tests / Python Tests (Python 3.12) (push) Has been cancelled
Details
Tests / Python Tests (Python 3.14) (push) Has been cancelled
Details
Tests / Test Summary (push) Has been cancelled
Details
Tests / Lint and Format (push) Has been cancelled
Details
Tests / Web Node Tests (push) Has been cancelled
Details
Tests / Import Check (Python 3.11) (push) Has been cancelled
Details
Tests / Import Check (Python 3.12) (push) Has been cancelled
Details
Tests / Python Tests (Python 3.13) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:00:43 +08:00
commit e4dcfc49aa
1668 changed files with 324490 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deeptutor/api/__init__.py
View File
deeptutor/api/main.py
+453
View File
@@ -0,0 +1,453 @@
from contextlib import asynccontextmanager
import logging
import sys
from fastapi import Depends, FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from deeptutor.logging import configure_logging
from deeptutor.services.config import (
ensure_runtime_settings_files,
export_runtime_settings_to_env,
load_auth_settings,
load_system_settings,
)
from deeptutor.services.config.origins import normalize_origins
from deeptutor.services.path_service import get_path_service
ensure_runtime_settings_files()
export_runtime_settings_to_env(overwrite=True)
configure_logging()
logger = logging.getLogger(__name__)
class _SuppressWsNoise(logging.Filter):
"""Suppress noisy uvicorn logs for WebSocket connection churn."""
_SUPPRESSED = ("connection open", "connection closed")
def filter(self, record: logging.LogRecord) -> bool:
msg = record.getMessage()
return not any(f in msg for f in self._SUPPRESSED)
logging.getLogger("uvicorn.error").addFilter(_SuppressWsNoise())
CONFIG_DRIFT_ERROR_TEMPLATE = (
"Configuration Drift Detected: Capability tool references {drift} are not "
"registered in the runtime tool registry. Register the missing tools or "
"remove the stale tool names from the capability manifests."
)
class SafeOutputStaticFiles(StaticFiles):
"""Static file mount that only exposes explicitly whitelisted artifacts."""
def __init__(self, *args, path_service, **kwargs):
super().__init__(*args, **kwargs)
self._path_service = path_service
async def get_response(self, path: str, scope):
if not self._path_service.is_public_output_path(path):
raise HTTPException(status_code=404, detail="Output not found")
return await super().get_response(path, scope)
def validate_tool_consistency():
"""
Validate that capability manifests only reference tools that are actually
registered in the runtime ``ToolRegistry``.
"""
try:
from deeptutor.runtime.registry.capability_registry import get_capability_registry
from deeptutor.runtime.registry.tool_registry import get_tool_registry
capability_registry = get_capability_registry()
tool_registry = get_tool_registry()
available_tools = set(tool_registry.list_tools())
referenced_tools = set()
for manifest in capability_registry.get_manifests():
referenced_tools.update(manifest.get("tools_used", []) or [])
drift = referenced_tools - available_tools
if drift:
raise RuntimeError(CONFIG_DRIFT_ERROR_TEMPLATE.format(drift=drift))
except RuntimeError:
logger.exception("Configuration validation failed")
raise
except Exception:
logger.exception("Failed to load configuration for validation")
raise
def _build_cors_settings() -> dict[str, object]:
"""Build CORS settings for both localhost and remote Docker deployments."""
system_settings = load_system_settings()
auth_settings = load_auth_settings()
frontend_port = str(system_settings["frontend_port"])
extra_origins = normalize_origins(
[system_settings["cors_origin"], system_settings["cors_origins"]]
)
origins = [
f"http://localhost:{frontend_port}",
f"http://127.0.0.1:{frontend_port}",
"http://localhost:3000",
"http://127.0.0.1:3000",
]
for origin in extra_origins:
if origin not in origins:
origins.append(origin)
# Auth is disabled by default. In that local/single-user mode, mirror the
# pre-v1.3.8 behavior and allow remote Docker/LAN origins out of the box.
# When auth is enabled, require explicit CORS_ORIGIN(S) for credentialed
# cross-origin requests.
allow_origin_regex = None if auth_settings["enabled"] else r"https?://.*"
mode = "explicit" if auth_settings["enabled"] else "permissive"
return {
"allow_origins": origins,
"allow_origin_regex": allow_origin_regex,
"mode": mode,
}
@asynccontextmanager
async def lifespan(app: FastAPI):
"""
Application lifecycle management
Gracefully handle startup and shutdown events, avoid CancelledError
"""
# Execute on startup
logger.info("Application startup")
# Validate configuration consistency
validate_tool_consistency()
# Initialize LLM client early so OPENAI_* env vars are available before
# any downstream provider integrations start.
try:
from deeptutor.services.llm import get_llm_client
llm_client = get_llm_client()
logger.info(f"LLM client initialized: model={llm_client.config.model}")
except Exception as e:
logger.warning(f"Failed to initialize LLM client at startup: {e}")
try:
from deeptutor.events.event_bus import get_event_bus
event_bus = get_event_bus()
await event_bus.start()
logger.info("EventBus started")
except Exception as e:
logger.warning(f"Failed to start EventBus: {e}")
try:
from deeptutor.services.partners import get_partner_manager
await get_partner_manager().auto_start_partners()
except Exception as e:
logger.warning(f"Failed to auto-start partners: {e}")
try:
from deeptutor.services.cron import get_cron_service
await get_cron_service().start()
except Exception as e:
logger.warning(f"Failed to start cron service: {e}")
# Ping PocketBase if configured — logs a warning (not an error) if unreachable
try:
from deeptutor.services.pocketbase_client import ping_pocketbase
await ping_pocketbase()
except Exception as e:
logger.warning(f"PocketBase startup check failed: {e}")
# Migrate any v1 memory files (PROFILE.md / SUMMARY.md) into a
# backup folder so the v2 three-layer subsystem starts clean.
try:
from deeptutor.services.memory import (
migrate_partner_surface_if_needed,
migrate_v1_if_needed,
)
backup = migrate_v1_if_needed()
if backup is not None:
logger.info("v1 memory archived to %s", backup)
# Rename the legacy ``tutorbot`` memory surface (footnote refs, L2
# doc, snapshot/trace dirs, L3 meta keys) to ``partner``.
migrate_partner_surface_if_needed()
except Exception as e:
logger.warning(f"v1 memory migration failed: {e}")
yield
# Execute on shutdown
logger.info("Application shutdown")
# Stop cron scheduler
try:
from deeptutor.services.cron import get_cron_service
await get_cron_service().stop()
except Exception as e:
logger.warning(f"Failed to stop cron service: {e}")
# Stop partners
try:
from deeptutor.services.partners import get_partner_manager
await get_partner_manager().stop_all(preserve_auto_start=True)
logger.info("Partners stopped")
except Exception as e:
logger.warning(f"Failed to stop partners: {e}")
# Stop EventBus
try:
from deeptutor.events.event_bus import get_event_bus
event_bus = get_event_bus()
await event_bus.stop()
logger.info("EventBus stopped")
except Exception as e:
logger.warning(f"Failed to stop EventBus: {e}")
app = FastAPI(
title="DeepTutor API",
version="1.0.0",
lifespan=lifespan,
# Disable automatic trailing slash redirects to prevent protocol downgrade issues
# when deployed behind HTTPS reverse proxies (e.g., nginx).
# Without this, FastAPI's 307 redirects may change HTTPS to HTTP.
# See: https://github.com/HKUDS/DeepTutor/issues/112
redirect_slashes=False,
)
# Access logging is funneled through this one middleware. uvicorn's own
# per-request access log is disabled on every launch path (run_server.py via
# access_log=False; the launcher and Docker via `--no-access-log`), so routine
# 200s — the chatty frontend polling of /settings, /tools, /knowledge/list,
# etc. — never reach the logs. Only non-200s are surfaced, since those are the
# ones worth seeing.
#
# The `deeptutor.access` logger gets its own INFO stdout handler rather than
# leaning on the root handlers: the root console handler runs at the global log
# level (WARNING by default), which would swallow these INFO access lines.
# propagate=False keeps them from also printing through root if the global
# level is ever lowered to INFO/DEBUG.
_access_logger = logging.getLogger("deeptutor.access")
if not any(getattr(h, "_deeptutor_access_handler", False) for h in _access_logger.handlers):
_access_handler = logging.StreamHandler(sys.stdout)
_access_handler.setLevel(logging.INFO)
_access_handler.setFormatter(logging.Formatter("%(message)s"))
_access_handler._deeptutor_access_handler = True # type: ignore[attr-defined]
_access_logger.addHandler(_access_handler)
_access_logger.setLevel(logging.INFO)
_access_logger.propagate = False
@app.middleware("http")
async def selective_access_log(request, call_next):
response = await call_next(request)
if response.status_code != 200:
_access_logger.info(
'%s - "%s %s HTTP/%s" %d',
request.client.host if request.client else "-",
request.method,
request.url.path,
request.scope.get("http_version", "1.1"),
response.status_code,
)
return response
_cors_settings = _build_cors_settings()
logger.info(
"CORS configured: mode=%s allow_origins=%s allow_origin_regex=%s",
_cors_settings["mode"],
_cors_settings["allow_origins"],
_cors_settings["allow_origin_regex"],
)
app.add_middleware(
CORSMiddleware,
allow_origins=_cors_settings["allow_origins"],
allow_origin_regex=_cors_settings["allow_origin_regex"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Mount a filtered view over user outputs.
# Only whitelisted artifact paths are readable through the static handler.
path_service = get_path_service()
user_dir = path_service.get_public_outputs_root()
# Initialize user directories on startup
try:
from deeptutor.services.setup import init_user_directories
init_user_directories()
except Exception:
# Fallback: just create the main directory if it doesn't exist
if not user_dir.exists():
user_dir.mkdir(parents=True)
app.mount(
"/api/outputs",
SafeOutputStaticFiles(directory=str(user_dir), path_service=path_service),
name="outputs",
)
# Import routers only after runtime settings are initialized.
# Some router modules load YAML settings at import time.
from deeptutor.api.routers import (
agent_config,
attachments,
auth,
book,
capabilities_settings,
chat,
co_writer,
dashboard,
imports,
knowledge,
mastery_path,
mcp_settings,
memory,
notebook,
partners,
personas,
plugins_api,
question,
question_notebook,
quiz_judge,
sessions,
settings,
skills,
subagents,
system,
unified_ws,
voice,
)
from deeptutor.api.routers import (
tools as tools_router,
)
from deeptutor.multi_user.router import router as multi_user_router # noqa: E402
# Auth router is public — login/logout/register/status require no token
app.include_router(auth.router, prefix="/api/v1/auth", tags=["auth"])
# All other routers require a valid session when AUTH_ENABLED=true.
# require_auth is a no-op when AUTH_ENABLED=false, so this is safe for local use.
from deeptutor.api.routers.auth import require_admin, require_auth # noqa: E402
_auth = [Depends(require_auth)]
# Partner data is anchored at the admin workspace (data/partners) and shared
# process-wide, so management is admin-gated in multi-user deployments
# (single-user local runs are implicitly admin — no behaviour change there).
_admin = [Depends(require_admin)]
app.include_router(
multi_user_router,
prefix="/api/v1/multi-user",
tags=["multi-user"],
dependencies=_auth,
)
app.include_router(chat.router, prefix="/api/v1", tags=["chat"], dependencies=_auth)
app.include_router(
question.router, prefix="/api/v1/question", tags=["question"], dependencies=_auth
)
app.include_router(
knowledge.router, prefix="/api/v1/knowledge", tags=["knowledge"], dependencies=_auth
)
app.include_router(imports.router, prefix="/api/v1/imports", tags=["imports"], dependencies=_auth)
app.include_router(
dashboard.router, prefix="/api/v1/dashboard", tags=["dashboard"], dependencies=_auth
)
app.include_router(
mastery_path.router,
prefix="/api/v1/learning",
tags=["mastery-path"],
dependencies=_auth,
)
app.include_router(
co_writer.router, prefix="/api/v1/co_writer", tags=["co_writer"], dependencies=_auth
)
app.include_router(
notebook.router, prefix="/api/v1/notebook", tags=["notebook"], dependencies=_auth
)
app.include_router(book.router, prefix="/api/v1/book", tags=["book"], dependencies=_auth)
app.include_router(memory.router, prefix="/api/v1/memory", tags=["memory"], dependencies=_auth)
app.include_router(
capabilities_settings.router,
prefix="/api/v1/capabilities",
tags=["capabilities"],
dependencies=_auth,
)
app.include_router(
sessions.router, prefix="/api/v1/sessions", tags=["sessions"], dependencies=_auth
)
app.include_router(
question_notebook.router,
prefix="/api/v1/question-notebook",
tags=["question-notebook"],
dependencies=_auth,
)
app.include_router(
settings.router, prefix="/api/v1/settings", tags=["settings"], dependencies=_auth
)
app.include_router(
mcp_settings.router,
prefix="/api/v1/settings/mcp",
tags=["mcp-settings"],
dependencies=_auth,
)
app.include_router(skills.router, prefix="/api/v1/skills", tags=["skills"], dependencies=_auth)
app.include_router(
subagents.router, prefix="/api/v1/subagents", tags=["subagents"], dependencies=_auth
)
app.include_router(
personas.router, prefix="/api/v1/personas", tags=["personas"], dependencies=_auth
)
app.include_router(tools_router.router, prefix="/api/v1/tools", tags=["tools"], dependencies=_auth)
app.include_router(system.router, prefix="/api/v1/system", tags=["system"], dependencies=_auth)
app.include_router(voice.router, prefix="/api/v1/voice", tags=["voice"], dependencies=_auth)
app.include_router(
plugins_api.router, prefix="/api/v1/plugins", tags=["plugins"], dependencies=_auth
)
app.include_router(
agent_config.router, prefix="/api/v1/agent-config", tags=["agent-config"], dependencies=_auth
)
app.include_router(
partners.router, prefix="/api/v1/partners", tags=["partners"], dependencies=_admin
)
app.include_router(
attachments.router,
prefix="/api/attachments",
tags=["attachments"],
dependencies=_auth,
)
# Unified WebSocket endpoint — auth is checked inside the handler (WebSockets
# cannot use FastAPI dependencies in the standard way)
app.include_router(unified_ws.router, prefix="/api/v1", tags=["unified-ws"])
# Quiz AI-judge WebSocket — same caveat as unified_ws above; auth is checked
# inside the handler so the WS upgrade isn't rejected by an HTTP-style dep.
app.include_router(quiz_judge.router, prefix="/api/v1", tags=["quiz-judge"])
@app.get("/")
async def root():
return {"message": "Welcome to DeepTutor API"}
if __name__ == "__main__":
from deeptutor.api.run_server import main as run_server_main
run_server_main()
deeptutor/api/routers/__init__.py
+1
View File
@@ -0,0 +1 @@
# Init file for routers
deeptutor/api/routers/_partners_channel_schema.py
+176
View File
@@ -0,0 +1,176 @@
"""Channel-schema introspection — bridges Pydantic channel configs to the Web UI.
Used by ``GET /api/v1/partners/channels/schema`` so the front-end can render
generic forms for ANY channel (built-in or plugin) without hard-coding fields.
Why live here vs inside ``deeptutor.partners.channels``?
* This is an API-shaping concern (JSON Schema flattening, secret-field
detection) — keeping it next to the route avoids polluting the runtime
channel package with HTTP-specific helpers.
"""
from __future__ import annotations
import inspect
from typing import Any
from pydantic import BaseModel
from deeptutor.services.partners.manager import _is_secret_field
def resolve_config_model(channel_cls: type) -> type[BaseModel] | None:
"""Find the Pydantic config model paired with ``channel_cls``.
Convention every built-in channel follows: ``XxxChannel`` lives in the
same module as ``XxxConfig`` (e.g. ``TelegramChannel`` ↔ ``TelegramConfig``).
Falls back to "any ``*Config`` BaseModel in the module".
"""
module = inspect.getmodule(channel_cls)
if module is None:
return None
expected = channel_cls.__name__.replace("Channel", "") + "Config"
candidate = getattr(module, expected, None)
if isinstance(candidate, type) and issubclass(candidate, BaseModel):
return candidate
for _, obj in inspect.getmembers(module):
if (
isinstance(obj, type)
and obj is not BaseModel
and issubclass(obj, BaseModel)
and obj.__name__.endswith("Config")
):
return obj
return None
def inline_refs(schema: dict[str, Any]) -> dict[str, Any]:
"""Flatten Pydantic's ``$defs`` / ``$ref`` so the front-end doesn't need a resolver.
Nested model fields (e.g. ``slack.dm: SlackDMConfig``) become inline
``type: object`` subtrees with their own ``properties``.
"""
defs: dict[str, Any] = dict(schema.get("$defs", {}))
def _walk(node: Any) -> Any:
if isinstance(node, dict):
ref = node.get("$ref")
if isinstance(ref, str) and ref.startswith("#/$defs/"):
ref_name = ref.rsplit("/", 1)[-1]
resolved = defs.get(ref_name, {})
merged = {**resolved}
# Allow per-field overrides (description, default) from the ref site.
for k, v in node.items():
if k != "$ref":
merged[k] = v
return _walk(merged)
return {k: _walk(v) for k, v in node.items()}
if isinstance(node, list):
return [_walk(item) for item in node]
return node
out = _walk(schema)
if isinstance(out, dict):
out.pop("$defs", None)
return out
def _schema_accepts_string(prop_schema: dict[str, Any]) -> bool:
"""True iff the JSON-Schema fragment can hold a string value.
Used to filter out booleans/integers/arrays whose name happens to contain
a secret-looking substring (e.g. ``user_token_read_only: bool``).
"""
t = prop_schema.get("type")
if t == "string":
return True
if isinstance(t, list) and "string" in t:
return True
for variant in prop_schema.get("anyOf", []):
if isinstance(variant, dict) and variant.get("type") == "string":
return True
return False
def collect_secret_fields(schema: dict[str, Any], prefix: str = "") -> list[str]:
"""Return dot-paths for every string-typed property whose name hints at a secret.
e.g. ``["token"]`` for telegram, ``["imap_password", "smtp_password"]``
for email, ``["bot_token", "app_token"]`` for slack. A field like
``user_token_read_only: bool`` is intentionally skipped.
"""
paths: list[str] = []
properties = schema.get("properties") if isinstance(schema, dict) else None
if not isinstance(properties, dict):
return paths
for prop_name, prop_schema in properties.items():
if not isinstance(prop_schema, dict):
continue
full = f"{prefix}{prop_name}" if not prefix else f"{prefix}.{prop_name}"
if _is_secret_field(prop_name) and _schema_accepts_string(prop_schema):
paths.append(full)
if prop_schema.get("type") == "object":
paths.extend(collect_secret_fields(prop_schema, prefix=full))
return paths
def channel_schema_payload(channel_cls: type) -> dict[str, Any] | None:
"""Build the per-channel schema payload, or ``None`` if no config model found."""
model = resolve_config_model(channel_cls)
if model is None:
return None
# by_alias=False → property names match Python field names (snake_case),
# which is exactly the shape we persist in ``config.yaml`` and what every
# channel's ``__init__`` expects when ``model_validate(dict)`` is called.
# The pydantic Base config has populate_by_name=True so the runtime still
# accepts both forms; we standardise on snake_case for the wire schema.
raw = model.model_json_schema(by_alias=False)
flat = inline_refs(raw)
secret_fields = collect_secret_fields(flat)
try:
default_config = model().model_dump(mode="json", by_alias=False)
except Exception:
default_config = {}
return {
"name": getattr(channel_cls, "name", channel_cls.__name__),
"display_name": getattr(channel_cls, "display_name", channel_cls.__name__),
"default_config": default_config,
"secret_fields": secret_fields,
"json_schema": flat,
}
def all_channel_schemas() -> dict[str, dict[str, Any]]:
"""Build the schema dict for every discovered channel (built-in + plugins).
Channels whose module failed to import (missing optional dependency)
still appear, marked ``available: False`` with the import error — the UI
shows them grayed out instead of silently dropping them.
"""
from deeptutor.partners.channels.registry import discover_all_with_errors
channels, errors = discover_all_with_errors()
out: dict[str, dict[str, Any]] = {}
for name, cls in channels.items():
payload = channel_schema_payload(cls)
if payload is not None:
payload["available"] = True
out[name] = payload
for name, reason in errors.items():
out[name] = {
"name": name,
"display_name": name.title(),
"available": False,
"unavailable_reason": reason,
"default_config": {"enabled": False},
"secret_fields": [],
"json_schema": None,
}
return out
deeptutor/api/routers/agent_config.py
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python
"""
Agent Configuration API - Provides agent metadata for data-driven UI.
"""
from fastapi import APIRouter
router = APIRouter()
# Agent registry - single source of truth for agent UI metadata
AGENT_REGISTRY = {
"solve": {
"icon": "HelpCircle",
"color": "blue",
"label_key": "Problem Solved",
},
"question": {
"icon": "FileText",
"color": "purple",
"label_key": "Question Generated",
},
"research": {
"icon": "Search",
"color": "emerald",
"label_key": "Research Report",
},
"co_writer": {
"icon": "PenTool",
"color": "amber",
"label_key": "Co-Writer",
},
}
@router.get("/agents")
async def get_agent_config():
"""
Get agent UI configuration.
Returns:
Dict mapping agent type to UI metadata (icon, color, label_key)
"""
return AGENT_REGISTRY
@router.get("/agents/{agent_type}")
async def get_single_agent_config(agent_type: str):
"""
Get UI configuration for a specific agent.
Args:
agent_type: Agent type (solve, question, research, etc.)
Returns:
Agent UI metadata or 404 if not found
"""
if agent_type in AGENT_REGISTRY:
return AGENT_REGISTRY[agent_type]
return {"error": f"Agent type '{agent_type}' not found"}
deeptutor/api/routers/attachments.py
+91
View File
@@ -0,0 +1,91 @@
"""HTTP endpoint for chat attachment downloads / previews.
The chat turn runtime persists every uploaded attachment to the
:class:`~deeptutor.services.storage.AttachmentStore` and records the public
URL on the message. The frontend preview drawer loads files via this
router, which only serves paths the store hands back — every component is
sanitised to defend against directory traversal.
URL shape::
GET /api/attachments/{session_id}/{attachment_id}/{filename}
The session id functions as the ACL boundary, mirroring how the rest of
the app treats sessions today (single-tenant, session ownership is local
trust). Once multi-user auth lands we should swap this for signed URLs.
"""
from __future__ import annotations
import logging
import mimetypes
from urllib.parse import quote
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse
from deeptutor.services.storage import (
LocalDiskAttachmentStore,
get_attachment_store,
)
logger = logging.getLogger(__name__)
router = APIRouter()
def _content_disposition(filename: str, *, disposition: str = "inline") -> str:
"""Build a Content-Disposition header that survives non-ASCII filenames.
HTTP/1.1 headers are latin-1, so dropping a Chinese / accented filename
straight into ``filename="..."`` blows up with UnicodeEncodeError. RFC
6266 / RFC 5987 cover this: emit ``filename*=UTF-8''<percent-encoded>``
plus an ASCII fallback on ``filename=`` for legacy clients.
"""
ascii_fallback = filename.encode("ascii", errors="replace").decode("ascii")
# Quotes / backslashes break the simple-quoted-string form; collapse them.
ascii_fallback = ascii_fallback.replace('"', "_").replace("\\", "_")
encoded = quote(filename, safe="")
return f"{disposition}; filename=\"{ascii_fallback}\"; filename*=UTF-8''{encoded}"
@router.get("/{session_id}/{attachment_id}/{filename:path}")
async def get_attachment(
session_id: str,
attachment_id: str,
filename: str,
):
"""Serve a previously uploaded chat attachment.
Responds with ``Content-Disposition: inline`` so browsers preview PDFs
and images directly in an ``<iframe>`` / ``<img>``. For unknown types
the browser still falls back to download, which is fine for the
drawer's "Download" button path.
"""
store = get_attachment_store()
if not isinstance(store, LocalDiskAttachmentStore):
# Future remote backends should issue a redirect to the signed URL
# here. Local-disk is the only backend today, so this branch just
# guards against an unexpected configuration.
raise HTTPException(status_code=501, detail="Attachment backend not servable")
target = store.resolve_path(
session_id=session_id,
attachment_id=attachment_id,
filename=filename,
)
if target is None:
raise HTTPException(status_code=404, detail="Attachment not found")
media_type, _ = mimetypes.guess_type(target.name)
if not media_type:
media_type = "application/octet-stream"
# ``inline`` lets the browser preview the file when possible while still
# honouring the suggested filename for the drawer's download action.
headers = {
"Content-Disposition": _content_disposition(target.name),
# User-uploaded data; do not let intermediaries cache it.
"Cache-Control": "private, max-age=0, must-revalidate",
}
return FileResponse(path=str(target), media_type=media_type, headers=headers)
deeptutor/api/routers/auth.py
+831
View File
@@ -0,0 +1,831 @@
"""Auth router — login, logout, status, registration, profile, and user-management endpoints."""
from contextvars import Token as _CtxToken
import logging
import re
from fastapi import (
APIRouter,
Cookie,
Depends,
File,
Header,
HTTPException,
Response,
UploadFile,
WebSocket,
status,
)
from fastapi.responses import FileResponse
from pydantic import BaseModel, field_validator
from deeptutor.services.config import load_auth_settings
# SameSite=None lets the cookie work when the browser accesses the frontend via
# 127.0.0.1 and the backend via localhost (different origins on the same machine).
# Browsers require Secure=True for SameSite=None, but that needs HTTPS — so in
# local dev we fall back to SameSite=Lax and tell users to use localhost:// URLs.
_SECURE = bool(load_auth_settings()["cookie_secure"])
_SAMESITE = "none" if _SECURE else "lax"
from deeptutor.multi_user.context import set_current_user, user_from_token_payload
from deeptutor.multi_user.paths import local_admin_user
from deeptutor.services.auth import (
AUTH_ENABLED,
POCKETBASE_ENABLED,
TOKEN_EXPIRE_HOURS,
TokenPayload,
add_user,
authenticate,
authenticate_pb,
create_token,
decode_token,
delete_user,
get_user_info,
is_first_user,
list_users,
register_pb,
set_avatar,
set_role,
)
logger = logging.getLogger(__name__)
router = APIRouter()
_COOKIE_NAME = "dt_token"
_COOKIE_MAX_AGE = TOKEN_EXPIRE_HOURS * 3600
# ---------------------------------------------------------------------------
# Schemas
# ---------------------------------------------------------------------------
class LoginRequest(BaseModel):
"""Payload for the POST /login endpoint."""
username: str
password: str
class RegisterRequest(BaseModel):
"""Payload for the POST /register endpoint."""
username: str
password: str
@field_validator("username")
@classmethod
def username_valid(cls, v: str) -> str:
import re
v = v.strip()
if not v:
raise ValueError("Email cannot be empty")
# Accept standard email addresses (used by PocketBase mode) or plain
# usernames (used by the built-in SQLite/JSON auth mode).
email_re = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
plain_re = re.compile(r"^[A-Za-z0-9_\-.]{3,64}$")
if not email_re.match(v) and not plain_re.match(v):
raise ValueError("Enter a valid email address")
return v
@field_validator("password")
@classmethod
def password_valid(cls, v: str) -> str:
if len(v) < 8:
raise ValueError("Password must be at least 8 characters")
return v
class SetRoleRequest(BaseModel):
"""Payload for the PUT /users/{username}/role endpoint."""
role: str
@field_validator("role")
@classmethod
def role_valid(cls, v: str) -> str:
if v not in ("admin", "user"):
raise ValueError("Role must be 'admin' or 'user'")
return v
class AuthStatusResponse(BaseModel):
"""Response body for the GET /status endpoint."""
enabled: bool
authenticated: bool
user_id: str | None = None
username: str | None = None
role: str | None = None
is_admin: bool = False
avatar: str = ""
class UserInfo(BaseModel):
"""Single user record returned by the GET /users and /profile endpoints."""
id: str = ""
username: str
role: str
created_at: str
disabled: bool = False
avatar: str = ""
# Markers settable through PUT /profile. Image markers ("img:<version>") are
# managed exclusively by the upload endpoint so users cannot point their
# avatar at a file that was never validated.
_ICON_MARKER_RE = re.compile(r"^icon:[a-z0-9-]{1,32}:[a-z0-9-]{1,32}$")
# User ids are generated as "u_<uuid hex>" (plus the "local-admin" /
# "env-admin" sentinels); reject anything else before it reaches the
# filesystem layer.
_USER_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
class UpdateProfileRequest(BaseModel):
"""Payload for the PUT /profile endpoint."""
avatar: str
@field_validator("avatar")
@classmethod
def avatar_valid(cls, v: str) -> str:
v = v.strip()
if v and not _ICON_MARKER_RE.match(v):
raise ValueError("Avatar must be empty or 'icon:<name>:<color>'")
return v
# ---------------------------------------------------------------------------
# Shared helper — extract token from cookie or Bearer header
# ---------------------------------------------------------------------------
def _bearer_token_from_header(authorization: str | None) -> str | None:
"""Parse ``Authorization: Bearer <token>`` without using ``HTTPBearer``.
``HTTPBearer`` is a class-based dependency whose ``__call__`` is annotated
``request: Request``. FastAPI doesn't inject a Request into WebSocket
dependency resolution, which makes ``HTTPBearer`` raise ``TypeError`` the
moment a router with this dep mounts a WS endpoint. Doing the parse by
hand keeps ``require_auth`` HTTP/WS-symmetric.
"""
if not authorization:
return None
parts = authorization.split(None, 1)
if len(parts) == 2 and parts[0].lower() == "bearer":
token = parts[1].strip()
return token or None
return None
def _extract_token(authorization: str | None, dt_token: str | None) -> str | None:
return _bearer_token_from_header(authorization) or dt_token
# ---------------------------------------------------------------------------
# Dependencies — reusable auth guards for other routers
# ---------------------------------------------------------------------------
def _install_current_user(payload: TokenPayload | None) -> _CtxToken:
"""Install the request-local current-user ContextVar from an auth result.
Single point of truth for ``payload → CurrentUser`` so HTTP and WebSocket
entry points produce identical user objects. ``payload is None`` means
"no JWT was required" (AUTH_ENABLED=false) and resolves to the local
admin user; a non-None payload resolves through ``user_from_token_payload``.
Returns the ContextVar reset token. HTTP callers ignore it (the request
ends with the task, so the var is GC'd with the task context). WebSocket
callers keep it and call ``reset_current_user`` in their ``finally`` block,
because a WS connection outlives the dependency-resolution task.
⚠ Invariant: every authenticated entry point MUST call this before the
handler runs. Skipping it leaves ``get_current_path_service()`` falling
back to the admin workspace — the silent-routing root cause of #481.
"""
user = local_admin_user() if payload is None else user_from_token_payload(payload)
return set_current_user(user)
async def require_auth(
authorization: str | None = Header(default=None, alias="Authorization"),
dt_token: str | None = Cookie(default=None),
) -> TokenPayload | None:
"""
FastAPI dependency that enforces authentication when AUTH_ENABLED=true.
Accepts the JWT from either:
- Authorization: Bearer <token> header
- dt_token cookie
``Header`` and ``Cookie`` are kept here in place of ``HTTPBearer`` so the
function stays usable from WebSocket call sites that don't go through
FastAPI's standard HTTP request lifecycle.
Returns the authenticated TokenPayload, or None if auth is disabled.
Raises HTTP 401 if auth is enabled but the token is missing or invalid.
Declared ``async def`` so the ``set_current_user`` call runs in the same
asyncio context as the endpoint. A sync dependency is dispatched via
``anyio.to_thread.run_sync``, which executes the function in a worker
thread under a *copy* of the request context; any ``ContextVar.set``
inside that thread is discarded when the thread returns, leaving the
endpoint to read the unset default. That regression was the root cause
of #481.
"""
if not AUTH_ENABLED:
_install_current_user(None)
return None
token = _extract_token(authorization, dt_token)
if not token:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Not authenticated",
headers={"WWW-Authenticate": "Bearer"},
)
payload = decode_token(token)
if not payload:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid or expired token",
headers={"WWW-Authenticate": "Bearer"},
)
_install_current_user(payload)
return payload
class _WsAuthFailed:
"""Sentinel: ws_require_auth failed and closed the WebSocket."""
ws_auth_failed: _WsAuthFailed = _WsAuthFailed()
async def ws_require_auth(ws: WebSocket) -> _CtxToken | _WsAuthFailed:
"""Authenticate a WebSocket connection and set the user ContextVar.
Must be called **before** ``ws.accept()`` so the server can reject
unauthenticated upgrades cleanly.
Returns a ContextVar reset token on success, or ``ws_auth_failed``
on failure (the WebSocket is already closed — the caller should
``return`` immediately).
Usage::
user_token = await ws_require_auth(ws)
if user_token is ws_auth_failed:
return
await ws.accept()
try:
...
finally:
reset_current_user(user_token)
"""
if not AUTH_ENABLED:
return _install_current_user(None)
token = ws.query_params.get("token") or ws.cookies.get("dt_token")
payload = decode_token(token) if token else None
if not payload:
await ws.close(code=4001)
return ws_auth_failed
return _install_current_user(payload)
async def require_admin(
payload: TokenPayload | None = Depends(require_auth),
) -> TokenPayload:
"""
FastAPI dependency that requires the caller to be an admin.
Raises HTTP 403 if the authenticated user is not an admin.
When AUTH_ENABLED=false, all requests are treated as admin.
``async def`` mirrors ``require_auth`` so the dependency chain stays on
the event loop and the user ContextVar set by ``require_auth`` is visible
to the endpoint.
"""
if not AUTH_ENABLED:
return _local_admin_token_payload()
if payload is None or payload.role != "admin":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Admin access required",
)
return payload
def _local_admin_token_payload() -> TokenPayload:
"""Synthetic admin payload used when AUTH_ENABLED=false.
Mirrors the local admin identity (LOCAL_ADMIN_USERNAME / LOCAL_ADMIN_ID)
so audit logs and self-reference checks behave the same as in multi-user
mode. Values are kept aligned with ``local_admin_user()`` in
``deeptutor/multi_user/paths.py``.
"""
from deeptutor.multi_user.models import LOCAL_ADMIN_ID, LOCAL_ADMIN_USERNAME
return TokenPayload(
username=LOCAL_ADMIN_USERNAME,
role="admin",
user_id=LOCAL_ADMIN_ID,
)
# ---------------------------------------------------------------------------
# Public endpoints (no auth required)
# ---------------------------------------------------------------------------
@router.get("/status", response_model=AuthStatusResponse)
async def auth_status(
authorization: str | None = Header(default=None, alias="Authorization"),
dt_token: str | None = Cookie(default=None),
) -> AuthStatusResponse:
"""Return whether auth is enabled and whether the current request is authenticated."""
if not AUTH_ENABLED:
return AuthStatusResponse(
enabled=False,
authenticated=True,
user_id="local-admin",
username="local",
role="admin",
is_admin=True,
)
token = _extract_token(authorization, dt_token)
payload = decode_token(token) if token else None
avatar = ""
if payload is not None:
info = get_user_info(payload.username)
if info:
avatar = str(info.get("avatar") or "")
return AuthStatusResponse(
enabled=True,
authenticated=payload is not None,
user_id=payload.user_id if payload else None,
username=payload.username if payload else None,
role=payload.role if payload else None,
is_admin=payload.role == "admin" if payload else False,
avatar=avatar,
)
@router.post("/login")
async def login(body: LoginRequest, response: Response) -> dict:
"""Validate credentials and set a JWT cookie."""
if not AUTH_ENABLED:
return {"ok": True, "message": "Auth is disabled — no login required."}
if POCKETBASE_ENABLED:
# PocketBase mode: email = username field for backwards-compat with the
# existing LoginRequest schema; users can pass their email as "username".
pb_result = authenticate_pb(body.username, body.password)
if not pb_result:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect email or password",
)
payload, pb_token = pb_result
response.set_cookie(
key=_COOKIE_NAME,
value=pb_token,
httponly=True,
samesite=_SAMESITE,
max_age=_COOKIE_MAX_AGE,
secure=_SECURE,
)
logger.info(f"User '{payload.username}' logged in via PocketBase (role={payload.role!r})")
return {
"ok": True,
"user_id": payload.user_id,
"username": payload.username,
"role": payload.role,
"is_admin": payload.role == "admin",
}
# Standard JWT + bcrypt mode
result = authenticate(body.username, body.password)
if not result:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect username or password",
)
token = create_token(result.username, result.role, result.user_id)
response.set_cookie(
key=_COOKIE_NAME,
value=token,
httponly=True,
samesite=_SAMESITE,
max_age=_COOKIE_MAX_AGE,
secure=_SECURE,
)
logger.info(f"User '{result.username}' logged in (role={result.role!r})")
return {
"ok": True,
"user_id": result.user_id,
"username": result.username,
"role": result.role,
"is_admin": result.role == "admin",
}
@router.post("/logout")
async def logout(response: Response) -> dict:
"""Clear the JWT cookie."""
response.delete_cookie(key=_COOKIE_NAME, samesite=_SAMESITE)
return {"ok": True}
@router.post("/register", status_code=status.HTTP_201_CREATED)
async def register(body: RegisterRequest) -> dict:
"""
Bootstrap-only registration.
Public endpoint that creates the *first* admin account when the user store
is empty. Once an admin exists, this endpoint is closed; further accounts
must be created by an admin via ``POST /api/v1/auth/users``.
Only available when AUTH_ENABLED=true.
"""
if not AUTH_ENABLED:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Auth is disabled — registration is not available.",
)
if POCKETBASE_ENABLED:
# PocketBase deployments are documented as single-user. Keep registration
# closed and require admins to provision users in the PocketBase admin UI.
if not is_first_user():
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Self-registration is closed. Ask an administrator to create your account.",
)
result = register_pb(username=body.username, email=body.username, password=body.password)
if not result:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="Registration failed — username or email may already be taken.",
)
logger.info(f"First user registered via PocketBase: '{body.username}'")
return {
"ok": True,
"user_id": result.get("id", ""),
"username": body.username,
"role": "user",
"is_first_user": True,
"is_admin": False,
}
# Standard mode — only allowed before the first admin exists.
if not is_first_user():
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Self-registration is closed. Ask an administrator to create your account.",
)
existing = {u["username"] for u in list_users()}
if body.username in existing:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="Username already taken",
)
add_user(body.username, body.password)
user_id = ""
role = "user"
for item in list_users():
if item.get("username") == body.username:
user_id = str(item.get("id") or "")
role = str(item.get("role") or "user")
break
logger.info(f"First user (admin) registered: '{body.username}'")
return {
"ok": True,
"user_id": user_id,
"username": body.username,
"role": role,
"is_first_user": True,
"is_admin": role == "admin",
}
@router.get("/is_first_user")
async def check_is_first_user() -> dict:
"""Return whether the user store is empty (used by the register UI)."""
return {"is_first_user": is_first_user() if AUTH_ENABLED else False}
# ---------------------------------------------------------------------------
# Profile endpoints (any authenticated user, self-service)
# ---------------------------------------------------------------------------
_AVATAR_MAX_BYTES = 1 * 1024 * 1024
_AVATAR_MEDIA_TYPES = {"png": "image/png", "jpg": "image/jpeg", "webp": "image/webp"}
def _sniff_image(data: bytes) -> str | None:
"""Detect a supported raster image format from its magic bytes.
The uploaded filename and Content-Type are attacker-controlled, so the
stored extension (and the media type served back) is derived from the
bytes alone. SVG is deliberately unsupported — serving user-supplied SVG
is a stored-XSS vector.
"""
if data[:8] == b"\x89PNG\r\n\x1a\n":
return "png"
if data[:3] == b"\xff\xd8\xff":
return "jpg"
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
return "webp"
return None
def _require_profile_identity(payload: TokenPayload | None) -> TokenPayload:
"""Shared guard for the self-service profile endpoints."""
if not AUTH_ENABLED or payload is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Auth is disabled — profiles are not available.",
)
return payload
@router.get("/profile", response_model=UserInfo)
async def get_profile(
payload: TokenPayload | None = Depends(require_auth),
) -> UserInfo:
"""Return the current user's own account info."""
current = _require_profile_identity(payload)
info = get_user_info(current.username)
if info is None:
# PocketBase-backed identities have no local record; fall back to the
# token claims so the profile page still renders.
return UserInfo(
id=current.user_id,
username=current.username,
role=current.role,
created_at="",
)
return UserInfo(**info)
@router.put("/profile")
async def update_profile(
body: UpdateProfileRequest,
payload: TokenPayload | None = Depends(require_auth),
) -> dict:
"""Update the current user's own avatar marker (icon choice or reset).
Only the validated ``icon:<name>:<color>`` form (or empty string) is
accepted here; ``img:`` markers are owned by the upload endpoint.
"""
current = _require_profile_identity(payload)
if not set_avatar(current.username, body.avatar):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
# The marker no longer references an uploaded image, so drop the file.
from deeptutor.multi_user.identity import delete_avatar_file
if current.user_id and _USER_ID_RE.match(current.user_id):
delete_avatar_file(current.user_id)
return {"ok": True, "avatar": body.avatar}
@router.put("/profile/avatar")
async def upload_avatar(
file: UploadFile = File(...),
payload: TokenPayload | None = Depends(require_auth),
) -> dict:
"""Upload an avatar image for the current user.
The client is expected to crop/resize before uploading; the server only
enforces a size cap and validates the format by magic bytes. Not available
in PocketBase mode (those identities have no local user record).
"""
current = _require_profile_identity(payload)
if POCKETBASE_ENABLED:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Avatar upload is not available in PocketBase mode.",
)
if not current.user_id or not _USER_ID_RE.match(current.user_id):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Cannot store an avatar for this account.",
)
info = get_user_info(current.username)
if info is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
data = await file.read(_AVATAR_MAX_BYTES + 1)
if len(data) > _AVATAR_MAX_BYTES:
raise HTTPException(
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
detail="Avatar image is too large (max 1 MB).",
)
ext = _sniff_image(data)
if ext is None:
raise HTTPException(
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
detail="Avatar must be a PNG, JPEG or WebP image.",
)
from deeptutor.multi_user.identity import save_avatar_file
# Bump the version embedded in the marker so clients cache-bust the URL.
previous = str(info.get("avatar") or "")
version = 1
if previous.startswith("img:"):
try:
version = int(previous.split(":", 1)[1]) + 1
except ValueError:
version = 1
marker = f"img:{version}"
save_avatar_file(current.user_id, data, ext)
if not set_avatar(current.username, marker):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
logger.info(f"User '{current.username}' uploaded a new avatar ({ext}, {len(data)} bytes)")
return {"ok": True, "avatar": marker}
@router.delete("/profile/avatar")
async def remove_avatar(
payload: TokenPayload | None = Depends(require_auth),
) -> dict:
"""Remove the current user's uploaded avatar image and reset the marker."""
current = _require_profile_identity(payload)
from deeptutor.multi_user.identity import delete_avatar_file
if current.user_id and _USER_ID_RE.match(current.user_id):
delete_avatar_file(current.user_id)
set_avatar(current.username, "")
return {"ok": True, "avatar": ""}
@router.get("/avatar/{user_id}")
async def get_avatar_image(
user_id: str,
_: TokenPayload | None = Depends(require_auth),
) -> FileResponse:
"""Serve a stored avatar image. Any authenticated user may view avatars
(they appear in the admin table and next to the viewer's own profile)."""
if not _USER_ID_RE.match(user_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Avatar not found")
from deeptutor.multi_user.identity import get_avatar_file
target = get_avatar_file(user_id)
if target is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Avatar not found")
media_type = _AVATAR_MEDIA_TYPES.get(target.suffix.lstrip("."), "application/octet-stream")
headers = {
# Private user content; the marker version in the URL handles busting.
"Cache-Control": "private, max-age=86400",
"X-Content-Type-Options": "nosniff",
"Content-Disposition": "inline",
}
return FileResponse(path=str(target), media_type=media_type, headers=headers)
# ---------------------------------------------------------------------------
# Admin-only endpoints
# ---------------------------------------------------------------------------
@router.get("/users", response_model=list[UserInfo])
async def get_users(_: TokenPayload = Depends(require_admin)) -> list[UserInfo]:
"""List all registered users. Requires admin role."""
return [UserInfo(**u) for u in list_users()]
@router.post("/users", status_code=status.HTTP_201_CREATED)
async def admin_create_user(
body: RegisterRequest,
current: TokenPayload = Depends(require_admin),
) -> dict:
"""Admin-only: create a new user account.
Replaces the public ``/register`` flow once the first admin exists. The
new account is always created with role=``user``; admins can promote
later via ``PUT /users/{username}/role``.
"""
if not AUTH_ENABLED:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Auth is disabled — user creation is not available.",
)
if POCKETBASE_ENABLED:
result = register_pb(username=body.username, email=body.username, password=body.password)
if not result:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="Failed to create user — username may already be taken.",
)
logger.info(
f"Admin '{current.username if current else 'local'}' created PocketBase user "
f"'{body.username}'"
)
return {
"ok": True,
"user_id": result.get("id", ""),
"username": body.username,
"role": "user",
"is_admin": False,
}
existing = {u["username"] for u in list_users()}
if body.username in existing:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="Username already taken",
)
add_user(body.username, body.password)
user_id = ""
role = "user"
for item in list_users():
if item.get("username") == body.username:
user_id = str(item.get("id") or "")
role = str(item.get("role") or "user")
break
logger.info(
f"Admin '{current.username if current else 'local'}' created user '{body.username}' "
f"(role={role!r})"
)
return {
"ok": True,
"user_id": user_id,
"username": body.username,
"role": role,
"is_admin": role == "admin",
}
@router.delete("/users/{username}", status_code=status.HTTP_200_OK)
async def remove_user(
username: str,
current: TokenPayload = Depends(require_admin),
) -> dict:
"""Delete a user. Admins cannot delete their own account."""
if current and username == current.username:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="You cannot delete your own account",
)
# Capture the id before the record disappears so the avatar file can go too.
info = get_user_info(username)
removed = delete_user(username)
if not removed:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
user_id = str(info.get("id") or "") if info else ""
if user_id and _USER_ID_RE.match(user_id):
from deeptutor.multi_user.identity import delete_avatar_file
delete_avatar_file(user_id)
logger.info(f"Admin '{current.username if current else 'local'}' deleted user '{username}'")
return {"ok": True}
@router.put("/users/{username}/role", status_code=status.HTTP_200_OK)
async def update_user_role(
username: str,
body: SetRoleRequest,
current: TokenPayload = Depends(require_admin),
) -> dict:
"""Change a user's role. Admins cannot change their own role."""
if current and username == current.username:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="You cannot change your own role",
)
updated = set_role(username, body.role)
if not updated:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
logger.info(
f"Admin '{current.username if current else 'local'}' set '{username}' role to {body.role!r}"
)
return {"ok": True, "username": username, "role": body.role}
deeptutor/api/routers/book.py
+669
View File
@@ -0,0 +1,669 @@
"""
Book Engine API Router
======================
REST + WebSocket endpoints for the ``BookEngine``. Phase 1 surface:
create / confirm / compile / read / delete + a per-book event stream.
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any
from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, Field
from deeptutor.book import (
BlockType,
BookProposal,
Spine,
get_book_engine,
)
from deeptutor.book.models import ContentType
from deeptutor.book.streaming import SOURCE as BOOK_SOURCE
from deeptutor.core.stream import StreamEventType
from deeptutor.core.stream_bus import StreamBus
router = APIRouter()
logger = logging.getLogger(__name__)
# ─────────────────────────────────────────────────────────────────────────────
# Request / response models
# ─────────────────────────────────────────────────────────────────────────────
class CreateBookRequest(BaseModel):
user_intent: str = Field(default="")
chat_session_id: str = Field(default="")
chat_selections: list[dict[str, Any]] = Field(default_factory=list)
notebook_refs: list[dict[str, Any]] = Field(default_factory=list)
knowledge_bases: list[str] = Field(default_factory=list)
question_categories: list[int] = Field(default_factory=list)
question_entries: list[int] = Field(default_factory=list)
language: str = Field(default="en")
class ConfirmProposalRequest(BaseModel):
book_id: str
proposal: dict[str, Any] | None = None # full edited BookProposal payload
class ConfirmSpineRequest(BaseModel):
book_id: str
spine: dict[str, Any] | None = None
auto_compile: bool = True
class CompilePageRequest(BaseModel):
book_id: str
page_id: str
force: bool = False
class RegenerateBlockRequest(BaseModel):
book_id: str
page_id: str
block_id: str
params_override: dict[str, Any] | None = None
class InsertBlockRequest(BaseModel):
book_id: str
page_id: str
block_type: str
params: dict[str, Any] | None = None
position: int | None = None
compile_now: bool = True
class DeleteBlockRequest(BaseModel):
book_id: str
page_id: str
block_id: str
class MoveBlockRequest(BaseModel):
book_id: str
page_id: str
block_id: str
new_position: int
class ChangeBlockTypeRequest(BaseModel):
book_id: str
page_id: str
block_id: str
new_type: str
params_override: dict[str, Any] | None = None
class DeepDiveRequest(BaseModel):
book_id: str
parent_page_id: str
topic: str
block_id: str | None = None
content_type: str = "concept"
class QuizAttemptRequest(BaseModel):
book_id: str
page_id: str
block_id: str
question_id: str = ""
user_answer: str = ""
is_correct: bool = False
request_remediation: bool = False
class SupplementRequest(BaseModel):
book_id: str
page_id: str
topic: str
class PageChatSessionRequest(BaseModel):
book_id: str
page_id: str
session_id: str
class RebuildBookRequest(BaseModel):
book_id: str
auto_compile: bool = True
# ─────────────────────────────────────────────────────────────────────────────
# REST endpoints
# ─────────────────────────────────────────────────────────────────────────────
@router.get("/health")
async def health_check() -> dict[str, str]:
return {"status": "healthy", "service": "book"}
@router.get("/books")
async def list_books() -> dict[str, Any]:
engine = get_book_engine()
return {"books": [b.model_dump(mode="json") for b in engine.list_books()]}
@router.get("/books/{book_id}")
async def get_book(book_id: str) -> dict[str, Any]:
engine = get_book_engine()
book = engine.load_book(book_id)
if book is None:
raise HTTPException(status_code=404, detail="Book not found")
spine = engine.load_spine(book_id)
pages = engine.list_pages(book_id)
progress = engine.load_progress(book_id)
return {
"book": book.model_dump(mode="json"),
"spine": spine.model_dump(mode="json") if spine else None,
"pages": [p.model_dump(mode="json") for p in pages],
"progress": progress.model_dump(mode="json"),
}
@router.get("/books/{book_id}/spine")
async def get_spine(book_id: str) -> dict[str, Any]:
engine = get_book_engine()
spine = engine.load_spine(book_id)
if spine is None:
raise HTTPException(status_code=404, detail="Spine not found")
return {"spine": spine.model_dump(mode="json")}
@router.get("/books/{book_id}/pages/{page_id}")
async def get_page(book_id: str, page_id: str) -> dict[str, Any]:
engine = get_book_engine()
page = engine.load_page(book_id, page_id)
if page is None:
raise HTTPException(status_code=404, detail="Page not found")
return {"page": page.model_dump(mode="json")}
@router.delete("/books/{book_id}")
async def delete_book(book_id: str) -> dict[str, Any]:
engine = get_book_engine()
ok = engine.delete_book(book_id)
if not ok:
raise HTTPException(status_code=404, detail="Book not found")
return {"deleted": True, "book_id": book_id}
@router.post("/books")
async def create_book(req: CreateBookRequest) -> dict[str, Any]:
"""Stage 1: capture inputs + run IdeationAgent."""
if not req.user_intent.strip():
raise HTTPException(status_code=400, detail="user_intent is required")
engine = get_book_engine()
try:
book, proposal = await engine.create_book(
user_intent=req.user_intent,
chat_session_id=req.chat_session_id,
chat_selections=req.chat_selections,
notebook_refs=req.notebook_refs,
knowledge_bases=req.knowledge_bases,
question_categories=req.question_categories,
question_entries=req.question_entries,
language=req.language,
)
except Exception as exc: # noqa: BLE001
logger.error(f"create_book failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
return {
"book": book.model_dump(mode="json"),
"proposal": proposal.model_dump(mode="json"),
}
@router.post("/books/confirm-proposal")
async def confirm_proposal(req: ConfirmProposalRequest) -> dict[str, Any]:
"""Stage 2: user confirms (and possibly edits) the proposal → SpineAgent."""
engine = get_book_engine()
edited: BookProposal | None = None
if req.proposal:
try:
edited = BookProposal.model_validate(req.proposal)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Invalid proposal: {exc}")
try:
book, spine = await engine.confirm_proposal(book_id=req.book_id, edited_proposal=edited)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc))
except Exception as exc: # noqa: BLE001
logger.error(f"confirm_proposal failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
return {
"book": book.model_dump(mode="json"),
"spine": spine.model_dump(mode="json"),
}
@router.post("/books/confirm-spine")
async def confirm_spine(req: ConfirmSpineRequest) -> dict[str, Any]:
"""Stage 3: user confirms the spine → create pending page shells."""
engine = get_book_engine()
edited: Spine | None = None
if req.spine:
try:
edited = Spine.model_validate(req.spine)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Invalid spine: {exc}")
try:
pages = await engine.confirm_spine(
book_id=req.book_id,
edited_spine=edited,
auto_compile=req.auto_compile,
)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc))
except Exception as exc: # noqa: BLE001
logger.error(f"confirm_spine failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
return {"pages": [p.model_dump(mode="json") for p in pages]}
@router.post("/books/compile-page")
async def compile_page(req: CompilePageRequest) -> dict[str, Any]:
"""Drive the compiler for the page the user just opened (current-page priority)."""
engine = get_book_engine()
try:
page = await engine.compile_page(book_id=req.book_id, page_id=req.page_id, force=req.force)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc))
except Exception as exc: # noqa: BLE001
logger.error(f"compile_page failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
return {"page": page.model_dump(mode="json")}
@router.post("/books/regenerate-block")
async def regenerate_block(req: RegenerateBlockRequest) -> dict[str, Any]:
engine = get_book_engine()
try:
block = await engine.regenerate_block(
book_id=req.book_id,
page_id=req.page_id,
block_id=req.block_id,
params_override=req.params_override,
)
except Exception as exc: # noqa: BLE001
logger.error(f"regenerate_block failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
if block is None:
raise HTTPException(status_code=404, detail="Block not found")
return {"block": block.model_dump(mode="json")}
def _coerce_block_type(name: str) -> BlockType:
try:
return BlockType(name)
except ValueError as exc:
raise HTTPException(status_code=400, detail=f"Unknown block type: {name}") from exc
def _coerce_content_type(name: str) -> ContentType:
try:
return ContentType(name)
except ValueError as exc:
raise HTTPException(status_code=400, detail=f"Unknown content type: {name}") from exc
@router.post("/books/insert-block")
async def insert_block(req: InsertBlockRequest) -> dict[str, Any]:
engine = get_book_engine()
block_type = _coerce_block_type(req.block_type)
try:
block = await engine.insert_block(
book_id=req.book_id,
page_id=req.page_id,
block_type=block_type,
params=req.params,
position=req.position,
compile_now=req.compile_now,
)
except Exception as exc: # noqa: BLE001
logger.error(f"insert_block failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
if block is None:
raise HTTPException(status_code=404, detail="Page or chapter not found")
return {"block": block.model_dump(mode="json")}
@router.post("/books/delete-block")
async def delete_block(req: DeleteBlockRequest) -> dict[str, Any]:
engine = get_book_engine()
ok = await engine.delete_block(book_id=req.book_id, page_id=req.page_id, block_id=req.block_id)
if not ok:
raise HTTPException(status_code=404, detail="Block not found")
return {"ok": True}
@router.post("/books/move-block")
async def move_block(req: MoveBlockRequest) -> dict[str, Any]:
engine = get_book_engine()
ok = await engine.move_block(
book_id=req.book_id,
page_id=req.page_id,
block_id=req.block_id,
new_position=req.new_position,
)
if not ok:
raise HTTPException(status_code=404, detail="Block not found")
return {"ok": True}
@router.post("/books/change-block-type")
async def change_block_type(req: ChangeBlockTypeRequest) -> dict[str, Any]:
engine = get_book_engine()
new_type = _coerce_block_type(req.new_type)
try:
block = await engine.change_block_type(
book_id=req.book_id,
page_id=req.page_id,
block_id=req.block_id,
new_type=new_type,
params_override=req.params_override,
)
except Exception as exc: # noqa: BLE001
logger.error(f"change_block_type failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
if block is None:
raise HTTPException(status_code=404, detail="Block not found")
return {"block": block.model_dump(mode="json")}
@router.post("/books/deep-dive")
async def deep_dive(req: DeepDiveRequest) -> dict[str, Any]:
engine = get_book_engine()
content_type = _coerce_content_type(req.content_type)
try:
page = await engine.create_deep_dive_subpage(
book_id=req.book_id,
parent_page_id=req.parent_page_id,
topic=req.topic,
block_id=req.block_id,
content_type=content_type,
)
except Exception as exc: # noqa: BLE001
logger.error(f"deep_dive failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
if page is None:
raise HTTPException(status_code=404, detail="Parent page not found")
return {"page": page.model_dump(mode="json")}
@router.post("/books/quiz-attempt")
async def quiz_attempt(req: QuizAttemptRequest) -> dict[str, Any]:
engine = get_book_engine()
progress = await engine.record_quiz_attempt(
book_id=req.book_id,
page_id=req.page_id,
block_id=req.block_id,
question_id=req.question_id,
user_answer=req.user_answer,
is_correct=req.is_correct,
)
return {"progress": progress.model_dump(mode="json")}
@router.get("/books/{book_id}/health")
async def book_health(book_id: str) -> dict[str, Any]:
engine = get_book_engine()
drift = engine.kb_drift_report(book_id)
log = engine.log_health(book_id)
return {"kb_drift": drift, "log_health": log}
@router.post("/books/{book_id}/refresh-fingerprints")
async def refresh_fingerprints(book_id: str) -> dict[str, Any]:
engine = get_book_engine()
result = engine.refresh_kb_fingerprints(book_id)
if result is None:
raise HTTPException(status_code=404, detail="Book not found")
return result
@router.post("/books/supplement")
async def supplement(req: SupplementRequest) -> dict[str, Any]:
engine = get_book_engine()
try:
block = await engine.supplement_for_weakness(
book_id=req.book_id,
page_id=req.page_id,
topic=req.topic,
)
except Exception as exc: # noqa: BLE001
logger.error(f"supplement failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
if block is None:
raise HTTPException(status_code=404, detail="Page not found")
return {"block": block.model_dump(mode="json")}
@router.post("/books/page-chat-session")
async def set_page_chat_session(req: PageChatSessionRequest) -> dict[str, Any]:
engine = get_book_engine()
book = engine.set_page_chat_session(
book_id=req.book_id,
page_id=req.page_id,
session_id=req.session_id,
)
if book is None:
raise HTTPException(status_code=404, detail="Book or page not found")
return {"book": book.model_dump(mode="json")}
@router.post("/books/rebuild")
async def rebuild_book(req: RebuildBookRequest) -> dict[str, Any]:
engine = get_book_engine()
try:
pages = await engine.rebuild_book(book_id=req.book_id, auto_compile=req.auto_compile)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc))
except Exception as exc: # noqa: BLE001
logger.error(f"rebuild_book failed: {exc}", exc_info=True)
raise HTTPException(status_code=500, detail=str(exc))
return {"pages": [p.model_dump(mode="json") for p in pages]}
# ─────────────────────────────────────────────────────────────────────────────
# WebSocket – streamed Book events
# ─────────────────────────────────────────────────────────────────────────────
def _serialize_event(event) -> dict[str, Any]:
return {
"type": event.type.value if hasattr(event.type, "value") else str(event.type),
"source": event.source,
"stage": event.stage,
"content": event.content,
"metadata": event.metadata or {},
}
@router.websocket("/ws")
async def book_websocket(ws: WebSocket) -> None:
"""Streaming endpoint.
Client message protocol::
{"type": "create", ...CreateBookRequest fields}
{"type": "confirm_proposal", "book_id": "...", "proposal": {...}}
{"type": "confirm_spine", "book_id": "...", "spine": {...}, "auto_compile": true}
{"type": "compile_page", "book_id": "...", "page_id": "..."}
{"type": "regenerate_block", "book_id": "...", "page_id": "...", "block_id": "...", "params_override": {}}
"""
from deeptutor.api.routers.auth import ws_auth_failed, ws_require_auth
from deeptutor.multi_user.context import reset_current_user
user_token = await ws_require_auth(ws)
if user_token is ws_auth_failed:
return
await ws.accept()
closed = False
async def send(data: dict[str, Any]) -> None:
nonlocal closed
if closed:
return
try:
await ws.send_json(data)
except Exception:
closed = True
async def stream_into_socket(bus: StreamBus) -> asyncio.Task:
async def _forward() -> None:
async for event in bus.subscribe():
if event.source != BOOK_SOURCE:
continue
await send(_serialize_event(event))
if event.type == StreamEventType.STAGE_END and event.stage in {
"ideation",
"spine",
"compilation",
}:
pass # keep streaming – multiple stages per task
return asyncio.create_task(_forward())
try:
engine = get_book_engine()
while not closed:
try:
data = await ws.receive_json()
except WebSocketDisconnect:
break
except Exception as exc:
await send({"type": "error", "content": f"Bad message: {exc}"})
continue
msg_type = str(data.get("type") or "").strip()
if not msg_type:
await send({"type": "error", "content": "Missing 'type' field"})
continue
bus = StreamBus()
forward_task = await stream_into_socket(bus)
try:
if msg_type == "create":
book, proposal = await engine.create_book(
user_intent=str(data.get("user_intent") or ""),
chat_session_id=str(data.get("chat_session_id") or ""),
chat_selections=data.get("chat_selections") or [],
notebook_refs=data.get("notebook_refs") or [],
knowledge_bases=data.get("knowledge_bases") or [],
question_categories=[
int(c) for c in (data.get("question_categories") or [])
],
question_entries=[int(e) for e in (data.get("question_entries") or [])],
language=str(data.get("language") or "en"),
stream=bus,
)
await send(
{
"type": "create_result",
"book": book.model_dump(mode="json"),
"proposal": proposal.model_dump(mode="json"),
}
)
elif msg_type == "confirm_proposal":
edited: BookProposal | None = None
if data.get("proposal"):
edited = BookProposal.model_validate(data["proposal"])
book, spine = await engine.confirm_proposal(
book_id=str(data.get("book_id") or ""),
edited_proposal=edited,
stream=bus,
)
await send(
{
"type": "confirm_proposal_result",
"book": book.model_dump(mode="json"),
"spine": spine.model_dump(mode="json"),
}
)
elif msg_type == "confirm_spine":
edited_spine: Spine | None = None
if data.get("spine"):
edited_spine = Spine.model_validate(data["spine"])
pages = await engine.confirm_spine(
book_id=str(data.get("book_id") or ""),
edited_spine=edited_spine,
auto_compile=bool(data.get("auto_compile", True)),
stream=bus,
)
await send(
{
"type": "confirm_spine_result",
"pages": [p.model_dump(mode="json") for p in pages],
}
)
elif msg_type == "compile_page":
page = await engine.compile_page(
book_id=str(data.get("book_id") or ""),
page_id=str(data.get("page_id") or ""),
stream=bus,
force=bool(data.get("force", False)),
)
await send(
{
"type": "compile_page_result",
"page": page.model_dump(mode="json"),
}
)
elif msg_type == "regenerate_block":
block = await engine.regenerate_block(
book_id=str(data.get("book_id") or ""),
page_id=str(data.get("page_id") or ""),
block_id=str(data.get("block_id") or ""),
params_override=data.get("params_override"),
stream=bus,
)
await send(
{
"type": "regenerate_block_result",
"block": block.model_dump(mode="json") if block else None,
}
)
else:
await send({"type": "error", "content": f"Unknown message type: {msg_type}"})
except Exception as exc:
logger.error(f"book ws action {msg_type} failed: {exc}", exc_info=True)
await send({"type": "error", "content": str(exc)})
finally:
await bus.close()
forward_task.cancel()
try:
await forward_task
except (asyncio.CancelledError, Exception):
pass
except WebSocketDisconnect:
pass
except Exception as exc:
logger.error(f"Book WS connection error: {exc}", exc_info=True)
finally:
closed = True
try:
await ws.close()
except Exception:
pass
if user_token is not None:
try:
reset_current_user(user_token)
except Exception:
pass
deeptutor/api/routers/capabilities_settings.py
+38
View File
@@ -0,0 +1,38 @@
"""Capabilities settings endpoint.
Surfaces the per-capability tunables (temperature, max_tokens, stage
budgets, iteration limits) currently scattered across
``data/user/settings/agents.yaml`` and ``data/user/settings/main.yaml``.
Mirrors the pattern used by ``/api/v1/memory/settings``:
* ``GET /settings`` → returns the full schema with defaults merged in.
* ``PUT /settings`` → merges payload back into both YAML files and
returns the new state.
Validation lives in
:mod:`deeptutor.services.config.capabilities_settings` so the API stays
a thin transport layer.
"""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter
router = APIRouter()
@router.get("/settings")
async def get_capabilities_settings_endpoint() -> dict[str, Any]:
from deeptutor.services.config.capabilities_settings import capabilities_settings_dict
return capabilities_settings_dict()
@router.put("/settings")
async def put_capabilities_settings(payload: dict[str, Any]) -> dict[str, Any]:
from deeptutor.services.config.capabilities_settings import save_capabilities_settings
return save_capabilities_settings(payload)
deeptutor/api/routers/chat.py
+249
View File
@@ -0,0 +1,249 @@
"""
Chat API Router
================
WebSocket endpoint for lightweight chat with session management.
REST endpoints for session operations.
"""
import logging
from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect
from deeptutor.agents.chat import ChatAgent, SessionManager
from deeptutor.services.config import PROJECT_ROOT, load_config_with_main
from deeptutor.services.llm.config import get_llm_config
from deeptutor.services.settings.interface_settings import get_ui_language
config = load_config_with_main("main.yaml", PROJECT_ROOT)
log_dir = config.get("paths", {}).get("user_log_dir") or config.get("logging", {}).get("log_dir")
logger = logging.getLogger(__name__)
router = APIRouter()
def _get_session_manager() -> SessionManager:
return SessionManager()
# =============================================================================
# REST Endpoints for Session Management
# =============================================================================
@router.get("/chat/sessions")
async def list_sessions(limit: int = 20):
return _get_session_manager().list_sessions(limit=limit, include_messages=False)
@router.get("/chat/sessions/{session_id}")
async def get_session(session_id: str):
session = _get_session_manager().get_session(session_id)
if not session:
raise HTTPException(status_code=404, detail="Session not found")
return session
@router.delete("/chat/sessions/{session_id}")
async def delete_session(session_id: str):
if _get_session_manager().delete_session(session_id):
return {"status": "deleted", "session_id": session_id}
raise HTTPException(status_code=404, detail="Session not found")
# =============================================================================
# WebSocket Endpoint for Chat
# =============================================================================
@router.websocket("/chat")
async def websocket_chat(websocket: WebSocket):
from deeptutor.api.routers.auth import ws_auth_failed, ws_require_auth
from deeptutor.multi_user.context import reset_current_user
user_token = await ws_require_auth(websocket)
if user_token is ws_auth_failed:
return
await websocket.accept()
try:
while True:
data = await websocket.receive_json()
requested_language = str(data.get("language") or "").lower().strip()
language = (
"zh"
if requested_language.startswith("zh")
else "en"
if requested_language.startswith("en")
else get_ui_language(default=config.get("system", {}).get("language", "en"))
)
message = data.get("message", "").strip()
session_id = data.get("session_id")
explicit_history = data.get("history")
kb_name = data.get("kb_name", "")
enable_rag = data.get("enable_rag", False)
enable_web_search = data.get("enable_web_search", False)
if not message:
await websocket.send_json({"type": "error", "message": "Message is required"})
continue
logger.info(
f"Chat request: session={session_id}, "
f"message={message[:50]}..., rag={enable_rag}, web={enable_web_search}"
)
try:
sm = _get_session_manager()
if session_id:
session = sm.get_session(session_id)
if not session:
session = sm.create_session(
title=message[:50] + ("..." if len(message) > 50 else ""),
settings={
"kb_name": kb_name,
"enable_rag": enable_rag,
"enable_web_search": enable_web_search,
},
)
session_id = session["session_id"]
else:
session = sm.create_session(
title=message[:50] + ("..." if len(message) > 50 else ""),
settings={
"kb_name": kb_name,
"enable_rag": enable_rag,
"enable_web_search": enable_web_search,
},
)
session_id = session["session_id"]
await websocket.send_json(
{
"type": "session",
"session_id": session_id,
}
)
if explicit_history is not None:
history = explicit_history
else:
history = [
{"role": msg["role"], "content": msg["content"]}
for msg in session.get("messages", [])
]
sm.add_message(
session_id=session_id,
role="user",
content=message,
)
try:
llm_config = get_llm_config()
api_key = llm_config.api_key
base_url = llm_config.base_url
api_version = getattr(llm_config, "api_version", None)
except Exception:
api_key = None
base_url = None
api_version = None
agent = ChatAgent(
language=language,
config=config,
api_key=api_key,
base_url=base_url,
api_version=api_version,
)
if enable_rag and kb_name:
await websocket.send_json(
{
"type": "status",
"stage": "rag",
"message": f"Searching knowledge base: {kb_name}...",
}
)
if enable_web_search:
await websocket.send_json(
{
"type": "status",
"stage": "web",
"message": "Searching the web...",
}
)
await websocket.send_json(
{
"type": "status",
"stage": "generating",
"message": "Generating response...",
}
)
full_response = ""
sources = {"rag": [], "web": []}
stream_generator = await agent.process(
message=message,
history=history,
kb_name=kb_name,
enable_rag=enable_rag,
enable_web_search=enable_web_search,
stream=True,
)
async for chunk_data in stream_generator:
if chunk_data["type"] == "chunk":
await websocket.send_json(
{
"type": "stream",
"content": chunk_data["content"],
}
)
full_response += chunk_data["content"]
elif chunk_data["type"] == "complete":
full_response = chunk_data["response"]
sources = chunk_data.get("sources", {"rag": [], "web": []})
if sources.get("rag") or sources.get("web"):
await websocket.send_json({"type": "sources", **sources})
await websocket.send_json(
{
"type": "result",
"content": full_response,
}
)
sm.add_message(
session_id=session_id,
role="assistant",
content=full_response,
sources=sources if (sources.get("rag") or sources.get("web")) else None,
)
logger.info(f"Chat completed: session={session_id}, {len(full_response)} chars")
except Exception as e:
logger.error(f"Chat processing error: {e}")
await websocket.send_json({"type": "error", "message": str(e)})
except WebSocketDisconnect:
logger.debug("Client disconnected from chat")
except Exception as e:
logger.error(f"WebSocket error: {e}")
try:
await websocket.send_json({"type": "error", "message": str(e)})
except Exception:
pass
finally:
if user_token is not None:
try:
reset_current_user(user_token)
except Exception:
pass
deeptutor/api/routers/co_writer.py
+618
View File
@@ -0,0 +1,618 @@
import asyncio
from datetime import datetime
import json
import logging
import re
import traceback
from typing import AsyncGenerator, Literal
import uuid
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from deeptutor.co_writer.edit_agent import (
EditAgent,
append_history,
load_history,
print_stats,
tool_calls_dir,
)
from deeptutor.co_writer.storage import (
CoWriterDocument,
CoWriterDocumentSummary,
get_co_writer_storage,
)
from deeptutor.core.stream_bus import StreamBus
from deeptutor.services.config import PROJECT_ROOT, load_config_with_main
from deeptutor.services.llm import clean_thinking_tags
from deeptutor.services.settings.interface_settings import get_ui_language
router = APIRouter()
# Initialize logger with config
config = load_config_with_main("main.yaml", PROJECT_ROOT)
log_dir = config.get("paths", {}).get("user_log_dir") or config.get("logging", {}).get("log_dir")
logger = logging.getLogger(__name__)
_edit_agent: EditAgent | None = None
def _current_language() -> str:
# Prefer UI settings, fall back to main.yaml system.language
return get_ui_language(default=config.get("system", {}).get("language", "en"))
def get_edit_agent() -> EditAgent:
"""
Get the singleton EditAgent instance with refreshed configuration.
Uses a singleton pattern with refresh_config() to ensure:
1. Efficient reuse of the agent instance
2. Latest LLM configuration from Settings is always used
"""
global _edit_agent
lang = _current_language()
if _edit_agent is None or getattr(_edit_agent, "language", None) != lang:
_edit_agent = EditAgent(language=lang)
# Refresh config to pick up any changes from Settings
_edit_agent.refresh_config()
return _edit_agent
# Generous ceilings — they exist to stop runaway payloads (OOM / surprise
# LLM bills), not to constrain normal documents.
_MAX_DOC_CHARS = 600_000
_MAX_SELECTION_CHARS = 120_000
_MAX_INSTRUCTION_CHARS = 10_000
class EditRequest(BaseModel):
text: str = Field(max_length=_MAX_DOC_CHARS)
instruction: str = Field(max_length=_MAX_INSTRUCTION_CHARS)
action: Literal["rewrite", "shorten", "expand"] = "rewrite"
source: Literal["rag", "web"] | None = None
kb_name: str | None = None
class EditResponse(BaseModel):
edited_text: str
operation_id: str
class ReactEditRequest(BaseModel):
selected_text: str = Field(max_length=_MAX_SELECTION_CHARS)
instruction: str = Field(default="", max_length=_MAX_INSTRUCTION_CHARS)
mode: Literal["rewrite", "shorten", "expand", "none"] = "rewrite"
tools: list[Literal["rag", "web"]] = []
kb_name: str | None = None
class ReactEditResponse(BaseModel):
edited_text: str
operation_id: str
tools_used: list[str] = []
class AutoMarkRequest(BaseModel):
text: str = Field(max_length=_MAX_DOC_CHARS)
class AutoMarkResponse(BaseModel):
marked_text: str
operation_id: str
def _default_mode_instruction(mode: str, language: str) -> str:
zh = language.startswith("zh")
defaults = {
"rewrite": "润色这段 markdown，保持原意、结构和语气自然。",
"shorten": "压缩这段 markdown，让表达更精炼，同时保留关键信息。",
"expand": "扩展这段 markdown，补充必要细节，同时保持原有风格。",
"none": "根据用户要求编辑这段 markdown。",
}
if zh:
return defaults.get(mode, defaults["none"])
defaults_en = {
"rewrite": "Rewrite this markdown snippet while preserving its meaning, structure, and tone.",
"shorten": "Shorten this markdown snippet while preserving the key information.",
"expand": "Expand this markdown snippet with helpful detail while keeping the original style.",
"none": "Edit this markdown snippet according to the user's request.",
}
return defaults_en.get(mode, defaults_en["none"])
def _build_react_edit_prompt(
*,
selected_text: str,
instruction: str,
mode: str,
language: str,
context: str = "",
) -> str:
user_instruction = instruction.strip() or _default_mode_instruction(mode, language)
if language.startswith("zh"):
context_block = f"参考资料（按需取用，不必全部使用）:\n{context}\n\n" if context else ""
return (
"你正在编辑一段从 Markdown 编辑器里选中的文本。\n\n"
f"编辑模式: {mode}\n"
f"用户要求: {user_instruction}\n\n"
f"{context_block}"
"待编辑的选中文本:\n"
"```markdown\n"
f"{selected_text}\n"
"```\n\n"
"要求:\n"
"1. 只输出编辑后的那段 Markdown 文本，供编辑器直接替换。\n"
"2. 不要输出解释、标题、前后缀、代码围栏。\n"
"3. 保持 Markdown 语法合法。\n"
"4. 如果给了参考资料，把相关事实自然融入结果，不要提工具或资料来源。\n"
)
context_block = (
f"Reference material (use what is relevant, ignore the rest):\n{context}\n\n"
if context
else ""
)
return (
"You are editing a text selection from a Markdown editor.\n\n"
f"Edit mode: {mode}\n"
f"User request: {user_instruction}\n\n"
f"{context_block}"
"Selected text to edit:\n"
"```markdown\n"
f"{selected_text}\n"
"```\n\n"
"Requirements:\n"
"1. Output only the edited Markdown snippet for direct replacement.\n"
"2. Do not include explanations, headings, prefixes, suffixes, or code fences.\n"
"3. Keep the Markdown valid.\n"
"4. If reference material is given, weave the relevant facts in naturally "
"without mentioning tools or sources.\n"
)
def _strip_markdown_fence(text: str) -> str:
cleaned = text.strip()
if cleaned.startswith("```") and cleaned.endswith("```"):
lines = cleaned.splitlines()
if len(lines) >= 3:
return "\n".join(lines[1:-1]).strip()
return cleaned
def _clean_react_edit_output(text: str, *, binding: str | None, model: str | None) -> str:
return _strip_markdown_fence(clean_thinking_tags(text, binding, model))
def _normalize_react_edit_tools(tools: list[str] | None) -> list[str]:
allowed = {"rag", "web"}
result: list[str] = []
for tool in tools or []:
name = str(tool or "").strip()
if name in allowed and name not in result:
result.append(name)
return result
def _prepare_react_edit_request(
request: ReactEditRequest, language: str
) -> tuple[str, str, list[str]]:
tools = _normalize_react_edit_tools(request.tools)
instruction = request.instruction.strip()
if request.mode == "none" and not instruction:
detail = (
"请输入编辑要求，或选择 shorten / expand / rewrite 模式。"
if language.startswith("zh")
else "Provide an edit instruction, or choose shorten / expand / rewrite mode."
)
raise HTTPException(status_code=400, detail=detail)
selected_text = request.selected_text.strip("\n")
if not selected_text.strip():
detail = (
"请先选中一段文本。"
if language.startswith("zh")
else "Please select a text passage first."
)
raise HTTPException(status_code=400, detail=detail)
return selected_text, instruction, tools
_TRACE_PREVIEW_CHARS = 1200
def _trace_preview(text: str) -> str:
cleaned = text.strip()
if len(cleaned) <= _TRACE_PREVIEW_CHARS:
return cleaned
return cleaned[:_TRACE_PREVIEW_CHARS].rstrip() + "…"
async def _run_react_edit(
request: ReactEditRequest,
*,
language: str,
stream: StreamBus | None = None,
) -> dict[str, object]:
selected_text, instruction, tools = _prepare_react_edit_request(request, language)
operation_id = datetime.now().strftime("%Y%m%d_%H%M%S") + "_" + uuid.uuid4().hex[:6]
agent = get_edit_agent()
# Optional reference retrieval before the edit. Each tool degrades to a
# plain edit on failure — retrieval must never block the user's edit.
query = instruction or selected_text[:400]
context_blocks: list[str] = []
tools_used: list[str] = []
for tool in tools:
kb_name = request.kb_name if tool == "rag" else None
if tool == "rag" and not kb_name:
continue
if stream is not None:
await stream.tool_call(
tool,
{"query": query, **({"kb_name": kb_name} if kb_name else {})},
source="co_writer_react_edit",
stage="exploring",
)
context, _file = await agent.gather_context(
source=tool,
query=query,
kb_name=kb_name,
operation_id=operation_id,
)
if stream is not None:
await stream.tool_result(
tool,
_trace_preview(context) if context else "(no result)",
source="co_writer_react_edit",
stage="exploring",
)
if context:
context_blocks.append(context)
tools_used.append(tool)
system_prompt = (
"You are an expert markdown editor."
if not language.startswith("zh")
else "你是一个严格的 Markdown 编辑助手。"
)
prompt = _build_react_edit_prompt(
selected_text=selected_text,
instruction=instruction,
mode=request.mode,
language=language,
context="\n\n".join(context_blocks),
)
response_chunks: list[str] = []
async def _consume() -> None:
async for chunk in agent.stream_llm(
user_prompt=prompt,
system_prompt=system_prompt,
stage=f"react_edit_{request.mode}",
):
if not chunk:
continue
response_chunks.append(chunk)
if stream is not None:
await stream.content(
chunk,
source="co_writer_react_edit",
stage="responding",
)
if stream is not None:
async with stream.stage("responding", source="co_writer_react_edit"):
await _consume()
else:
await _consume()
edited_text = _clean_react_edit_output(
"".join(response_chunks),
binding=agent.binding,
model=agent.get_model(),
)
append_history(
{
"id": operation_id,
"timestamp": datetime.now().isoformat(),
"action": "react_edit",
"mode": request.mode,
"tools": tools_used,
"kb_name": request.kb_name,
"input": {
"selected_text": request.selected_text,
"instruction": instruction,
},
"output": {"edited_text": edited_text},
"model": agent.get_model(),
}
)
print_stats()
result = {
"edited_text": edited_text,
"operation_id": operation_id,
"tools_used": tools_used,
}
if stream is not None:
await stream.result(result, source="co_writer_react_edit")
return result
async def _stream_react_edit(request: ReactEditRequest) -> AsyncGenerator[str, None]:
language = _current_language()
bus = StreamBus()
error_holder: dict[str, str] = {}
result_holder: dict[str, object] | None = None
async def _run() -> None:
nonlocal result_holder
try:
result_holder = await _run_react_edit(request, language=language, stream=bus)
except HTTPException as exc:
error_holder["detail"] = str(exc.detail)
except Exception as exc:
error_holder["detail"] = str(exc)
finally:
await bus.close()
task = asyncio.create_task(_run())
try:
async for event in bus.subscribe():
yield f"event: stream\ndata: {json.dumps(event.to_dict(), default=str)}\n\n"
await task
if error_holder:
yield f"event: error\ndata: {json.dumps(error_holder, default=str)}\n\n"
else:
yield f"event: result\ndata: {json.dumps(result_holder or {}, default=str)}\n\n"
finally:
if not task.done():
task.cancel()
@router.post("/edit", response_model=EditResponse)
async def edit_text(request: EditRequest):
try:
# Get agent with refreshed LLM configuration from Settings
agent = get_edit_agent()
result = await agent.process(
text=request.text,
instruction=request.instruction,
action=request.action,
source=request.source,
kb_name=request.kb_name,
)
# Print token stats
print_stats()
return result
except Exception as e:
traceback.print_exc()
raise HTTPException(status_code=500, detail=str(e))
@router.post("/edit_react", response_model=ReactEditResponse)
async def edit_text_react(request: ReactEditRequest):
try:
return await _run_react_edit(request, language=_current_language())
except HTTPException:
raise
except Exception as e:
traceback.print_exc()
raise HTTPException(status_code=500, detail=str(e))
@router.post("/edit_react/stream")
async def edit_text_react_stream(request: ReactEditRequest):
try:
_prepare_react_edit_request(request, _current_language())
except HTTPException:
raise
return StreamingResponse(
_stream_react_edit(request),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
@router.post("/automark", response_model=AutoMarkResponse)
async def auto_mark_text(request: AutoMarkRequest):
"""AI auto-mark text"""
try:
# Get agent with refreshed LLM configuration from Settings
agent = get_edit_agent()
result = await agent.auto_mark(text=request.text)
# Print token stats
print_stats()
return result
except Exception as e:
traceback.print_exc()
raise HTTPException(status_code=500, detail=str(e))
@router.get("/history")
async def get_history():
"""Get all operation history"""
try:
history = load_history()
return {"history": history, "total": len(history)}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/history/{operation_id}")
async def get_operation(operation_id: str):
"""Get single operation details"""
try:
history = load_history()
for op in history:
if op.get("id") == operation_id:
return op
raise HTTPException(status_code=404, detail="Operation not found")
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/tool_calls/{operation_id}")
async def get_tool_call(operation_id: str):
"""Get tool call details"""
try:
# Find matching file
for filepath in tool_calls_dir().glob(f"{operation_id}_*.json"):
with open(filepath, encoding="utf-8") as f:
return json.load(f)
raise HTTPException(status_code=404, detail="Tool call not found")
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
# ─────────────────────────────────────────────────────────────────────────────
# Document CRUD (multi-project Co-Writer)
# ─────────────────────────────────────────────────────────────────────────────
# Storage builds paths as `documents/doc_{doc_id}`; an unvalidated id like
# "a/../../x" would escape the documents root (and DELETE runs rmtree).
_DOC_ID_RE = re.compile(r"^[0-9a-f]{8,32}$")
def _validate_doc_id(doc_id: str) -> str:
if not _DOC_ID_RE.fullmatch(doc_id):
raise HTTPException(status_code=404, detail="Document not found")
return doc_id
class CreateDocumentRequest(BaseModel):
title: str | None = None
content: str = ""
class UpdateDocumentRequest(BaseModel):
title: str | None = None
content: str | None = None
class DocumentResponse(BaseModel):
id: str
title: str
content: str
created_at: float
updated_at: float
@classmethod
def from_model(cls, doc: CoWriterDocument) -> "DocumentResponse":
return cls(
id=doc.id,
title=doc.title,
content=doc.content,
created_at=doc.created_at,
updated_at=doc.updated_at,
)
class DocumentSummaryResponse(BaseModel):
id: str
title: str
created_at: float
updated_at: float
preview: str = ""
@classmethod
def from_summary(cls, summary: CoWriterDocumentSummary) -> "DocumentSummaryResponse":
return cls(
id=summary.id,
title=summary.title,
created_at=summary.created_at,
updated_at=summary.updated_at,
preview=summary.preview,
)
@router.get("/documents")
async def list_documents() -> dict[str, list[DocumentSummaryResponse]]:
"""List all Co-Writer documents (summary view, sorted by recency)."""
try:
storage = get_co_writer_storage()
summaries = storage.list_documents()
return {"documents": [DocumentSummaryResponse.from_summary(s) for s in summaries]}
except Exception as e:
traceback.print_exc()
raise HTTPException(status_code=500, detail=str(e))
@router.post("/documents", response_model=DocumentResponse)
async def create_document(request: CreateDocumentRequest) -> DocumentResponse:
"""Create a new Co-Writer document."""
try:
storage = get_co_writer_storage()
document = storage.create_document(title=request.title, content=request.content)
return DocumentResponse.from_model(document)
except Exception as e:
traceback.print_exc()
raise HTTPException(status_code=500, detail=str(e))
@router.get("/documents/{doc_id}", response_model=DocumentResponse)
async def get_document(doc_id: str) -> DocumentResponse:
"""Get a single Co-Writer document by id."""
try:
storage = get_co_writer_storage()
document = storage.load_document(_validate_doc_id(doc_id))
if document is None:
raise HTTPException(status_code=404, detail="Document not found")
return DocumentResponse.from_model(document)
except HTTPException:
raise
except Exception as e:
traceback.print_exc()
raise HTTPException(status_code=500, detail=str(e))
@router.put("/documents/{doc_id}", response_model=DocumentResponse)
async def update_document(doc_id: str, request: UpdateDocumentRequest) -> DocumentResponse:
"""Update a Co-Writer document (title and/or content)."""
try:
storage = get_co_writer_storage()
document = storage.update_document(
_validate_doc_id(doc_id), title=request.title, content=request.content
)
if document is None:
raise HTTPException(status_code=404, detail="Document not found")
return DocumentResponse.from_model(document)
except HTTPException:
raise
except Exception as e:
traceback.print_exc()
raise HTTPException(status_code=500, detail=str(e))
@router.delete("/documents/{doc_id}")
async def delete_document(doc_id: str) -> dict[str, bool]:
"""Delete a Co-Writer document."""
try:
storage = get_co_writer_storage()
_validate_doc_id(doc_id)
if not storage.doc_exists(doc_id):
raise HTTPException(status_code=404, detail="Document not found")
success = storage.delete_document(doc_id)
return {"deleted": success}
except HTTPException:
raise
except Exception as e:
traceback.print_exc()
raise HTTPException(status_code=500, detail=str(e))
deeptutor/api/routers/dashboard.py
+61
View File
@@ -0,0 +1,61 @@
"""Dashboard API backed by the unified SQLite session store."""
from typing import Any
from fastapi import APIRouter, HTTPException
from deeptutor.services.session import get_session_store
router = APIRouter()
@router.get("/recent")
async def get_recent_activities(limit: int = 50, type: str | None = None):
store = get_session_store()
sessions = await store.list_sessions(limit=limit, offset=0)
activities: list[dict[str, Any]] = []
for session in sessions:
capability = str(session.get("capability") or "chat")
activity_type = capability.replace("deep_", "")
if type is not None and activity_type != type:
continue
activities.append(
{
"id": session.get("session_id"),
"type": activity_type,
"capability": capability,
"title": session.get("title", "Untitled"),
"timestamp": session.get("updated_at", session.get("created_at", 0)),
"summary": (session.get("last_message") or "")[:160],
"session_ref": f"sessions/{session.get('session_id')}",
"message_count": session.get("message_count", 0),
"status": session.get("status", "idle"),
"active_turn_id": session.get("active_turn_id"),
}
)
return activities[:limit]
@router.get("/{entry_id}")
async def get_activity_entry(entry_id: str):
store = get_session_store()
session = await store.get_session_with_messages(entry_id)
if session is None:
raise HTTPException(status_code=404, detail="Entry not found")
capability = str(session.get("capability") or "chat")
return {
"id": session.get("session_id"),
"type": capability.replace("deep_", ""),
"capability": capability,
"title": session.get("title"),
"timestamp": session.get("updated_at", session.get("created_at")),
"content": {
"messages": session.get("messages", []),
"active_turns": session.get("active_turns", []),
"status": session.get("status", "idle"),
"summary": session.get("compressed_summary", ""),
},
}
deeptutor/api/routers/imports.py
+146
View File
@@ -0,0 +1,146 @@
"""
Import chat histories from external coding CLIs (Claude Code, Codex) into the
user's learning space as normal, re-openable sessions.
Reading the user's local ``~/.claude`` / ``~/.codex`` happens in the browser
(File System Access API) — those files live on the user's machine, not the
server. The browser normalizes each conversation to the small JSON shape below
and POSTs it here; this router only validates and persists. Imported sessions
share the session tables with native chats (so the chat loop can re-open and
continue them) but carry an ``imported_`` id prefix that keeps them in their
own Space category. Re-importing the same folder is idempotent (dedup by id).
"""
from __future__ import annotations
import logging
from typing import Any
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field, field_validator
from deeptutor.services.session import (
get_sqlite_session_store,
make_imported_session_id,
)
logger = logging.getLogger(__name__)
router = APIRouter()
# Browser adapters only emit these; reject anything else so a malformed payload
# can never seed an unsupported provider category.
_ALLOWED_SOURCES = {"claude_code", "codex"}
# Defensive ceilings — a single import request should never grow unbounded.
_MAX_SESSIONS_PER_REQUEST = 1000
_MAX_MESSAGES_PER_SESSION = 10000
class ImportedMessage(BaseModel):
role: str = Field(..., pattern="^(user|assistant)$")
content: str = ""
created_at: float | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class ImportedSession(BaseModel):
external_id: str = Field(..., min_length=1, max_length=256)
title: str = ""
source_cwd: str = ""
created_at: float
updated_at: float
messages: list[ImportedMessage] = Field(default_factory=list)
class ChatHistoryImportRequest(BaseModel):
source: str = Field(..., min_length=1)
# All sessions in one request belong to one agent (a named, scoped slice of
# a source folder). Empty when the client hasn't adopted the agent model yet
# — the backend stays backwards-compatible by simply omitting attribution.
agent_id: str = Field(default="", max_length=256)
agent_name: str = Field(default="", max_length=256)
sessions: list[ImportedSession] = Field(default_factory=list)
@field_validator("source")
@classmethod
def _normalize_source(cls, value: str) -> str:
normalized = (value or "").strip().lower()
if normalized not in _ALLOWED_SOURCES:
raise ValueError(f"Unsupported import source: {value!r}")
return normalized
@router.post("/chat-history")
async def import_chat_history(payload: ChatHistoryImportRequest) -> dict[str, Any]:
if not payload.sessions:
raise HTTPException(status_code=400, detail="No sessions to import")
if len(payload.sessions) > _MAX_SESSIONS_PER_REQUEST:
raise HTTPException(
status_code=413,
detail=f"Too many sessions in one request (max {_MAX_SESSIONS_PER_REQUEST})",
)
store = get_sqlite_session_store()
imported = 0
skipped = 0
results: list[dict[str, Any]] = []
for incoming in payload.sessions:
# Drop content-less rows (e.g. tool-only turns the adapter could not
# reduce to text) so the transcript stays a clean human conversation.
messages = [m for m in incoming.messages if (m.content or "").strip()]
if not messages:
skipped += 1
results.append(
{"external_id": incoming.external_id, "imported": False, "reason": "empty"}
)
continue
if len(messages) > _MAX_MESSAGES_PER_SESSION:
messages = messages[:_MAX_MESSAGES_PER_SESSION]
session_id = make_imported_session_id(payload.source, incoming.external_id)
import_meta: dict[str, Any] = {
"source": payload.source,
"source_cwd": incoming.source_cwd,
"external_id": incoming.external_id,
}
if payload.agent_id:
import_meta["agent_id"] = payload.agent_id
if payload.agent_name:
import_meta["agent_name"] = payload.agent_name
preferences = {"import": import_meta}
try:
result = await store.import_session(
session_id,
incoming.title,
incoming.created_at,
incoming.updated_at,
preferences,
[m.model_dump() for m in messages],
)
except Exception:
logger.exception("failed to import session %s", incoming.external_id)
skipped += 1
results.append(
{"external_id": incoming.external_id, "imported": False, "reason": "error"}
)
continue
if result.get("imported"):
imported += 1
else:
skipped += 1
results.append({"external_id": incoming.external_id, **result})
return {"imported": imported, "skipped": skipped, "sessions": results}
@router.get("/chat-history")
async def list_imported_chat_history(
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
) -> dict[str, Any]:
store = get_sqlite_session_store()
sessions = await store.list_imported_sessions(limit=limit, offset=offset)
return {"sessions": sessions}
deeptutor/api/routers/knowledge.py
+2761
View File
File diff suppressed because it is too large Load Diff
deeptutor/api/routers/mastery_path.py
+308
View File
@@ -0,0 +1,308 @@
"""Guided Learning API Router."""
from __future__ import annotations
import html
import json
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from pydantic import ValidationError as PydanticValidationError
from deeptutor.learning import policy as learning_policy
from deeptutor.learning import prompts as learning_prompts
from deeptutor.learning.models import (
KnowledgePoint,
KnowledgeType,
LearningModule,
LearningStage,
)
from deeptutor.learning.service import LearningService
from deeptutor.learning.storage import LearningStore
from deeptutor.services.settings.interface_settings import get_ui_language
from deeptutor.utils.json_parser import parse_json_response
router = APIRouter()
def get_learning_service() -> LearningService:
# Create a fresh store + service per request to avoid object-level race conditions.
store = LearningStore()
return LearningService(store)
def _validate_book_id(book_id: str) -> None:
"""Reject empty or path-traversal-bearing book ids (shared by all endpoints)."""
if not book_id or ".." in book_id or "/" in book_id or "\\" in book_id or ":" in book_id:
raise HTTPException(status_code=400, detail="Invalid book_id")
def _parse_modules(body_modules: list[dict]) -> list[LearningModule]:
"""Parse raw module dicts into LearningModule objects (shared by init/replace)."""
modules: list[LearningModule] = []
for i, m in enumerate(body_modules):
kps_data = m.get("knowledge_points", [])
try:
kps = [KnowledgePoint(**kp) for kp in kps_data]
except PydanticValidationError as exc:
raise HTTPException(
status_code=422,
detail=f"Invalid knowledge_point data in modules[{i}]: {exc.errors()}",
) from exc
# Remove knowledge_points from m to avoid duplicate argument to LearningModule.
m_clean = {k: v for k, v in m.items() if k != "knowledge_points"}
try:
modules.append(LearningModule(knowledge_points=kps, **m_clean))
except PydanticValidationError as exc:
raise HTTPException(
status_code=422,
detail=f"Invalid module data in modules[{i}]: {exc.errors()}",
) from exc
return modules
def _validate_runnable_modules(modules: list[LearningModule], *, status_code: int = 400) -> None:
if not modules:
raise HTTPException(
status_code=status_code, detail="At least one learning module is required"
)
for mod in modules:
if not mod.knowledge_points:
raise HTTPException(
status_code=status_code,
detail=f"Module {mod.id!r} must contain at least one knowledge point",
)
async def _cancel_active_learning_turn(book_id: str) -> None:
from deeptutor.services.session import get_turn_runtime_manager
runtime = get_turn_runtime_manager()
active_turn = await runtime.store.get_active_turn(book_id)
if active_turn:
await runtime.cancel_turn(active_turn["id"])
# ── Request models ───────────────────────────────────────────────────────────
class InitModulesRequest(BaseModel):
modules: list[dict] # list of LearningModule-compatible dicts
class ChapterImport(BaseModel):
title: str
knowledge_points: list[str] = []
class ImportFromBookRequest(BaseModel):
chapters: list[ChapterImport]
# ── Endpoints ────────────────────────────────────────────────────────────────
@router.get("/progress")
async def list_all_progress():
service = get_learning_service()
return service.list_progress()
@router.get("/progress/{book_id}")
async def get_progress(book_id: str):
_validate_book_id(book_id)
service = get_learning_service()
progress = service.get_or_create(book_id)
return progress.model_dump()
@router.get("/progress/{book_id}/map")
async def get_progress_map(book_id: str):
"""The dashboard view of a path: the gate-decided next step plus a map of
every objective's status (new / learning / mastered). The per-type gate
lives in ``learning.policy`` so the dashboard and the tutor agree."""
_validate_book_id(book_id)
service = get_learning_service()
progress = service.get_or_create(book_id)
return {
"book_id": book_id,
"next": learning_policy.next_objective(progress).to_dict(),
"map": learning_policy.map_summary(progress),
}
@router.post("/progress/{book_id}/init-modules")
async def init_modules(book_id: str, body: InitModulesRequest):
_validate_book_id(book_id)
modules = _parse_modules(body.modules)
_validate_runnable_modules(modules)
await _cancel_active_learning_turn(book_id)
service = get_learning_service()
progress = service.get_or_create(book_id)
service.init_modules(progress, modules)
progress.current_module_id = modules[0].id
progress.current_kp_index = 0
service.save(progress)
return {"status": "ok", "module_count": len(modules)}
@router.post("/progress/{book_id}/import-from-book")
async def import_from_book(book_id: str, body: ImportFromBookRequest):
_validate_book_id(book_id)
modules = []
for i, ch in enumerate(body.chapters):
kps = [
KnowledgePoint(
id=f"{book_id}_ch{i}_kp{j}",
name=kp_name,
type=KnowledgeType("concept"),
module_id=f"{book_id}_ch{i}",
)
for j, kp_name in enumerate(ch.knowledge_points)
]
modules.append(
LearningModule(
id=f"{book_id}_ch{i}",
name=ch.title or f"Chapter {i + 1}",
order=i,
pass_threshold=0.7,
knowledge_points=kps,
)
)
_validate_runnable_modules(modules)
await _cancel_active_learning_turn(book_id)
service = get_learning_service()
progress = service.get_or_create(book_id)
service.init_modules(progress, modules)
progress.current_module_id = modules[0].id
progress.current_kp_index = 0
service.save(progress)
return {"status": "ok", "module_count": len(modules)}
@router.delete("/progress/{book_id}")
async def delete_progress(book_id: str):
_validate_book_id(book_id)
store = LearningStore()
if not store.exists(book_id):
raise HTTPException(status_code=404, detail="Progress not found")
store.delete(book_id)
return {"status": "ok"}
@router.post("/progress/{book_id}/redo")
async def redo_progress(book_id: str):
_validate_book_id(book_id)
store = LearningStore()
progress = store.load(book_id)
if progress is None:
raise HTTPException(status_code=404, detail="Progress not found")
progress.current_stage = LearningStage.DIAGNOSTIC
progress.mastery_levels = {}
progress.qualitative_mastery = {}
progress.quiz_attempts = []
progress.error_records = []
progress.repetition_states = {}
progress.review_queue = []
progress.pending_question = None
progress.feynman_retries = {}
progress.feynman_explanations = {}
progress.stage_failure_counts = {}
progress.stage_failure_notes = {}
progress.diagnostic = None
progress.current_kp_index = 0
progress.current_module_id = progress.modules[0].id if progress.modules else ""
store.save(progress)
return {"status": "ok"}
class NotebookRecordInput(BaseModel):
id: str
type: str = "note"
title: str = ""
output: str = ""
class GenerateFromNotebookRequest(BaseModel):
notebook_id: str
records: list[NotebookRecordInput]
@router.post("/progress/{book_id}/generate-from-notebook")
async def generate_from_notebook(book_id: str, body: GenerateFromNotebookRequest):
_validate_book_id(book_id)
if not body.records:
raise HTTPException(status_code=400, detail="No records provided")
records_data = [
{
"type": html.escape(r.type[:50], quote=False),
"title": html.escape(r.title[:200], quote=False),
"output": html.escape(r.output[:500], quote=False),
}
for r in body.records[:20]
]
records_json = json.dumps(records_data, ensure_ascii=False)
from deeptutor.services.llm import complete
language = get_ui_language()
system_prompt, prompt = learning_prompts.notebook_generation_prompts(language, records_json)
response = await complete(prompt=prompt, system_prompt=system_prompt)
# LLMs commonly fence/slightly-malform JSON; use the shared fence-stripping
# repair parser instead of bare json.loads so the common case isn't a 502.
data = parse_json_response(response, fallback=None)
if not isinstance(data, dict):
raise HTTPException(status_code=502, detail="LLM returned invalid JSON")
modules_raw = data.get("modules", [])
if not isinstance(modules_raw, list):
raise HTTPException(
status_code=502, detail="LLM returned invalid structure: modules is not a list"
)
_ALLOWED_KP_TYPES = {"memory", "concept", "procedure", "design"}
modules = []
for i, m in enumerate(modules_raw):
if not isinstance(m, dict) or "name" not in m:
continue
fallback_name = learning_prompts.default_module_name(language, i + 1)
module_name = str(m.get("name") or fallback_name).strip()[:200] or fallback_name
kps = []
for j, kp in enumerate(m.get("knowledge_points", [])):
if not isinstance(kp, dict) or "name" not in kp:
continue
kp_name = str(kp["name"]).strip()[:200]
if len(kp_name) < 2:
continue
kp_type = str(kp.get("type", "concept")).strip()
if kp_type not in _ALLOWED_KP_TYPES:
kp_type = "concept"
kps.append(
KnowledgePoint(
id=f"{book_id}_nb{i}_kp{j}",
name=kp_name,
type=KnowledgeType(kp_type),
module_id=f"{book_id}_nb{i}",
)
)
modules.append(
LearningModule(
id=f"{book_id}_nb{i}",
name=module_name,
order=i,
pass_threshold=0.7,
knowledge_points=kps,
)
)
_validate_runnable_modules(modules, status_code=502)
await _cancel_active_learning_turn(book_id)
service = get_learning_service()
progress = service.get_or_create(book_id)
service.init_modules(progress, modules)
progress.current_module_id = modules[0].id
progress.current_kp_index = 0
service.save(progress)
return {
"status": "ok",
"module_count": len(modules),
"modules": [m.model_dump() for m in modules],
}
deeptutor/api/routers/mcp_settings.py
+93
View File
@@ -0,0 +1,93 @@
"""
MCP Settings API Router
=======================
Manage the deployment-global MCP server registry: read/update the config,
inspect live connection status, and probe a server before saving.
Mounted at ``/api/v1/settings/mcp``. Admin-gated: the registry is
deployment-global state, and a stdio server's ``command`` runs on the host
as the app user — letting non-admins edit it would be privilege escalation.
Per-user MCP access is granted through the multi-user grant whitelist
(``mcp_tools``), not by sharing this registry.
"""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field, ValidationError
from deeptutor.api.routers.auth import require_admin
from deeptutor.core.i18n import t
from deeptutor.services.mcp import (
MCPConfig,
MCPServerConfig,
get_mcp_manager,
load_mcp_config,
save_mcp_config,
validate_mcp_url,
)
from deeptutor.services.mcp.manager import probe_server
router = APIRouter(dependencies=[Depends(require_admin)])
class MCPSettingsPayload(BaseModel):
servers: dict[str, MCPServerConfig] = Field(default_factory=dict)
def _validate_servers(config: MCPConfig) -> None:
for name, cfg in config.servers.items():
transport = cfg.resolved_type()
if transport is None:
raise HTTPException(
status_code=400,
detail=t("mcp.configure_command_or_url", name=name),
)
if transport in {"sse", "streamableHttp"}:
ok, error = validate_mcp_url(cfg.url)
if not ok:
raise HTTPException(
status_code=400, detail=t("mcp.server_error", name=name, error=error)
)
@router.get("")
async def get_mcp_settings() -> dict[str, Any]:
config = load_mcp_config()
manager = get_mcp_manager()
await manager.ensure_started()
return {
"servers": {name: cfg.model_dump(mode="json") for name, cfg in config.servers.items()},
"status": manager.status(),
}
@router.put("")
async def update_mcp_settings(payload: MCPSettingsPayload) -> dict[str, Any]:
try:
config = MCPConfig(servers=payload.servers)
except (ValidationError, ValueError) as exc:
raise HTTPException(status_code=400, detail=str(exc))
_validate_servers(config)
save_mcp_config(config)
manager = get_mcp_manager()
await manager.reload()
return {"status": manager.status()}
@router.post("/test")
async def test_mcp_server(cfg: MCPServerConfig) -> dict[str, Any]:
transport = cfg.resolved_type()
if transport is None:
raise HTTPException(
status_code=400,
detail=t("mcp.configure_before_testing"),
)
if transport in {"sse", "streamableHttp"}:
ok, error = validate_mcp_url(cfg.url)
if not ok:
raise HTTPException(status_code=400, detail=error)
return await probe_server(cfg)
deeptutor/api/routers/memory.py
+785
View File
@@ -0,0 +1,785 @@
"""Memory v3 API — workbench backend.
Three layers, three modes (update / audit / dedup). Long-running work
is owned by the :mod:`runs` manager so a refresh / nav-away does not
kill it; clients re-attach by polling ``/runs/{id}/events?since=N``.
- ``GET /overview`` → all 11 docs' state + L1 backlog
- ``GET /doc/{layer}/{key}`` → raw MD
- ``GET /doc/{layer}/{key}/lines`` → line-numbered view
- ``PUT /doc/{layer}/{key}`` → user-edited save
- ``DELETE /doc/{layer}/{key}/entry/{id}`` → drop one entry
- ``POST /runs/start`` → start update/audit/dedup; returns run_id
- ``GET /runs/{id}`` → run state
- ``GET /runs/{id}/events?since=N`` → SSE-replay events from cursor N
- ``POST /runs/{id}/cancel`` → cooperative cancellation
- ``POST /runs/{id}/undo`` → restore latest run write
- ``GET /runs?layer=L2&key=chat`` → active+recent runs for one doc
- ``GET /settings`` → memory: settings subtree
- ``PUT /settings`` → save memory: settings subtree
- ``GET /trace/{surface}`` → paginated L1 events
- ``DELETE /trace/{surface}/day/{date}`` → drop one day of trace
- ``DELETE /trace/{surface}`` → drop all trace for a surface
- ``GET /backup`` → list v1-migration backup dirs (if any)
The legacy per-mode endpoints (``POST /doc/{layer}/{key}/update`` etc.)
are kept for the moment as thin wrappers that start a run and stream
its events — older clients keep working.
"""
from __future__ import annotations
from dataclasses import asdict
from datetime import date as date_cls
import json
import logging
import re
from typing import Literal
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from deeptutor.services.memory import (
L3_SLOTS,
SURFACES,
Surface,
get_memory_store,
paths,
)
_ENTRY_ID_RE = re.compile(r"^m_[0-9A-HJKMNP-TV-Z]{26}$")
logger = logging.getLogger(__name__)
router = APIRouter()
Layer = Literal["L2", "L3"]
# ── Helpers ──────────────────────────────────────────────────────────────
def _validate_doc_key(layer: Layer, key: str) -> None:
if layer == "L2" and key not in SURFACES:
raise HTTPException(status_code=404, detail=f"unknown surface {key!r}")
if layer == "L3" and key not in L3_SLOTS:
raise HTTPException(status_code=404, detail=f"unknown L3 slot {key!r}")
def _validate_layer(layer: str) -> Layer:
if layer not in {"L2", "L3"}:
raise HTTPException(status_code=400, detail="layer must be L2 or L3")
return layer # type: ignore[return-value]
def _validate_surface(surface: str) -> Surface:
if surface not in SURFACES:
raise HTTPException(status_code=404, detail=f"unknown surface {surface!r}")
return surface # type: ignore[return-value]
# ── Overview / list ──────────────────────────────────────────────────────
@router.get("/overview")
async def get_overview():
store = get_memory_store()
rows = [asdict(r) for r in store.overview()]
backup_dir = paths.backup_root()
backups: list[str] = []
if backup_dir.exists():
backups = sorted(p.name for p in backup_dir.iterdir() if p.is_dir())
return {"docs": rows, "backups": backups}
@router.get("/resolve_entry/{entry_id}")
async def resolve_entry(entry_id: str):
"""Find which L2 doc owns this entry id.
L3 docs cite L2 entries by their ``m_<ULID>`` entry id; the workbench
UI uses this resolver to turn an L3 footnote click into a navigation
to the right L2 surface + scroll-to anchor.
Scans the seven L2 mds in order; first hit wins. 404 if no L2 doc
contains the id (e.g. the entry was deleted or the id is stale).
"""
if not _ENTRY_ID_RE.match(entry_id):
raise HTTPException(status_code=400, detail="not a valid entry id")
from deeptutor.services.memory.document import parse
for surface in SURFACES:
path = paths.l2_file(surface)
if not path.exists():
continue
try:
doc = parse(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001 — malformed L2 should not 500 the resolver
continue
for entry in doc.all_entries():
if entry.id == entry_id:
return {"layer": "L2", "key": surface, "entry_id": entry_id}
raise HTTPException(status_code=404, detail="entry not found in any L2 doc")
@router.get("/backup")
async def list_backups():
backup_dir = paths.backup_root()
if not backup_dir.exists():
return {"backups": []}
out: list[dict] = []
for entry in sorted(backup_dir.iterdir()):
if entry.is_dir():
files = sorted(p.name for p in entry.iterdir())
out.append({"name": entry.name, "files": files})
return {"backups": out}
# ── Doc read / write / delete ────────────────────────────────────────────
@router.get("/doc/{layer}/{key}")
async def get_doc(layer: str, key: str):
lyr = _validate_layer(layer)
_validate_doc_key(lyr, key)
return {"layer": lyr, "key": key, "content": get_memory_store().read_raw(lyr, key)}
class DocWriteRequest(BaseModel):
content: str
@router.put("/doc/{layer}/{key}")
async def put_doc(layer: str, key: str, payload: DocWriteRequest):
lyr = _validate_layer(layer)
_validate_doc_key(lyr, key)
await get_memory_store().overwrite_doc(lyr, key, payload.content)
return {"layer": lyr, "key": key, "saved": True}
@router.delete("/doc/{layer}/{key}/entry/{entry_id}")
async def delete_entry(layer: str, key: str, entry_id: str):
lyr = _validate_layer(layer)
_validate_doc_key(lyr, key)
ok = await get_memory_store().delete_entry(lyr, key, entry_id)
if not ok:
raise HTTPException(status_code=404, detail="entry not found")
return {"layer": lyr, "key": key, "deleted": entry_id}
@router.post("/doc/{layer}/{key}/reset")
async def reset_doc(layer: str, key: str):
"""Wipe the doc + its meta sidecar so the next update starts fresh.
Destructive — the caller has confirmed. After this returns the .md
file is gone *and* the ``seen_entity_refs`` set is cleared, so a
subsequent ``run_update`` re-ingests every L1 entity instead of
treating them as already-seen.
Refuses while a consolidator run is active on this doc; the caller
can cancel first.
"""
lyr = _validate_layer(layer)
_validate_doc_key(lyr, key)
from deeptutor.services.memory.consolidator.runs import get_run_manager
if get_run_manager().active_for(lyr, key) is not None:
raise HTTPException(
status_code=409,
detail="cancel the active run before resetting this doc",
)
from deeptutor.services.memory.consolidator import meta as meta_mod
doc_path = paths.l2_file(key) if lyr == "L2" else paths.l3_file(key) # type: ignore[arg-type]
meta_path = (
meta_mod.l2_meta_path(key) # type: ignore[arg-type]
if lyr == "L2"
else meta_mod.l3_meta_path(key) # type: ignore[arg-type]
)
removed_doc = False
removed_meta = False
try:
if doc_path.exists():
doc_path.unlink()
removed_doc = True
if meta_path.exists():
meta_path.unlink()
removed_meta = True
except OSError as exc:
raise HTTPException(status_code=500, detail=f"reset failed: {exc}") from exc
return {
"layer": lyr,
"key": key,
"reset": True,
"removed_doc": removed_doc,
"removed_meta": removed_meta,
}
# ── Doc update (SSE-streamed consolidator) ───────────────────────────────
class LLMSelectionPayload(BaseModel):
profile_id: str
model_id: str
class RunStartRequest(BaseModel):
layer: str
key: str
mode: Literal["update", "audit", "dedup", "merge"]
language: str = "en"
budget: int | None = None
iterations: int | None = None
llm_selection: LLMSelectionPayload | None = None
def _runner_for(req: RunStartRequest):
"""Return an ``async on_event → None`` runner for the requested mode."""
from deeptutor.services.memory.consolidator import (
run_audit,
run_dedup,
run_merge,
run_update,
)
selection = (
{"profile_id": req.llm_selection.profile_id, "model_id": req.llm_selection.model_id}
if req.llm_selection
else None
)
if req.mode == "update":
async def go(on_event):
await run_update(
req.layer,
req.key,
language=req.language,
budget=req.budget,
llm_selection=selection,
on_event=on_event,
)
return go
if req.mode == "audit":
async def go(on_event):
await run_audit(
req.layer,
req.key,
language=req.language,
budget=req.budget,
llm_selection=selection,
on_event=on_event,
)
return go
if req.mode == "dedup":
async def go(on_event):
await run_dedup(
req.layer,
req.key,
language=req.language,
iterations=req.iterations,
llm_selection=selection,
on_event=on_event,
)
return go
if req.mode == "merge":
async def go(on_event):
await run_merge(
req.layer,
req.key,
language=req.language,
on_event=on_event,
)
return go
raise HTTPException(status_code=400, detail=f"unknown mode {req.mode!r}")
@router.post("/runs/start")
async def start_run(req: RunStartRequest):
"""Start one consolidator mode and return a run handle.
The run survives client disconnects; reconnect via
``GET /runs/{id}/events?since=N``.
"""
lyr = _validate_layer(req.layer)
_validate_doc_key(lyr, req.key)
if lyr == "L3" and req.key == "preferences" and req.mode not in ("dedup", "merge"):
raise HTTPException(
status_code=405,
detail="preferences is written by the write_memory tool, not consolidated",
)
from deeptutor.services.memory.consolidator.runs import (
RunBusyError,
get_run_manager,
)
manager = get_run_manager()
runner = _runner_for(req)
selection = (
{"profile_id": req.llm_selection.profile_id, "model_id": req.llm_selection.model_id}
if req.llm_selection
else None
)
try:
run = await manager.start(
layer=lyr,
key=req.key,
mode=req.mode,
runner=runner,
params={
"budget": req.budget,
"iterations": req.iterations,
"language": req.language,
"llm_selection": selection,
},
language=req.language,
)
except RunBusyError as exc:
raise HTTPException(status_code=409, detail=str(exc))
return run.to_dict()
@router.get("/runs/{run_id}")
async def get_run(run_id: str):
from deeptutor.services.memory.consolidator.runs import get_run_manager
run = get_run_manager().get(run_id)
if run is None:
raise HTTPException(status_code=404, detail="unknown run_id")
return run.to_dict()
@router.post("/runs/{run_id}/cancel")
async def cancel_run(run_id: str):
from deeptutor.services.memory.consolidator.runs import get_run_manager
ok = await get_run_manager().cancel(run_id)
if not ok:
raise HTTPException(status_code=409, detail="not active")
return {"run_id": run_id, "cancelled": True}
@router.post("/runs/{run_id}/undo")
async def undo_run_edit(run_id: str):
from deeptutor.services.memory.consolidator.runs import (
RunBusyError,
get_run_manager,
)
manager = get_run_manager()
try:
event = await manager.undo_last(run_id)
except KeyError:
raise HTTPException(status_code=404, detail="unknown run_id")
except RunBusyError as exc:
raise HTTPException(status_code=409, detail=str(exc))
if event is None:
raise HTTPException(status_code=409, detail="nothing to undo")
run = manager.get(run_id)
return {
"run_id": run_id,
"undone": True,
"undo_count": len(run.undo_stack) if run else 0,
"event": {"seq": event.seq, "ts": event.ts, **event.payload},
}
@router.get("/runs")
async def list_runs(layer: str | None = None, key: str | None = None):
from deeptutor.services.memory.consolidator.runs import get_run_manager
lyr = _validate_layer(layer) if layer is not None else None
if lyr and key is not None:
_validate_doc_key(lyr, key)
runs = get_run_manager().list_for(layer=lyr, key=key)
return {"runs": [r.to_dict() for r in runs]}
@router.get("/runs/{run_id}/events")
async def stream_run_events(run_id: str, since: int = 0):
"""SSE-replay events from ``since`` (exclusive) until the run ends.
Reconnecting after a refresh: pass the largest ``seq`` previously
observed. The manager replays the buffered tail, then blocks on
new events until the run reaches a terminal state.
"""
from deeptutor.services.memory.consolidator.runs import get_run_manager
manager = get_run_manager()
run = manager.get(run_id)
if run is None:
raise HTTPException(status_code=404, detail="unknown run_id")
async def producer():
cursor = max(0, since)
# Initial backfill (if any) is delivered as a batch up front.
while True:
events = await manager.wait_for_events(run, since=cursor)
for ev in events:
yield (
"data: "
+ json.dumps(
{"seq": ev.seq, "ts": ev.ts, **ev.payload},
ensure_ascii=False,
)
+ "\n\n"
)
cursor = max(cursor, run.events[-1].seq + 1 if run.events else cursor)
if not run.active:
# Drain any final events that arrived between wait return and now.
final = run.events[cursor:]
for ev in final:
yield (
"data: "
+ json.dumps(
{"seq": ev.seq, "ts": ev.ts, **ev.payload},
ensure_ascii=False,
)
+ "\n\n"
)
break
return StreamingResponse(producer(), media_type="text/event-stream")
# ── Legacy per-mode endpoints (kept as thin wrappers over /runs/start) ──
def _legacy_run_stream(req: RunStartRequest) -> StreamingResponse:
"""Old contract: POST /doc/{layer}/{key}/<mode> streams events inline."""
from deeptutor.services.memory.consolidator.runs import (
RunBusyError,
get_run_manager,
)
async def producer():
manager = get_run_manager()
runner = _runner_for(req)
selection = (
{"profile_id": req.llm_selection.profile_id, "model_id": req.llm_selection.model_id}
if req.llm_selection
else None
)
try:
run = await manager.start(
layer=req.layer,
key=req.key,
mode=req.mode,
runner=runner,
params={
"budget": req.budget,
"iterations": req.iterations,
"language": req.language,
"llm_selection": selection,
},
language=req.language,
)
except RunBusyError as exc:
yield (
"data: "
+ json.dumps({"stage": "error", "message": str(exc)}, ensure_ascii=False)
+ "\n\n"
)
return
cursor = 0
while True:
events = await manager.wait_for_events(run, since=cursor)
for ev in events:
yield (
"data: "
+ json.dumps({**ev.payload, "seq": ev.seq}, ensure_ascii=False)
+ "\n\n"
)
cursor = ev.seq + 1
if not run.active:
break
return StreamingResponse(producer(), media_type="text/event-stream")
class UpdateRequest(BaseModel):
language: str = "en"
budget: int | None = None
llm_selection: LLMSelectionPayload | None = None
class AuditRequest(BaseModel):
language: str = "en"
budget: int | None = None
llm_selection: LLMSelectionPayload | None = None
class DedupRequest(BaseModel):
language: str = "en"
iterations: int | None = None
llm_selection: LLMSelectionPayload | None = None
@router.post("/doc/{layer}/{key}/update")
async def update_doc(layer: str, key: str, payload: UpdateRequest | None = None):
lyr = _validate_layer(layer)
_validate_doc_key(lyr, key)
req = RunStartRequest(
layer=lyr,
key=key,
mode="update",
language=(payload.language if payload else "en") or "en",
budget=payload.budget if payload else None,
llm_selection=payload.llm_selection if payload else None,
)
return _legacy_run_stream(req)
@router.post("/doc/{layer}/{key}/audit")
async def audit_doc(layer: str, key: str, payload: AuditRequest | None = None):
lyr = _validate_layer(layer)
_validate_doc_key(lyr, key)
req = RunStartRequest(
layer=lyr,
key=key,
mode="audit",
language=(payload.language if payload else "en") or "en",
budget=payload.budget if payload else None,
llm_selection=payload.llm_selection if payload else None,
)
return _legacy_run_stream(req)
@router.post("/doc/{layer}/{key}/dedup")
async def dedup_doc(layer: str, key: str, payload: DedupRequest | None = None):
lyr = _validate_layer(layer)
_validate_doc_key(lyr, key)
req = RunStartRequest(
layer=lyr,
key=key,
mode="dedup",
language=(payload.language if payload else "en") or "en",
iterations=payload.iterations if payload else None,
llm_selection=payload.llm_selection if payload else None,
)
return _legacy_run_stream(req)
@router.get("/doc/{layer}/{key}/lines")
async def get_doc_lines(layer: str, key: str):
"""Return the line-numbered, footnote-stripped view of a doc.
Used by the workbench's "show line numbers" toggle so the same line
indices the audit/dedup LLMs see are visible to the user. Footnote
block is omitted because edit ops never reference it directly.
"""
lyr = _validate_layer(layer)
_validate_doc_key(lyr, key)
from deeptutor.services.memory import paths
from deeptutor.services.memory.consolidator.line_doc import render_view
from deeptutor.services.memory.document import Document, parse
path = paths.l2_file(key) if lyr == "L2" else paths.l3_file(key) # type: ignore[arg-type]
doc = (
parse(path.read_text(encoding="utf-8"))
if path.exists()
else Document(title=_default_title(lyr, key))
)
view = render_view(doc)
return {
"layer": lyr,
"key": key,
"lines": [
{
"number": line.number,
"kind": line.kind,
"text": line.text,
"entry_id": line.entry_id,
"section": line.section,
}
for line in view.lines
],
}
# ── Settings ────────────────────────────────────────────────────────────
@router.get("/settings")
async def get_memory_settings_endpoint():
"""Return the current ``memory:`` subtree (defaults merged in)."""
from deeptutor.services.memory.settings import memory_settings_dict
return memory_settings_dict()
@router.put("/settings")
async def put_memory_settings(payload: dict):
"""Merge the payload into the ``memory:`` subtree and persist."""
from deeptutor.services.memory.settings import (
memory_settings_dict,
save_memory_settings,
)
save_memory_settings(payload)
return memory_settings_dict()
def _default_title(layer: str, key: str) -> str:
if layer == "L2":
return f"{key} memory"
return {
"recent": "Recent summary",
"profile": "User profile",
"scope": "Knowledge scope",
"preferences": "Preferences",
}.get(key, f"{key} memory")
class ApplyOpsRequest(BaseModel):
ops: list[dict]
@router.post("/doc/{layer}/{key}/apply")
async def apply_doc_ops(layer: str, key: str, payload: ApplyOpsRequest):
"""Commit a list of previously-previewed ops to a doc atomically."""
lyr = _validate_layer(layer)
_validate_doc_key(lyr, key)
if lyr == "L3" and key == "preferences":
raise HTTPException(
status_code=405,
detail="preferences is written by the write_memory tool, not consolidated",
)
if not payload.ops:
return {"accepted": True, "reason": "no ops to apply", "results": []}
report = await get_memory_store().apply_ops_payload(lyr, key, payload.ops)
return {
"accepted": report.accepted,
"reason": report.reason,
"results": [
{
"status": r.status,
"entry_id": r.entry_id,
"detail": r.detail,
}
for r in report.results
],
}
# ── Trace browser ────────────────────────────────────────────────────────
@router.get("/trace/{surface}")
async def get_trace(surface: str, limit: int = 200, offset: int = 0):
surf = _validate_surface(surface)
from deeptutor.services.memory.trace import iter_since
events = []
for i, event in enumerate(iter_since(surf)):
if i < offset:
continue
if len(events) >= max(1, min(limit, 1000)):
break
events.append(asdict(event))
return {"surface": surf, "events": events, "offset": offset, "limit": limit}
@router.delete("/trace/{surface}")
async def clear_trace(surface: str):
surf = _validate_surface(surface)
removed = 0
for path in paths.trace_dir(surf).glob("*.jsonl"):
try:
path.unlink()
removed += 1
except OSError:
continue
return {"surface": surf, "removed_files": removed}
@router.delete("/trace/{surface}/day/{day}")
async def clear_trace_day(surface: str, day: str):
surf = _validate_surface(surface)
try:
parsed = date_cls.fromisoformat(day)
except ValueError:
raise HTTPException(status_code=400, detail="day must be YYYY-MM-DD")
path = paths.trace_file(surf, parsed)
if not path.exists():
raise HTTPException(status_code=404, detail="no trace for that day")
try:
path.unlink()
except OSError as exc:
raise HTTPException(status_code=500, detail=str(exc))
return {"surface": surf, "day": day, "deleted": True}
# ── Snapshot (L1 workspace mirror) ───────────────────────────────────────
@router.get("/snapshot/{surface}")
async def get_snapshot(surface: str):
"""Return the current entity list for ``surface`` from workspace.
Snapshot is always derived live from workspace at call time. The response
also includes ``pending_changes`` — the diff vs the last persisted state.
Refresh commits these pending changes into ``changes.jsonl``.
"""
surf = _validate_surface(surface)
from deeptutor.services.memory import snapshot as snap
entities = snap.read_snapshot(surf)
pending = snap.pending_changes(surf, entities)
state = snap.current_state(surf)
return {
"surface": surf,
"entities": [e.to_dict() for e in entities],
"last_refresh": state.get("last_refresh"),
"pending_changes": [c.to_dict() for c in pending],
}
@router.post("/snapshot/{surface}/refresh")
async def refresh_snapshot(surface: str):
"""Reconcile persisted state with current workspace; record diffs."""
surf = _validate_surface(surface)
from deeptutor.services.memory import snapshot as snap
changes = snap.refresh_snapshot(surf)
state = snap.current_state(surf)
return {
"surface": surf,
"changes": [c.to_dict() for c in changes],
"last_refresh": state.get("last_refresh"),
}
@router.get("/snapshot/{surface}/changes")
async def get_changes(surface: str, limit: int = 200, offset: int = 0):
surf = _validate_surface(surface)
from deeptutor.services.memory import snapshot as snap
entries = snap.read_changes(surf, limit=limit, offset=offset)
return {
"surface": surf,
"changes": [c.to_dict() for c in entries],
"limit": limit,
"offset": offset,
}
@router.delete("/snapshot/{surface}/changes")
async def clear_snapshot_changes(surface: str):
surf = _validate_surface(surface)
from deeptutor.services.memory import snapshot as snap
snap.clear_changes(surf)
return {"surface": surf, "cleared": True}
deeptutor/api/routers/notebook.py
+358
View File
@@ -0,0 +1,358 @@
"""
Notebook API Router
Provides notebook creation, querying, updating, deletion, and record management functions
"""
import json
from typing import AsyncGenerator, Literal
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from deeptutor.agents.notebook import NotebookSummarizeAgent
from deeptutor.services.llm import clean_thinking_tags
from deeptutor.services.notebook import notebook_manager
router = APIRouter()
# === Request/Response Models ===
class CreateNotebookRequest(BaseModel):
"""Create notebook request"""
name: str
description: str = ""
color: str = "#3B82F6"
icon: str = "book"
class UpdateNotebookRequest(BaseModel):
"""Update notebook request"""
name: str | None = None
description: str | None = None
color: str | None = None
icon: str | None = None
class AddRecordRequest(BaseModel):
"""Add record request"""
notebook_ids: list[str]
record_type: Literal["solve", "question", "research", "chat", "co_writer", "tutorbot"]
title: str
summary: str = ""
user_query: str
output: str
metadata: dict = {}
kb_name: str | None = None
class RemoveRecordRequest(BaseModel):
"""Remove record request"""
record_id: str
class UpdateRecordRequest(BaseModel):
"""Update an existing notebook record."""
title: str | None = None
summary: str | None = None
user_query: str | None = None
output: str | None = None
metadata: dict | None = None
kb_name: str | None = None
# === API Endpoints ===
async def _build_record_summary(request: AddRecordRequest) -> str:
if request.summary.strip():
return clean_thinking_tags(request.summary).strip()
agent = NotebookSummarizeAgent(language=str(request.metadata.get("ui_language", "en")))
return clean_thinking_tags(
await agent.summarize(
title=request.title,
record_type=request.record_type,
user_query=request.user_query,
output=request.output,
metadata=request.metadata,
)
).strip()
async def _stream_add_record_with_summary(
request: AddRecordRequest,
) -> AsyncGenerator[str, None]:
try:
agent = NotebookSummarizeAgent(language=str(request.metadata.get("ui_language", "en")))
summary_parts: list[str] = []
if request.summary.strip():
summary = clean_thinking_tags(request.summary).strip()
summary_parts.append(summary)
if summary:
yield f"data: {json.dumps({'type': 'summary_chunk', 'content': summary}, ensure_ascii=False)}\n\n"
else:
async for chunk in agent.stream_summary(
title=request.title,
record_type=request.record_type,
user_query=request.user_query,
output=request.output,
metadata=request.metadata,
):
if not chunk:
continue
summary_parts.append(chunk)
summary = clean_thinking_tags("".join(summary_parts)).strip()
if summary:
yield f"data: {json.dumps({'type': 'summary_chunk', 'content': summary}, ensure_ascii=False)}\n\n"
summary = clean_thinking_tags("".join(summary_parts)).strip()
result = notebook_manager.add_record(
notebook_ids=request.notebook_ids,
record_type=request.record_type,
title=request.title,
summary=summary,
user_query=request.user_query,
output=request.output,
metadata=request.metadata,
kb_name=request.kb_name,
)
payload = {
"type": "result",
"success": True,
"summary": summary,
"record": result["record"],
"added_to_notebooks": result["added_to_notebooks"],
}
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
except Exception as exc:
payload = {"type": "error", "detail": str(exc)}
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
@router.get("/list")
async def list_notebooks():
"""
Get all notebook list
Returns:
Notebook list (includes summary information)
"""
try:
notebooks = notebook_manager.list_notebooks()
return {"notebooks": notebooks, "total": len(notebooks)}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/statistics")
async def get_statistics():
"""
Get notebook statistics
Returns:
Statistics information
"""
try:
stats = notebook_manager.get_statistics()
return stats
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.post("/create")
async def create_notebook(request: CreateNotebookRequest):
"""
Create new notebook
Args:
request: Create request
Returns:
Created notebook information
"""
try:
notebook = notebook_manager.create_notebook(
name=request.name,
description=request.description,
color=request.color,
icon=request.icon,
)
return {"success": True, "notebook": notebook}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/{notebook_id}")
async def get_notebook(notebook_id: str):
"""
Get notebook details
Args:
notebook_id: Notebook ID
Returns:
Notebook details (includes all records)
"""
try:
notebook = notebook_manager.get_notebook(notebook_id)
if not notebook:
raise HTTPException(status_code=404, detail="Notebook not found")
return notebook
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.put("/{notebook_id}")
async def update_notebook(notebook_id: str, request: UpdateNotebookRequest):
"""
Update notebook information
Args:
notebook_id: Notebook ID
request: Update request
Returns:
Updated notebook information
"""
try:
notebook = notebook_manager.update_notebook(
notebook_id=notebook_id,
name=request.name,
description=request.description,
color=request.color,
icon=request.icon,
)
if not notebook:
raise HTTPException(status_code=404, detail="Notebook not found")
return {"success": True, "notebook": notebook}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.delete("/{notebook_id}")
async def delete_notebook(notebook_id: str):
"""
Delete notebook
Args:
notebook_id: Notebook ID
Returns:
Deletion result
"""
try:
success = notebook_manager.delete_notebook(notebook_id)
if not success:
raise HTTPException(status_code=404, detail="Notebook not found")
return {"success": True, "message": "Notebook deleted successfully"}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.post("/add_record")
async def add_record(request: AddRecordRequest):
"""
Add record to notebook
Args:
request: Add record request
Returns:
Addition result
"""
try:
summary = await _build_record_summary(request)
result = notebook_manager.add_record(
notebook_ids=request.notebook_ids,
record_type=request.record_type,
title=request.title,
summary=summary,
user_query=request.user_query,
output=request.output,
metadata=request.metadata,
kb_name=request.kb_name,
)
return {
"success": True,
"summary": summary,
"record": result["record"],
"added_to_notebooks": result["added_to_notebooks"],
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.post("/add_record_with_summary")
async def add_record_with_summary(request: AddRecordRequest):
"""Add record to notebook and stream generated summary."""
return StreamingResponse(
_stream_add_record_with_summary(request),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
@router.delete("/{notebook_id}/records/{record_id}")
async def remove_record(notebook_id: str, record_id: str):
"""
Remove record from notebook
Args:
notebook_id: Notebook ID
record_id: Record ID
Returns:
Deletion result
"""
try:
success = notebook_manager.remove_record(notebook_id, record_id)
if not success:
raise HTTPException(status_code=404, detail="Record not found")
return {"success": True, "message": "Record removed successfully"}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.put("/{notebook_id}/records/{record_id}")
async def update_record(notebook_id: str, record_id: str, request: UpdateRecordRequest):
"""Update an existing notebook record in place."""
try:
updated = notebook_manager.update_record(
notebook_id=notebook_id,
record_id=record_id,
title=request.title,
summary=request.summary,
user_query=request.user_query,
output=request.output,
metadata=request.metadata,
kb_name=request.kb_name,
)
if not updated:
raise HTTPException(status_code=404, detail="Record not found")
return {"success": True, "record": updated}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/health")
async def health_check():
"""Health check"""
return {"status": "healthy", "service": "notebook"}
deeptutor/api/routers/partners.py
+1162
View File
File diff suppressed because it is too large Load Diff
deeptutor/api/routers/personas.py
+138
View File
@@ -0,0 +1,138 @@
"""
Personas API Router
===================
CRUD endpoints for user-authored PERSONA.md files stored under
``data/user/workspace/personas/<name>/PERSONA.md``.
Personas are behaviour/voice presets, not capability skills: admin-authored
personas are visible to every user as read-only deployment presets (no grant
mechanism — a persona carries no privileged workflow, only style guidance).
Users create and manage their own personas in their own workspace; a user
persona shadows an admin persona of the same name.
Mounted at ``/api/v1/personas``.
"""
from __future__ import annotations
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, Field
from deeptutor.core.i18n import t
from deeptutor.multi_user.context import get_current_user
from deeptutor.multi_user.paths import get_admin_path_service
from deeptutor.services.persona import (
InvalidPersonaNameError,
PersonaExistsError,
PersonaNotFoundError,
PersonaService,
get_persona_service,
)
router = APIRouter()
class CreatePersonaRequest(BaseModel):
name: str = Field(..., min_length=1, max_length=64)
description: str = ""
content: str = ""
class UpdatePersonaRequest(BaseModel):
description: str | None = None
content: str | None = None
rename_to: str | None = None
def _admin_persona_service() -> PersonaService:
return PersonaService(root=get_admin_path_service().get_workspace_dir() / "personas")
@router.get("/list")
async def list_personas() -> dict[str, list[dict[str, object]]]:
service = get_persona_service()
own = [info.to_dict() for info in service.list_personas()]
user = get_current_user()
if user.is_admin:
return {"personas": own}
own_names = {item["name"] for item in own}
merged = list(own)
for preset in _admin_persona_service().list_personas():
if preset.name in own_names:
continue
entry = preset.to_dict()
entry.update({"source": "admin", "read_only": True})
merged.append(entry)
return {"personas": merged}
@router.get("/{name}")
async def get_persona(name: str) -> dict[str, object]:
service = get_persona_service()
try:
return service.get_detail(name).to_dict()
except PersonaNotFoundError:
pass
except InvalidPersonaNameError as exc:
raise HTTPException(status_code=400, detail=str(exc))
user = get_current_user()
if not user.is_admin:
try:
detail = _admin_persona_service().get_detail(name).to_dict()
detail.update({"source": "admin", "read_only": True})
return detail
except (PersonaNotFoundError, InvalidPersonaNameError):
pass
raise HTTPException(status_code=404, detail=t("api.persona_not_found", name=name))
@router.post("/create")
async def create_persona(payload: CreatePersonaRequest) -> dict[str, object]:
service = get_persona_service()
try:
info = service.create(
name=payload.name,
description=payload.description,
content=payload.content,
)
return info.to_dict()
except PersonaExistsError:
raise HTTPException(
status_code=409,
detail=t("api.persona_already_exists", name=payload.name),
)
except InvalidPersonaNameError as exc:
raise HTTPException(status_code=400, detail=str(exc))
@router.put("/{name}")
async def update_persona(name: str, payload: UpdatePersonaRequest) -> dict[str, object]:
service = get_persona_service()
try:
info = service.update(
name,
description=payload.description,
content=payload.content,
rename_to=payload.rename_to,
)
return info.to_dict()
except PersonaNotFoundError:
raise HTTPException(status_code=404, detail=t("api.persona_not_found", name=name))
except PersonaExistsError as exc:
raise HTTPException(status_code=409, detail=str(exc))
except InvalidPersonaNameError as exc:
raise HTTPException(status_code=400, detail=str(exc))
@router.delete("/{name}")
async def delete_persona(name: str) -> dict[str, str]:
service = get_persona_service()
try:
service.delete(name)
return {"status": "deleted", "name": name}
except PersonaNotFoundError:
raise HTTPException(status_code=404, detail=t("api.persona_not_found", name=name))
except InvalidPersonaNameError as exc:
raise HTTPException(status_code=400, detail=str(exc))
deeptutor/api/routers/plugins_api.py
+432
View File
@@ -0,0 +1,432 @@
"""
Plugins API Router
==================
Lists registered tools, capabilities, and playground plugins.
Provides direct tool execution for the Playground tester.
"""
import asyncio
import contextlib
import json
import logging
import re
import time
from typing import Any, AsyncGenerator
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, ConfigDict, Field
from deeptutor.core.i18n import t
from deeptutor.i18n.metadata_i18n import tool_description_i18n
from deeptutor.logging import (
ProcessLogEvent,
bind_log_context,
capture_process_logs,
current_log_context,
)
from deeptutor.runtime.registry.capability_registry import get_capability_registry
from deeptutor.runtime.registry.tool_registry import get_tool_registry
logger = logging.getLogger(__name__)
router = APIRouter()
ANSI_ESCAPE_RE = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
def _discover_plugins() -> list[Any]:
try:
from deeptutor.plugins.loader import discover_plugins
except Exception:
logger.debug("Plugin loader unavailable; returning no plugins.", exc_info=True)
return []
return discover_plugins()
class ToolExecuteRequest(BaseModel):
params: dict[str, Any] = Field(default_factory=dict)
class CapabilityExecuteRequest(BaseModel):
model_config = ConfigDict(populate_by_name=True)
content: str
tools: list[str] = Field(default_factory=list, alias="enabledTools")
knowledge_bases: list[str] = Field(default_factory=list, alias="knowledgeBases")
language: str = "en"
config: dict[str, Any] = Field(default_factory=dict)
attachments: list[dict[str, Any]] = Field(default_factory=list)
# ``bot_id`` is the legacy TutorBot field name; it now addresses a partner.
partner_id: str | None = Field(default=None, alias="bot_id")
session_id: str | None = None
chat_id: str | None = None
llm_selection: dict[str, str] | None = Field(default=None, alias="llmSelection")
@router.get("/list")
async def list_plugins():
tool_registry = get_tool_registry()
capability_registry = get_capability_registry()
plugin_manifests = _discover_plugins()
tools = [
{
"name": definition.name,
"description": definition.description,
"description_i18n": tool_description_i18n(definition.name, definition.description),
"parameters": [
{
"name": parameter.name,
"type": parameter.type,
"description": parameter.description,
"required": parameter.required,
"default": parameter.default,
"enum": parameter.enum,
}
for parameter in definition.parameters
],
}
for definition in tool_registry.get_definitions()
]
capabilities = capability_registry.get_manifests()
plugins = [
{
"name": plugin.name,
"type": plugin.type,
"description": plugin.description,
"stages": plugin.stages,
"version": plugin.version,
"author": plugin.author,
}
for plugin in plugin_manifests
]
return {
"tools": tools,
"capabilities": capabilities,
"plugins": plugins,
}
@router.post("/tools/{tool_name}/execute")
async def execute_tool(tool_name: str, body: ToolExecuteRequest):
"""Execute a single tool with explicit parameters (for Playground testing)."""
registry = get_tool_registry()
tool = registry.get(tool_name)
if not tool:
raise HTTPException(status_code=404, detail=t("api.tool_not_found", name=tool_name))
try:
result = await tool.execute(**body.params)
return {
"success": result.success,
"content": result.content,
"sources": result.sources,
"metadata": result.metadata,
}
except Exception as exc:
logger.exception("Tool execution failed: %s", tool_name)
raise HTTPException(status_code=500, detail=str(exc))
def _sse(event: str, payload: dict[str, Any]) -> str:
return f"event: {event}\ndata: {json.dumps(payload, ensure_ascii=False, default=str)}\n\n"
def _queue_process_emit(
queue: asyncio.Queue[dict[str, Any]],
loop: asyncio.AbstractEventLoop,
):
def emit(event: ProcessLogEvent) -> None:
loop.call_soon_threadsafe(
queue.put_nowait,
{"kind": "process_log", "payload": event.to_dict()},
)
return emit
class _QueueTextStream:
"""Capture stdout/stderr lines as process-log events."""
def __init__(
self,
queue: asyncio.Queue[dict[str, Any]],
loop: asyncio.AbstractEventLoop,
stream,
*,
logger_name: str,
):
self._queue = queue
self._loop = loop
self._stream = stream
self._logger_name = logger_name
self._buffer = ""
def write(self, text: str) -> int:
if self._stream is not None:
self._stream.write(text)
self._stream.flush()
self._buffer += text
while "\n" in self._buffer:
line, self._buffer = self._buffer.split("\n", 1)
self._emit_line(line.rstrip("\r"))
return len(text)
def flush(self):
if self._stream is not None:
self._stream.flush()
if self._buffer.strip():
self._emit_line(self._buffer)
self._buffer = ""
def isatty(self) -> bool:
return False
def _emit_line(self, line: str) -> None:
clean = ANSI_ESCAPE_RE.sub("", line).strip()
if not clean:
return
event = ProcessLogEvent(
level="INFO",
message=clean,
logger=self._logger_name,
timestamp=time.time(),
context=current_log_context(),
)
self._loop.call_soon_threadsafe(
self._queue.put_nowait,
{"kind": "process_log", "payload": event.to_dict()},
)
async def _execute_stream(tool_name: str, params: dict[str, Any]) -> AsyncGenerator[str, None]:
"""Run a tool while streaming structured process logs and the final result."""
registry = get_tool_registry()
tool = registry.get(tool_name)
if not tool:
yield _sse("error", {"detail": t("api.tool_not_found", name=tool_name)})
return
event_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
loop = asyncio.get_running_loop()
stdout_stream = _QueueTextStream(
event_queue, loop, stream=None, logger_name="deeptutor.playground.stdout"
)
stderr_stream = _QueueTextStream(
event_queue, loop, stream=None, logger_name="deeptutor.playground.stderr"
)
result_holder: dict[str, Any] = {}
error_holder: dict[str, str] = {}
done = asyncio.Event()
task_id = f"playground_tool_{tool_name}_{int(time.time() * 1000)}"
async def _run():
try:
import sys
stdout_stream._stream = sys.stdout
stderr_stream._stream = sys.stderr
with bind_log_context(task_id=task_id, capability="playground", sink="ui"):
with capture_process_logs(_queue_process_emit(event_queue, loop), task_id=task_id):
with (
contextlib.redirect_stdout(stdout_stream),
contextlib.redirect_stderr(stderr_stream),
):
result = await tool.execute(**params)
result_holder["data"] = {
"success": result.success,
"content": result.content,
"sources": result.sources,
"metadata": result.metadata,
}
except Exception as exc:
error_holder["detail"] = str(exc)
finally:
stdout_stream.flush()
stderr_stream.flush()
done.set()
task = asyncio.create_task(_run())
t0 = time.monotonic()
try:
while not done.is_set():
try:
item = await asyncio.wait_for(event_queue.get(), timeout=0.15)
yield _sse(item["kind"], item["payload"])
except asyncio.TimeoutError:
pass
while not event_queue.empty():
item = event_queue.get_nowait()
yield _sse(item["kind"], item["payload"])
elapsed_ms = round((time.monotonic() - t0) * 1000)
if error_holder:
yield _sse("error", {"detail": error_holder["detail"], "elapsed_ms": elapsed_ms})
else:
payload = {**result_holder.get("data", {}), "elapsed_ms": elapsed_ms}
yield _sse("result", payload)
finally:
if not task.done():
task.cancel()
@router.post("/tools/{tool_name}/execute-stream")
async def execute_tool_stream(tool_name: str, body: ToolExecuteRequest):
"""Execute a tool and stream process logs + result as SSE."""
return StreamingResponse(
_execute_stream(tool_name, body.params),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
async def _execute_capability_stream(
capability_name: str,
body: CapabilityExecuteRequest,
) -> AsyncGenerator[str, None]:
"""Run a capability while streaming process logs, trace events, and the result."""
partner_id = (body.partner_id or "").strip()
if capability_name == "chat" and partner_id:
from deeptutor.api.routers.partners import (
ChatMessageRequest,
_ensure_running_partner,
_partner_chat_stream,
)
if not body.content.strip():
yield _sse("error", {"detail": "content is required"})
return
try:
await _ensure_running_partner(partner_id)
except HTTPException as exc:
yield _sse("error", {"detail": exc.detail, "status_code": exc.status_code})
return
request = ChatMessageRequest(
content=body.content,
session_id=body.session_id,
chat_id=body.chat_id,
llm_selection=body.llm_selection,
)
async for chunk in _partner_chat_stream(partner_id, request):
yield chunk
return
from deeptutor.core.context import Attachment, UnifiedContext
from deeptutor.runtime.orchestrator import ChatOrchestrator
orch = ChatOrchestrator()
if capability_name not in orch.list_capabilities():
yield _sse("error", {"detail": f"Capability {capability_name!r} not found"})
return
attachments = [
Attachment(
type=a.get("type", "file"),
url=a.get("url", ""),
base64=a.get("base64", ""),
filename=a.get("filename", ""),
mime_type=a.get("mime_type", ""),
)
for a in body.attachments
]
ctx = UnifiedContext(
user_message=body.content,
enabled_tools=body.tools,
active_capability=capability_name,
knowledge_bases=body.knowledge_bases,
attachments=attachments,
config_overrides=body.config,
language=body.language,
)
event_queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
loop = asyncio.get_running_loop()
stdout_stream = _QueueTextStream(
event_queue, loop, stream=None, logger_name="deeptutor.playground.stdout"
)
stderr_stream = _QueueTextStream(
event_queue, loop, stream=None, logger_name="deeptutor.playground.stderr"
)
final_result: dict[str, Any] | None = None
error_holder: dict[str, str] = {}
done = asyncio.Event()
task_id = f"playground_capability_{capability_name}_{int(time.time() * 1000)}"
async def _run():
nonlocal final_result
try:
import sys
stdout_stream._stream = sys.stdout
stderr_stream._stream = sys.stderr
with bind_log_context(
task_id=task_id,
capability=capability_name,
sink="ui",
):
with capture_process_logs(_queue_process_emit(event_queue, loop), task_id=task_id):
with (
contextlib.redirect_stdout(stdout_stream),
contextlib.redirect_stderr(stderr_stream),
):
async for event in orch.handle(ctx):
if event.type.value == "result":
final_result = dict(event.metadata)
continue
await event_queue.put({"kind": "stream", "payload": event.to_dict()})
except Exception as exc:
error_holder["detail"] = str(exc)
finally:
stdout_stream.flush()
stderr_stream.flush()
done.set()
task = asyncio.create_task(_run())
t0 = time.monotonic()
try:
while not done.is_set():
try:
item = await asyncio.wait_for(event_queue.get(), timeout=0.15)
yield _sse(item["kind"], item["payload"])
except asyncio.TimeoutError:
pass
while not event_queue.empty():
item = event_queue.get_nowait()
yield _sse(item["kind"], item["payload"])
elapsed_ms = round((time.monotonic() - t0) * 1000)
if error_holder:
yield _sse("error", {"detail": error_holder["detail"], "elapsed_ms": elapsed_ms})
else:
yield _sse(
"result",
{"success": True, "data": final_result or {}, "elapsed_ms": elapsed_ms},
)
finally:
if not task.done():
task.cancel()
@router.post("/capabilities/{capability_name}/execute-stream")
async def execute_capability_stream(
capability_name: str,
body: CapabilityExecuteRequest,
):
"""Execute a capability and stream logs + trace + final result as SSE."""
return StreamingResponse(
_execute_capability_stream(capability_name, body),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
deeptutor/api/routers/question.py
+570
View File
@@ -0,0 +1,570 @@
import asyncio
import base64
from datetime import datetime
import logging
from pathlib import Path
import re
import sys
import traceback
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from deeptutor.agents.question import AgentCoordinator
from deeptutor.api.utils.task_id_manager import TaskIDManager
from deeptutor.logging import (
ProcessLogEvent,
bind_log_context,
capture_process_logs,
current_log_context,
)
from deeptutor.services.config import PROJECT_ROOT, load_config_with_main
from deeptutor.services.llm.config import get_llm_config
from deeptutor.services.path_service import get_path_service
from deeptutor.services.settings.interface_settings import get_ui_language
from deeptutor.tools.question import mimic_exam_questions
from deeptutor.utils.document_validator import DocumentValidator
from deeptutor.utils.error_utils import format_exception_message
# Setup module logger with unified logging system (from config)
config = load_config_with_main("main.yaml", PROJECT_ROOT)
log_dir = config.get("paths", {}).get("user_log_dir") or config.get("logging", {}).get("log_dir")
logger = logging.getLogger(__name__)
router = APIRouter()
def _mimic_output_dir():
# Resolved per-call so a per-user PathService (set after auth) routes
# generated mimic papers under the caller's own workspace instead of
# admin's directory frozen at import time.
return get_path_service().get_question_dir() / "mimic_papers"
@router.websocket("/mimic")
async def websocket_mimic_generate(websocket: WebSocket):
"""
WebSocket endpoint for mimic exam paper question generation.
Supports two modes:
1. Upload PDF directly via WebSocket (base64 encoded)
2. Use a pre-parsed paper directory path
Message format for PDF upload:
{
"mode": "upload",
"pdf_data": "base64_encoded_pdf_content",
"pdf_name": "exam.pdf",
"kb_name": "knowledge_base_name",
"max_questions": 5 // optional
}
Message format for pre-parsed:
{
"mode": "parsed",
"paper_path": "directory_name",
"kb_name": "knowledge_base_name",
"max_questions": 5 // optional
}
"""
from deeptutor.api.routers.auth import ws_auth_failed, ws_require_auth
from deeptutor.multi_user.context import reset_current_user
user_token = await ws_require_auth(websocket)
if user_token is ws_auth_failed:
return
await websocket.accept()
pusher_task = None
original_stdout = sys.stdout
try:
# 1. Wait for config
data = await websocket.receive_json()
mode = data.get("mode", "parsed") # "upload" or "parsed"
kb_name = data.get("kb_name", "ai_textbook")
max_questions = data.get("max_questions")
logger.info(f"Starting mimic generation (mode: {mode}, kb: {kb_name})")
# 2. Setup Log Queue
log_queue = asyncio.Queue()
loop = asyncio.get_running_loop()
task_id = f"question_mimic_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
def emit_process_log(event: ProcessLogEvent) -> None:
loop.call_soon_threadsafe(log_queue.put_nowait, event.to_dict())
async def log_pusher():
while True:
entry = await log_queue.get()
try:
await websocket.send_json(entry)
except Exception:
break
log_queue.task_done()
pusher_task = asyncio.create_task(log_pusher())
# 3. Stdout interceptor for capturing prints
# ANSI escape sequence pattern for stripping color codes
ANSI_ESCAPE_PATTERN = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]")
class StdoutInterceptor:
def __init__(self, queue, original):
self.queue = queue
self.original_stdout = original
self._closed = False
def write(self, message):
if self._closed:
return
# Write to terminal first (with ANSI codes for color)
try:
self.original_stdout.write(message)
except Exception:
pass
# Strip ANSI escape codes before sending to frontend
clean_message = ANSI_ESCAPE_PATTERN.sub("", message).strip()
# Then send to frontend (non-blocking)
if clean_message:
try:
event = ProcessLogEvent(
level="INFO",
message=clean_message,
logger="deeptutor.question.stdout",
timestamp=datetime.now().timestamp(),
context=current_log_context(),
)
self.queue.put_nowait(event.to_dict())
except (asyncio.QueueFull, RuntimeError):
pass
def flush(self):
if not self._closed:
try:
self.original_stdout.flush()
except Exception:
pass
def close(self):
"""Mark interceptor as closed to prevent further writes."""
self._closed = True
interceptor = StdoutInterceptor(log_queue, original_stdout)
sys.stdout = interceptor
try:
await websocket.send_json(
{"type": "status", "stage": "init", "content": "Initializing..."}
)
pdf_path = None
paper_dir = None
# Handle PDF upload mode
if mode == "upload":
pdf_data = data.get("pdf_data")
pdf_name = data.get("pdf_name", "exam.pdf")
if not pdf_data:
await websocket.send_json(
{"type": "error", "content": "PDF data is required for upload mode"}
)
return
# Decode PDF data first to check size
try:
pdf_bytes = base64.b64decode(pdf_data)
except Exception as e:
await websocket.send_json(
{"type": "error", "content": f"Invalid base64 PDF data: {e}"}
)
return
# Pre-validate filename and file size before writing
try:
safe_name = DocumentValidator.validate_upload_safety(
pdf_name, len(pdf_bytes), {".pdf"}
)
except ValueError as e:
await websocket.send_json({"type": "error", "content": str(e)})
return
# Create batch directory for this mimic session
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
pdf_stem = Path(safe_name).stem
batch_dir = _mimic_output_dir() / f"mimic_{timestamp}_{pdf_stem}"
batch_dir.mkdir(parents=True, exist_ok=True)
# Save uploaded PDF in batch directory
pdf_path = batch_dir / safe_name
await websocket.send_json(
{"type": "status", "stage": "upload", "content": f"Saving PDF: {safe_name}"}
)
# Write the validated PDF bytes
with open(pdf_path, "wb") as f:
f.write(pdf_bytes)
# Additional validation (file readability, etc.)
try:
DocumentValidator.validate_file(pdf_path)
except (ValueError, FileNotFoundError, PermissionError) as e:
# Clean up invalid or inaccessible file
pdf_path.unlink(missing_ok=True)
await websocket.send_json({"type": "error", "content": str(e)})
return
await websocket.send_json(
{
"type": "status",
"stage": "parsing",
"content": "Parsing PDF exam paper (MinerU)...",
}
)
logger.info(f"Saved and validated uploaded PDF to: {pdf_path}")
# Pass batch_dir as output directory
pdf_path = str(pdf_path)
output_dir = str(batch_dir)
elif mode == "parsed":
paper_path = data.get("paper_path")
if not paper_path:
await websocket.send_json(
{"type": "error", "content": "paper_path is required for parsed mode"}
)
return
paper_dir = paper_path
# Create batch directory for parsed mode too
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
batch_dir = _mimic_output_dir() / f"mimic_{timestamp}_{Path(paper_path).name}"
batch_dir.mkdir(parents=True, exist_ok=True)
output_dir = str(batch_dir)
else:
await websocket.send_json({"type": "error", "content": f"Unknown mode: {mode}"})
return
# Create WebSocket callback for real-time progress updates
async def ws_callback(event_type: str, data: dict):
"""Send progress updates to the frontend via WebSocket."""
try:
message = {"type": event_type, **data}
await websocket.send_json(message)
except Exception as e:
logger.debug(f"WebSocket send failed: {e}")
# Run the complete mimic workflow with callback
await websocket.send_json(
{
"type": "status",
"stage": "processing",
"content": "Executing question generation workflow...",
}
)
with bind_log_context(task_id=task_id, capability="deep_question", sink="ui"):
with capture_process_logs(emit_process_log, task_id=task_id):
result = await mimic_exam_questions(
pdf_path=pdf_path,
paper_dir=paper_dir,
kb_name=kb_name,
output_dir=output_dir,
max_questions=max_questions,
ws_callback=ws_callback,
)
if result.get("success"):
# Results are already sent via ws_callback during generation
# Just send the final complete signal
total_ref = result.get("total_reference_questions", 0)
generated = result.get("generated_questions", [])
failed = result.get("failed_questions", [])
logger.info(
f"Mimic generation complete: {len(generated)} succeeded, {len(failed)} failed"
)
try:
await websocket.send_json({"type": "complete"})
except (RuntimeError, WebSocketDisconnect):
logger.debug("WebSocket closed before complete signal could be sent")
else:
error_msg = result.get("error", "Unknown error")
try:
await websocket.send_json({"type": "error", "content": error_msg})
except (RuntimeError, WebSocketDisconnect):
pass
logger.error(f"Mimic generation failed: {error_msg}")
finally:
# Close interceptor and restore stdout
if "interceptor" in locals():
interceptor.close()
sys.stdout = original_stdout
except WebSocketDisconnect:
logger.debug("Client disconnected during mimic generation")
except Exception as e:
logger.exception("Mimic generation error")
error_msg = format_exception_message(e)
try:
await websocket.send_json({"type": "error", "content": error_msg})
except Exception:
pass
finally:
# Ensure stdout is always restored
sys.stdout = original_stdout
# Clean up pusher task
if pusher_task:
try:
pusher_task.cancel()
await pusher_task
except asyncio.CancelledError:
pass # Expected when cancelling
except Exception:
pass
# Drain any remaining items in the queue
try:
while not log_queue.empty():
log_queue.get_nowait()
except Exception:
pass
# Close WebSocket
try:
await websocket.close()
except Exception:
pass
if user_token is not None:
try:
reset_current_user(user_token)
except Exception:
pass
@router.websocket("/generate")
async def websocket_question_generate(websocket: WebSocket):
from deeptutor.api.routers.auth import ws_auth_failed, ws_require_auth
from deeptutor.multi_user.context import reset_current_user
user_token = await ws_require_auth(websocket)
if user_token is ws_auth_failed:
return
await websocket.accept()
# Get task ID manager
task_manager = TaskIDManager.get_instance()
try:
# 1. Wait for config
data = await websocket.receive_json()
requirement = data.get("requirement")
kb_name = data.get("kb_name", "ai_textbook")
count = data.get("count", 1)
if not requirement:
try:
await websocket.send_json({"type": "error", "content": "Requirement is required"})
except (RuntimeError, WebSocketDisconnect):
pass
return
# Generate task ID
task_key = f"question_{kb_name}_{hash(str(requirement))}"
task_id = task_manager.generate_task_id("question_gen", task_key)
# Send task ID to frontend
try:
await websocket.send_json({"type": "task_id", "task_id": task_id})
except (RuntimeError, WebSocketDisconnect):
logger.debug("WebSocket closed, cannot send task_id")
return
logger.info(
f"[{task_id}] Starting question generation: {requirement.get('knowledge_point', 'Unknown')}"
)
# 2. Initialize Coordinator
path_service = get_path_service()
output_base = path_service.get_question_batch_dir(task_id)
try:
llm_config = get_llm_config()
api_key = llm_config.api_key
base_url = llm_config.base_url
api_version = getattr(llm_config, "api_version", None)
except Exception:
api_key = None
base_url = None
api_version = None
coordinator = AgentCoordinator(
api_key=api_key,
base_url=base_url,
api_version=api_version,
kb_name=kb_name,
language=get_ui_language(default=config.get("system", {}).get("language", "en")),
output_dir=str(output_base),
)
# 3. Setup Log Queue for WebSocket streaming
log_queue = asyncio.Queue()
loop = asyncio.get_running_loop()
def emit_process_log(event: ProcessLogEvent) -> None:
loop.call_soon_threadsafe(log_queue.put_nowait, event.to_dict())
# WebSocket callback for coordinator to send structured updates
async def ws_callback(data: dict):
try:
await log_queue.put(data)
except Exception:
pass
coordinator.set_ws_callback(ws_callback)
# 4. Define background pusher for logs
async def log_pusher():
while True:
entry = await log_queue.get()
try:
await websocket.send_json(entry)
except Exception:
break
log_queue.task_done()
pusher_task = asyncio.create_task(log_pusher())
# 5. Run generation while streaming logs bound to this task.
try:
with bind_log_context(task_id=task_id, capability="deep_question", sink="ui"):
with capture_process_logs(emit_process_log, task_id=task_id):
try:
await websocket.send_json({"type": "status", "content": "started"})
except (RuntimeError, WebSocketDisconnect):
logger.debug("WebSocket closed, stopping question generation")
return
# Extract fields from requirement dict
user_topic = (
requirement.get("knowledge_point", "")
if isinstance(requirement, dict)
else str(requirement)
)
preference = (
requirement.get("preference", "") if isinstance(requirement, dict) else ""
)
difficulty = (
requirement.get("difficulty", "") if isinstance(requirement, dict) else ""
)
question_type = (
requirement.get("question_type", "")
if isinstance(requirement, dict)
else ""
)
logger.info(
f"Starting question generation for {count} question(s), topic: {user_topic}"
)
batch_result = await coordinator.generate_from_topic(
user_topic=user_topic,
preference=preference,
num_questions=count,
difficulty=difficulty,
question_type=question_type,
)
# Send batch summary
try:
await websocket.send_json(
{
"type": "batch_summary",
"requested": count,
"completed": batch_result.get("completed", 0),
"failed": batch_result.get("failed", 0),
}
)
except (RuntimeError, WebSocketDisconnect):
pass
if not batch_result.get("success"):
logger.warning(
f"Question generation had failures: {batch_result.get('failed', 0)} failed"
)
# Wait for any pending messages in the queue to be sent
# Give the pusher a moment to process remaining messages
await asyncio.sleep(0.1)
while not log_queue.empty():
await asyncio.sleep(0.05)
# Send complete signal
try:
await websocket.send_json({"type": "complete"})
logger.info(f"[{task_id}] Question generation completed")
task_manager.update_task_status(task_id, "completed")
except (RuntimeError, WebSocketDisconnect):
logger.debug("WebSocket closed, cannot send complete signal")
except Exception as e:
error_msg = format_exception_message(e)
error_traceback = traceback.format_exc()
logger.error(f"Question generation error: {error_msg}")
logger.error(f"Error traceback:\n{error_traceback}")
# Log additional context if available
try:
context_result = locals().get("batch_result")
if context_result is not None:
logger.error(
f"Result type: {type(context_result)}, result keys: {context_result.keys() if isinstance(context_result, dict) else 'N/A'}"
)
if isinstance(context_result, dict) and "validation" in context_result:
validation = context_result["validation"]
logger.error(f"Validation type: {type(validation)}")
if isinstance(validation, dict):
logger.error(f"Validation keys: {validation.keys()}")
logger.error(
f"Issues type: {type(validation.get('issues'))}, value: {validation.get('issues')}"
)
logger.error(
f"Suggestions type: {type(validation.get('suggestions'))}, value: {validation.get('suggestions')}"
)
except Exception as context_error:
logger.warning(f"Failed to log error context: {context_error}")
try:
await websocket.send_json({"type": "error", "content": error_msg})
except (RuntimeError, WebSocketDisconnect):
logger.debug("WebSocket closed, cannot send error message")
task_manager.update_task_status(task_id, "error", error=error_msg)
finally:
pusher_task.cancel()
try:
await pusher_task
except asyncio.CancelledError:
pass
await websocket.close()
except WebSocketDisconnect:
logger.debug("Client disconnected")
except Exception as e:
error_msg = format_exception_message(e)
logger.error(f"WebSocket error: {error_msg}")
finally:
if user_token is not None:
try:
reset_current_user(user_token)
except Exception:
pass
deeptutor/api/routers/question_notebook.py
+335
View File
@@ -0,0 +1,335 @@
"""
Question Notebook API — persists quiz questions, bookmarks, and categories.
"""
from __future__ import annotations
import base64 as _b64
import logging
import uuid as _uuid
from fastapi import APIRouter, HTTPException, Query, Response
from pydantic import BaseModel, Field
from deeptutor.services.session import get_sqlite_session_store
from deeptutor.services.storage import get_attachment_store
logger = logging.getLogger(__name__)
router = APIRouter()
# ── Models ────────────────────────────────────────────────────────
class AnswerImageItem(BaseModel):
"""Persisted reference to one image attached to a learner's answer.
The bytes live in the AttachmentStore at ``url``; we never round-trip
base64 back to the client so notebook lookups stay cheap.
"""
id: str = ""
url: str = ""
filename: str = ""
mime_type: str = ""
class NotebookEntryItem(BaseModel):
id: int
session_id: str
session_title: str = ""
turn_id: str = ""
question_id: str = ""
question: str
question_type: str = ""
options: dict[str, str] = {}
correct_answer: str = ""
explanation: str = ""
difficulty: str = ""
user_answer: str = ""
user_answer_images: list[AnswerImageItem] = []
is_correct: bool = False
bookmarked: bool = False
followup_session_id: str = ""
ai_judgment: str = ""
created_at: float
updated_at: float
categories: list[CategoryItem] | None = None
class NotebookEntryListResponse(BaseModel):
items: list[NotebookEntryItem]
total: int
class EntryUpdateRequest(BaseModel):
bookmarked: bool | None = None
followup_session_id: str | None = None
ai_judgment: str | None = None
class CategoryItem(BaseModel):
id: int
name: str
created_at: float = 0
entry_count: int = 0
class CategoryCreateRequest(BaseModel):
name: str = Field(..., min_length=1, max_length=100)
class CategoryRenameRequest(BaseModel):
name: str = Field(..., min_length=1, max_length=100)
class CategoryAddRequest(BaseModel):
category_id: int
class AnswerImageUpload(BaseModel):
"""One image attached to the learner's answer.
Either ``base64`` (new upload) or ``url`` (re-submit of an already
persisted image) must be set. ``id`` is preserved when the client
sends one so the same logical image keeps a stable AttachmentStore
record across resubmissions.
"""
id: str = ""
base64: str = ""
url: str = ""
filename: str = "answer.png"
mime_type: str = "image/png"
class UpsertEntryRequest(BaseModel):
session_id: str
turn_id: str = ""
question_id: str
question: str
question_type: str = ""
options: dict[str, str] | None = None
correct_answer: str = ""
explanation: str = ""
difficulty: str = ""
user_answer: str = ""
# Optional: list of images attached as part of the learner's answer.
# ``None`` means "don't touch any previously-stored images on update";
# an empty list explicitly clears them.
user_answer_images: list[AnswerImageUpload] | None = None
is_correct: bool = False
# ── Entry endpoints ──────────────────────────────────────────────
async def _persist_answer_images(
session_id: str, images: list[AnswerImageUpload] | None
) -> list[dict[str, str]] | None:
"""Materialise base64 image uploads into the AttachmentStore.
Returns a list of ``{id, url, filename, mime_type}`` records suitable
for ``notebook_entries.user_answer_images_json``. ``None`` is returned
when ``images`` is ``None`` (no change to existing stored images).
Records whose bytes fail to upload are dropped from the result with
a warning — losing an image is better than failing the whole upsert.
"""
if images is None:
return None
attachment_store = get_attachment_store()
records: list[dict[str, str]] = []
for image in images:
record_id = (image.id or _uuid.uuid4().hex[:12]).strip()
filename = (image.filename or "answer.png").strip() or "answer.png"
mime_type = (image.mime_type or "image/png").strip() or "image/png"
url = (image.url or "").strip()
if not url and image.base64:
try:
raw_bytes = _b64.b64decode(image.base64, validate=False)
except Exception as exc:
logger.warning("answer image %s rejected: invalid base64 (%s)", filename, exc)
continue
try:
url = await attachment_store.put(
session_id=session_id,
attachment_id=record_id,
filename=filename,
data=raw_bytes,
mime_type=mime_type,
)
except Exception as exc:
logger.warning("attachment store rejected answer image %s: %s", filename, exc)
continue
if not url:
# No url and no base64 — nothing usable.
continue
records.append(
{
"id": record_id,
"url": url,
"filename": filename,
"mime_type": mime_type,
}
)
return records
@router.post("/entries/upsert")
async def upsert_single_entry(payload: UpsertEntryRequest):
store = get_sqlite_session_store()
images_records = await _persist_answer_images(payload.session_id, payload.user_answer_images)
item = payload.model_dump()
# The store expects ``user_answer_images`` as a plain list of dicts
# (or absent to mean "leave the stored images alone"). Strip the
# upload payload version and replace with the persisted records.
item.pop("user_answer_images", None)
if images_records is not None:
item["user_answer_images"] = images_records
try:
await store.upsert_notebook_entries(payload.session_id, [item])
except ValueError as e:
raise HTTPException(status_code=404, detail=str(e))
entry = await store.find_notebook_entry(
payload.session_id, payload.question_id, turn_id=payload.turn_id
)
if entry is None:
raise HTTPException(status_code=500, detail="Upsert failed")
return entry
@router.get("/entries", response_model=NotebookEntryListResponse)
async def list_entries(
category_id: int | None = Query(default=None),
bookmarked: bool | None = Query(default=None),
is_correct: bool | None = Query(default=None),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
) -> NotebookEntryListResponse:
store = get_sqlite_session_store()
result = await store.list_notebook_entries(
category_id=category_id,
bookmarked=bookmarked,
is_correct=is_correct,
limit=limit,
offset=offset,
)
return NotebookEntryListResponse(
items=[NotebookEntryItem(**item) for item in result["items"]],
total=result["total"],
)
@router.get("/entries/lookup/by-question")
async def lookup_entry(
session_id: str = Query(...),
question_id: str = Query(...),
turn_id: str | None = Query(default=None),
missing_ok: bool = Query(
default=False,
description="Return 204 No Content instead of 404 when the entry is "
"absent — used by the quiz viewer to probe not-yet-saved questions "
"without logging noisy 404s.",
),
):
store = get_sqlite_session_store()
entry = await store.find_notebook_entry(session_id, question_id, turn_id=turn_id)
if entry is None:
if missing_ok:
return Response(status_code=204)
raise HTTPException(status_code=404, detail="Entry not found")
return entry
@router.get("/entries/{entry_id}", response_model=NotebookEntryItem)
async def get_entry(entry_id: int) -> NotebookEntryItem:
store = get_sqlite_session_store()
entry = await store.get_notebook_entry(entry_id)
if entry is None:
raise HTTPException(status_code=404, detail="Entry not found")
return NotebookEntryItem(**entry)
@router.patch("/entries/{entry_id}")
async def update_entry(entry_id: int, payload: EntryUpdateRequest):
store = get_sqlite_session_store()
updates = payload.model_dump(exclude_none=True)
if not updates:
raise HTTPException(status_code=400, detail="No fields to update")
updated = await store.update_notebook_entry(entry_id, updates)
if not updated:
raise HTTPException(status_code=404, detail="Entry not found")
return {"updated": True, "id": entry_id}
@router.delete("/entries/{entry_id}")
async def delete_entry(entry_id: int):
store = get_sqlite_session_store()
deleted = await store.delete_notebook_entry(entry_id)
if not deleted:
raise HTTPException(status_code=404, detail="Entry not found")
return {"deleted": True, "id": entry_id}
# ── Entry ↔ Category linking ────────────────────────────────────
@router.post("/entries/{entry_id}/categories")
async def add_entry_to_category(entry_id: int, payload: CategoryAddRequest):
store = get_sqlite_session_store()
entry = await store.get_notebook_entry(entry_id)
if entry is None:
raise HTTPException(status_code=404, detail="Entry not found")
ok = await store.add_entry_to_category(entry_id, payload.category_id)
if not ok:
raise HTTPException(status_code=400, detail="Failed to add to category")
return {"added": True, "entry_id": entry_id, "category_id": payload.category_id}
@router.delete("/entries/{entry_id}/categories/{category_id}")
async def remove_entry_from_category(entry_id: int, category_id: int):
store = get_sqlite_session_store()
removed = await store.remove_entry_from_category(entry_id, category_id)
if not removed:
raise HTTPException(status_code=404, detail="Link not found")
return {"removed": True, "entry_id": entry_id, "category_id": category_id}
# ── Category CRUD ────────────────────────────────────────────────
@router.get("/categories", response_model=list[CategoryItem])
async def list_categories():
store = get_sqlite_session_store()
return await store.list_categories()
@router.post("/categories", response_model=CategoryItem, status_code=201)
async def create_category(payload: CategoryCreateRequest):
store = get_sqlite_session_store()
try:
return await store.create_category(payload.name)
except Exception:
raise HTTPException(status_code=409, detail="Category name already exists")
@router.patch("/categories/{category_id}")
async def rename_category(category_id: int, payload: CategoryRenameRequest):
store = get_sqlite_session_store()
updated = await store.rename_category(category_id, payload.name)
if not updated:
raise HTTPException(status_code=404, detail="Category not found")
return {"updated": True, "id": category_id, "name": payload.name}
@router.delete("/categories/{category_id}")
async def delete_category(category_id: int):
store = get_sqlite_session_store()
deleted = await store.delete_category(category_id)
if not deleted:
raise HTTPException(status_code=404, detail="Category not found")
return {"deleted": True, "id": category_id}
deeptutor/api/routers/quiz_judge.py
+427
View File
@@ -0,0 +1,427 @@
"""AI judge WebSocket — grades a learner's quiz answer.
Mounted on its own (without router-level HTTP auth dependencies) because
WebSocket upgrades cannot use FastAPI's HTTP dependency injection, so we
rely on ``ws_require_auth`` inside the handler — mirroring the pattern
used by ``unified_ws``.
"""
from __future__ import annotations
import base64 as _b64
import logging
from typing import Any
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from deeptutor.services.config import PROJECT_ROOT, load_config_with_main
from deeptutor.services.llm import stream as llm_stream
from deeptutor.services.settings.interface_settings import get_ui_language
from deeptutor.utils.error_utils import format_exception_message
logger = logging.getLogger(__name__)
_config = load_config_with_main("main.yaml", PROJECT_ROOT)
router = APIRouter()
_JUDGE_SYSTEM_PROMPTS = {
"zh": (
"你是一名严谨且鼓励学习者的助教，正在批改一道测验题。"
"请基于题目、参考答案与解析，对学习者的作答给出针对性的判定与反馈。\n\n"
"回答要求：\n"
"- 先用一行明确结论：✅ 正确 / ⚠️ 部分正确 / ❌ 不正确，并简短点明关键判定依据。\n"
"- 然后分条列出：哪里做对了、哪里出错或缺漏、应该如何改正。\n"
"- 若题目本身有多种合理答案，请承认学习者的合理之处。\n"
"- 直接以学习者的作答为对象，不要泛泛而谈。\n"
"- 全程使用中文。"
),
"en": (
"You are a rigorous yet encouraging teaching assistant grading a learner's quiz answer. "
"Use the question, reference answer, and explanation to deliver a targeted assessment.\n\n"
"Requirements:\n"
"- Open with one line that states the verdict: ✅ Correct / ⚠️ Partially correct / ❌ Incorrect, "
"and the key reason.\n"
"- Then list: what the learner got right, what is wrong or missing, and how to fix it.\n"
"- If multiple reasonable answers exist, acknowledge what the learner did well.\n"
"- Speak directly to the learner's submission — do not give a generic lecture.\n"
"- Reply in English."
),
}
def _build_judge_user_prompt(
*,
language: str,
question: str,
question_type: str,
options: dict | None,
correct_answer: str,
explanation: str,
user_answer: str,
has_image: bool,
image_count: int = 0,
) -> str:
options_block = ""
if options:
try:
options_block = "\n".join(f" {k}. {v}" for k, v in options.items())
except Exception:
options_block = ""
if language == "zh":
parts = [
f"题目类型：{question_type or 'unknown'}",
f"题干：\n{question}",
]
if options_block:
parts.append(f"选项：\n{options_block}")
if correct_answer:
parts.append(f"参考答案：\n{correct_answer}")
if explanation:
parts.append(f"参考解析：\n{explanation}")
parts.append(
"学习者作答：\n"
+ (
user_answer.strip()
if user_answer and user_answer.strip()
else "（仅提交了图片，无文字作答）"
)
)
if has_image:
count_text = (
f"学习者另附了 {image_count} 张图片作为作答内容"
if image_count > 1
else "学习者另附了一张图片作为作答内容"
)
parts.append(f"{count_text}，请结合图片中的文字/公式/草图一并判定。")
parts.append("请针对该学习者的具体作答给出 AI 评判。")
else:
parts = [
f"Question type: {question_type or 'unknown'}",
f"Question:\n{question}",
]
if options_block:
parts.append(f"Options:\n{options_block}")
if correct_answer:
parts.append(f"Reference answer:\n{correct_answer}")
if explanation:
parts.append(f"Reference explanation:\n{explanation}")
parts.append(
"Learner's answer:\n"
+ (
user_answer.strip()
if user_answer and user_answer.strip()
else "(only an image was submitted, no typed answer)"
)
)
if has_image:
if image_count > 1:
parts.append(
f"The learner attached {image_count} images as part of the answer. "
"Read their text/formulas/sketches and factor them into the judgment."
)
else:
parts.append(
"The learner attached an image as part of the answer. "
"Read its text/formulas/sketches and factor it into the judgment."
)
parts.append("Produce an AI judgment that addresses this learner's specific answer.")
return "\n\n".join(parts)
async def _build_multimodal_user_content(
*,
text: str,
image_records: list[dict[str, str]],
) -> list[dict[str, Any]]:
"""Compose an OpenAI-style content-parts array with text + image blocks.
For ``url``-only records we resolve local AttachmentStore paths to
base64 here (most providers can fetch external URLs themselves, but
locally-hosted ``/api/attachments/...`` is only reachable from the
browser). Falls back to passing the URL through when resolution is
not possible.
"""
from urllib.parse import unquote, urlparse
from deeptutor.services.storage import get_attachment_store
content: list[dict[str, Any]] = [{"type": "text", "text": text}]
attachment_store = get_attachment_store()
resolve = getattr(attachment_store, "resolve_path", None)
for record in image_records:
b64 = record.get("base64") or ""
url = record.get("url") or ""
filename = record.get("filename") or "answer.png"
mime_type = record.get("mime_type") or _guess_image_mime(filename)
if not b64 and url and resolve is not None:
try:
parsed = urlparse(url)
parts = (parsed.path or url).strip("/").split("/")
# Expected shape: api/attachments/{sid}/{aid}/{name}
if len(parts) >= 5 and parts[0] == "api" and parts[1] == "attachments":
sid = unquote(parts[2])
aid = unquote(parts[3])
name = unquote("/".join(parts[4:]))
target = resolve(session_id=sid, attachment_id=aid, filename=name)
if target is not None and target.exists():
b64 = _b64.b64encode(target.read_bytes()).decode("ascii")
except Exception as exc:
logger.debug("Could not resolve %s to bytes: %s", url, exc)
if b64:
data_url = f"data:{mime_type};base64,{b64}"
content.append({"type": "image_url", "image_url": {"url": data_url}})
elif url:
content.append({"type": "image_url", "image_url": {"url": url}})
return content
def _guess_image_mime(filename: str | None) -> str:
if not filename:
return "image/png"
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
return {
"png": "image/png",
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"gif": "image/gif",
"webp": "image/webp",
}.get(ext, "image/png")
@router.websocket("/question/judge")
async def websocket_quiz_judge(websocket: WebSocket):
"""Stream an AI judgment for a single quiz answer.
Auth is enforced via ``ws_require_auth`` rather than a router-level
HTTP dependency — see module docstring.
Client → Server (initial JSON):
{
"question": str,
"question_type": str,
"options": dict | null,
"correct_answer": str,
"explanation": str,
"user_answer": str,
# New: list of image entries. Each entry has either ``base64``
# (no ``data:`` prefix) or ``url`` (already hosted via the
# AttachmentStore). ``user_answer_image`` (single, base64) is
# still accepted for backward compatibility.
"user_answer_images": [
{"base64": str, "url": str, "filename": str, "mime_type": str},
...
] | null,
"user_answer_image": str | null, # legacy single-image form
"image_filename": str | null, # legacy filename for the above
"language": "zh" | "en",
}
Server → Client (streaming):
{"type": "started"}
{"type": "text", "content": "..."} # zero or more
{"type": "done"}
{"type": "error", "content": "..."}
"""
from deeptutor.api.routers.auth import ws_auth_failed, ws_require_auth
from deeptutor.multi_user.context import reset_current_user
user_token = await ws_require_auth(websocket)
if user_token is ws_auth_failed:
return
await websocket.accept()
async def safe_send(payload: dict[str, Any]) -> bool:
try:
await websocket.send_json(payload)
return True
except (WebSocketDisconnect, RuntimeError, ConnectionError):
return False
try:
data = await websocket.receive_json()
except WebSocketDisconnect:
return
except Exception as exc:
await safe_send({"type": "error", "content": f"Invalid request: {exc}"})
try:
await websocket.close()
except Exception:
pass
if user_token is not None:
try:
reset_current_user(user_token)
except Exception:
pass
return
question_text = (data.get("question") or "").strip()
if not question_text:
await safe_send({"type": "error", "content": "Question is required"})
try:
await websocket.close()
except Exception:
pass
if user_token is not None:
try:
reset_current_user(user_token)
except Exception:
pass
return
requested_language = (data.get("language") or "").strip().lower()
if requested_language not in ("zh", "en"):
requested_language = get_ui_language(
default=_config.get("system", {}).get("language", "en")
)
if requested_language not in ("zh", "en"):
requested_language = "en"
user_answer = data.get("user_answer") or ""
# Resolve the image set. New clients send ``user_answer_images`` (list);
# legacy clients send the single ``user_answer_image`` + ``image_filename``
# pair. Build a uniform list of ``{base64, url, filename, mime_type}`` so
# the downstream multimodal-message builder doesn't care which form
# arrived.
raw_images = data.get("user_answer_images")
image_records: list[dict[str, str]] = []
if isinstance(raw_images, list):
for entry in raw_images:
if not isinstance(entry, dict):
continue
b64 = entry.get("base64") or ""
url = entry.get("url") or ""
if isinstance(b64, str) and b64.startswith("data:"):
try:
b64 = b64.split(",", 1)[1]
except IndexError:
b64 = ""
if not b64 and not url:
continue
filename = entry.get("filename") or "answer.png"
mime_type = entry.get("mime_type") or _guess_image_mime(filename)
image_records.append(
{
"base64": b64,
"url": url,
"filename": filename,
"mime_type": mime_type,
}
)
else:
legacy_b64 = data.get("user_answer_image") or ""
if isinstance(legacy_b64, str) and legacy_b64.startswith("data:"):
try:
legacy_b64 = legacy_b64.split(",", 1)[1]
except IndexError:
legacy_b64 = ""
if legacy_b64:
legacy_filename = data.get("image_filename") or "answer.png"
image_records.append(
{
"base64": legacy_b64,
"url": "",
"filename": legacy_filename,
"mime_type": _guess_image_mime(legacy_filename),
}
)
has_image = bool(image_records)
options_value = data.get("options") if isinstance(data.get("options"), dict) else None
system_prompt = _JUDGE_SYSTEM_PROMPTS.get(requested_language, _JUDGE_SYSTEM_PROMPTS["en"])
user_prompt = _build_judge_user_prompt(
language=requested_language,
question=question_text,
question_type=data.get("question_type") or "",
options=options_value,
correct_answer=data.get("correct_answer") or "",
explanation=data.get("explanation") or "",
user_answer=user_answer,
has_image=has_image,
image_count=len(image_records),
)
if not (user_answer.strip() or has_image):
await safe_send(
{
"type": "error",
"content": ("No answer to judge — submit a typed answer or attach an image."),
}
)
try:
await websocket.close()
except Exception:
pass
if user_token is not None:
try:
reset_current_user(user_token)
except Exception:
pass
return
await safe_send({"type": "started"})
# Build a multimodal user message when ≥1 image was attached. We pass
# the full ``messages`` array to ``factory.stream`` so it forwards the
# content-parts unchanged (the single-image ``image_data`` kwarg only
# supports one image).
stream_kwargs: dict[str, Any] = {}
if has_image:
from deeptutor.services.llm import config as _llm_config_mod
from deeptutor.services.llm.capabilities import supports_vision
llm_cfg = _llm_config_mod.get_llm_config()
binding = getattr(llm_cfg, "binding", "openai") or "openai"
model = getattr(llm_cfg, "model", "") or ""
if supports_vision(binding, model):
user_content = await _build_multimodal_user_content(
text=user_prompt,
image_records=image_records,
)
stream_kwargs["messages"] = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_content},
]
else:
# Vision-incapable model — fall back to text-only judge so the
# learner still gets feedback on their typed answer.
logger.info(
"Judge: %s/%s does not support vision; dropping %d image(s)",
binding,
model,
len(image_records),
)
try:
async for chunk in llm_stream(
prompt=user_prompt,
system_prompt=system_prompt,
**stream_kwargs,
):
if not chunk:
continue
if not await safe_send({"type": "text", "content": chunk}):
break
await safe_send({"type": "done"})
except WebSocketDisconnect:
logger.debug("AI judge client disconnected mid-stream")
except Exception as exc:
logger.exception("AI judge stream failed")
await safe_send({"type": "error", "content": format_exception_message(exc)})
finally:
try:
await websocket.close()
except Exception:
pass
if user_token is not None:
try:
reset_current_user(user_token)
except Exception:
pass
deeptutor/api/routers/sessions.py
+230
View File
@@ -0,0 +1,230 @@
"""
Unified session history API.
"""
from __future__ import annotations
import logging
from typing import Any
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field, field_validator
from deeptutor.services.session import get_session_store, get_sqlite_session_store
from deeptutor.services.storage.attachment_store import get_attachment_store
logger = logging.getLogger(__name__)
router = APIRouter()
class SessionRenameRequest(BaseModel):
title: str = Field(..., min_length=1, max_length=100)
class BranchSelectionRequest(BaseModel):
"""Edit-branch picker state: `{parent_message_id: chosen_child_id}`.
Stored inside the session preferences blob so it survives reloads
without a dedicated column.
"""
selected_branches: dict[str, int] = Field(default_factory=dict)
class QuizResultItem(BaseModel):
question_id: str = ""
question: str = Field(..., min_length=1)
question_type: str = ""
options: dict[str, str] | None = None
user_answer: str = ""
correct_answer: str = ""
explanation: str | None = ""
difficulty: str | None = ""
is_correct: bool
@field_validator("options", mode="before")
@classmethod
def _coerce_options(cls, v):
return v if isinstance(v, dict) else {}
@field_validator("explanation", "difficulty", mode="before")
@classmethod
def _coerce_str(cls, v):
return v if isinstance(v, str) else ""
class QuizResultsRequest(BaseModel):
answers: list[QuizResultItem] = Field(default_factory=list)
turn_id: str = ""
def _format_quiz_results_message(answers: list[QuizResultItem]) -> str:
total = len(answers)
correct = sum(1 for item in answers if item.is_correct)
score_pct = round((correct / total) * 100) if total else 0
lines = ["[Quiz Performance]"]
for idx, item in enumerate(answers, 1):
question = item.question.strip().replace("\n", " ")
user_answer = (item.user_answer or "").strip() or "(blank)"
status = "Correct" if item.is_correct else "Incorrect"
suffix = f" ({status})"
if not item.is_correct and (item.correct_answer or "").strip():
suffix = f" ({status}, correct: {(item.correct_answer or '').strip()})"
qid = f"[{item.question_id}] " if item.question_id else ""
lines.append(f"{idx}. {qid}Q: {question} -> Answered: {user_answer}{suffix}")
lines.append(f"Score: {correct}/{total} ({score_pct}%)")
return "\n".join(lines)
@router.get("")
async def list_sessions(
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
):
store = get_session_store()
sessions = await store.list_sessions(limit=limit, offset=offset)
return {"sessions": sessions}
# Cap (in characters) for a single event payload returned to the UI. RAG
# tools can attach whole KB documents to ``tool_result``/``observation``
# events; the frontend TraceSurface only needs a preview, and the LLM context
# is built from a separate content-only store, so capping here never affects
# model input.
MAX_EVENT_PAYLOAD = 1024 * 1024
_TRUNCATION_NOTICE = "\n\n[... content truncated]"
_TRUNCATABLE_EVENT_TYPES = ("tool_result", "observation")
def _truncate_oversized_events(
messages: list[dict[str, Any]], limit: int = MAX_EVENT_PAYLOAD
) -> None:
"""Cap oversized ``tool_result``/``observation`` payloads in place.
The session store already returns each message's events as a parsed
``events`` list (see ``SqliteSessionStore._serialize_message``), so we
mutate that list directly. Only the UI rendering path is affected.
"""
def _cap(container: dict[str, Any], field: str) -> bool:
value = container.get(field)
if isinstance(value, str) and len(value) > limit:
container[field] = value[:limit] + _TRUNCATION_NOTICE
return True
return False
for msg in messages:
events = msg.get("events")
if not isinstance(events, list):
continue
for event in events:
if not isinstance(event, dict) or event.get("type") not in _TRUNCATABLE_EVENT_TYPES:
continue
truncated = _cap(event, "content")
tool_metadata = (event.get("metadata") or {}).get("tool_metadata")
if isinstance(tool_metadata, dict):
for field in ("content", "answer"):
truncated = _cap(tool_metadata, field) or truncated
if truncated:
event["_truncated"] = True
@router.get("/{session_id}")
async def get_session(session_id: str):
store = get_session_store()
session = await store.get_session_with_messages(session_id)
if session is None:
raise HTTPException(status_code=404, detail="Session not found")
_truncate_oversized_events(session.get("messages", []))
return session
@router.patch("/{session_id}")
async def rename_session(session_id: str, payload: SessionRenameRequest):
store = get_session_store()
updated = await store.update_session_title(session_id, payload.title)
if not updated:
raise HTTPException(status_code=404, detail="Session not found")
session = await store.get_session(session_id)
return {"session": session}
@router.delete("/{session_id}")
async def delete_session(session_id: str):
store = get_session_store()
deleted = await store.delete_session(session_id)
if not deleted:
raise HTTPException(status_code=404, detail="Session not found")
try:
await get_attachment_store().delete_session(session_id)
except Exception:
logger.exception("failed to clean up attachments for session %s", session_id)
return {"deleted": True, "session_id": session_id}
@router.put("/{session_id}/branch-selection")
async def update_branch_selection(session_id: str, payload: BranchSelectionRequest):
store = get_sqlite_session_store()
session = await store.get_session(session_id)
if session is None:
raise HTTPException(status_code=404, detail="Session not found")
updated = await store.update_session_preferences(
session_id, {"selected_branches": dict(payload.selected_branches)}
)
if not updated:
raise HTTPException(status_code=404, detail="Session not found")
return {"selected_branches": payload.selected_branches}
@router.delete("/{session_id}/messages/{message_id}")
async def delete_turn_by_message(session_id: str, message_id: int):
store = get_sqlite_session_store()
result = await store.delete_turn_by_message(session_id, message_id)
if result["was_running"]:
raise HTTPException(
status_code=409, detail="Cannot delete a message while its turn is running"
)
if not result["deleted"]:
raise HTTPException(status_code=404, detail="Message not found")
attachment_store = get_attachment_store()
for aid in result["attachment_ids"]:
try:
await attachment_store.delete_attachment(session_id, aid)
except Exception:
logger.exception("failed to delete attachment %s for session %s", aid, session_id)
return result
@router.post("/{session_id}/quiz-results")
async def record_quiz_results(session_id: str, payload: QuizResultsRequest):
if not payload.answers:
raise HTTPException(status_code=400, detail="Quiz results are required")
store = get_sqlite_session_store()
session = await store.get_session(session_id)
if session is None:
raise HTTPException(status_code=404, detail="Session not found")
content = _format_quiz_results_message(payload.answers)
await store.add_message(
session_id=session_id,
role="user",
content=content,
capability="deep_question",
)
notebook_count = 0
try:
notebook_count = await store.upsert_notebook_entries(
session_id,
[{**item.model_dump(), "turn_id": payload.turn_id} for item in payload.answers],
)
except Exception:
logger.warning(
"Failed to upsert notebook entries for session %s", session_id, exc_info=True
)
return {
"recorded": True,
"session_id": session_id,
"answer_count": len(payload.answers),
"notebook_count": notebook_count,
"content": content,
}
deeptutor/api/routers/settings.py
+1102
View File
File diff suppressed because it is too large Load Diff
deeptutor/api/routers/skills.py
+305
View File
@@ -0,0 +1,305 @@
"""
Skills API Router
=================
CRUD endpoints for user-authored SKILL.md files stored under
``data/user/workspace/skills/<name>/SKILL.md``.
Mounted at ``/api/v1/skills``.
"""
from __future__ import annotations
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, Field
from deeptutor.multi_user.context import get_current_user
from deeptutor.multi_user.skill_access import (
assigned_skill_detail,
assigned_skill_ids,
assigned_skill_infos,
)
from deeptutor.services.skill import get_skill_service
from deeptutor.services.skill.service import (
InvalidSkillNameError,
InvalidTagError,
SkillExistsError,
SkillImportError,
SkillNotFoundError,
SkillReadOnlyError,
TagExistsError,
TagNotFoundError,
)
router = APIRouter()
class CreateSkillRequest(BaseModel):
name: str = Field(..., min_length=1, max_length=64)
description: str = ""
content: str = ""
tags: list[str] = Field(default_factory=list)
class UpdateSkillRequest(BaseModel):
description: str | None = None
content: str | None = None
rename_to: str | None = None
tags: list[str] | None = None
class InstallSkillRequest(BaseModel):
"""Install a hub skill into the caller's own skill layer.
``ref`` is a ``<hub>:<slug>[@version]`` reference (the bare hub prefix
defaults to ``eduhub``). The web "import from EduHub" flow always builds an
``eduhub:`` ref so a spoofed bridge message can't redirect the install to an
arbitrary registry.
"""
ref: str = Field(..., min_length=1, max_length=256)
name: str | None = None
force: bool = False
allow_unverified: bool = False
class CreateTagRequest(BaseModel):
name: str = Field(..., min_length=1, max_length=32)
class RenameTagRequest(BaseModel):
rename_to: str = Field(..., min_length=1, max_length=32)
# ── tag routes (declared before `/{name}` so they are not shadowed) ──
@router.get("/tags/list")
async def list_tags() -> dict[str, list[str]]:
service = get_skill_service()
return {"tags": service.list_tags()}
@router.post("/tags/create")
async def create_tag(payload: CreateTagRequest) -> dict[str, str]:
service = get_skill_service()
try:
tag = service.create_tag(payload.name)
except TagExistsError as exc:
raise HTTPException(status_code=409, detail=f"Tag already exists: {exc}")
except InvalidTagError as exc:
raise HTTPException(status_code=400, detail=str(exc))
return {"name": tag}
@router.put("/tags/{tag}")
async def rename_tag(tag: str, payload: RenameTagRequest) -> dict[str, str]:
service = get_skill_service()
try:
new_tag = service.rename_tag(tag, payload.rename_to)
except TagNotFoundError:
raise HTTPException(status_code=404, detail=f"Tag not found: {tag}")
except TagExistsError as exc:
raise HTTPException(status_code=409, detail=f"Tag already exists: {exc}")
except InvalidTagError as exc:
raise HTTPException(status_code=400, detail=str(exc))
return {"name": new_tag}
@router.delete("/tags/{tag}")
async def delete_tag(tag: str) -> dict[str, str]:
service = get_skill_service()
try:
service.delete_tag(tag)
except TagNotFoundError:
raise HTTPException(status_code=404, detail=f"Tag not found: {tag}")
except InvalidTagError as exc:
raise HTTPException(status_code=400, detail=str(exc))
return {"status": "deleted", "name": tag}
# ── skill routes ───────────────────────────────────────────────────
@router.get("/list")
async def list_skills() -> dict[str, list[dict[str, object]]]:
service = get_skill_service()
own_items = [info.to_dict() for info in service.list_skills()]
user = get_current_user()
if user.is_admin:
return {"skills": own_items}
own_names = {item.get("name") for item in own_items}
merged = list(own_items)
for assigned in assigned_skill_infos(user.id):
if assigned.get("name") in own_names:
continue
merged.append(assigned)
return {"skills": merged}
# ── hub browse (in-app EduHub skill browser; declared before `/{name}`) ──
@router.get("/hub/catalog")
async def hub_catalog(hub: str = "eduhub", q: str = "", limit: int = 50) -> dict[str, object]:
"""Proxy a skill hub's public catalog for the in-app browser.
The web "Import from EduHub" panel renders these rows in DeepTutor's own
UI — no embedded iframe, no login — so users can browse, search, and
one-click download skills. Returns ``web_url`` (the hub's site origin) so
the panel can offer a "view on EduHub" link out.
"""
import asyncio
from deeptutor.services.skill.hub import ClawHubProvider, HubError, get_hub_provider
try:
provider = get_hub_provider(hub)
except HubError as exc:
raise HTTPException(status_code=400, detail=str(exc))
if not isinstance(provider, ClawHubProvider):
raise HTTPException(status_code=400, detail=f"Hub `{hub}` does not support browsing.")
try:
skills = await asyncio.to_thread(provider.catalog, query=q, limit=limit)
except HubError as exc:
raise HTTPException(status_code=502, detail=str(exc))
return {"hub": provider.name, "web_url": provider.web_origin, "skills": skills}
@router.get("/hub/detail")
async def hub_detail(slug: str, hub: str = "eduhub") -> dict[str, object]:
"""Full metadata + SKILL.md body for one hub skill (browser detail view)."""
import asyncio
from deeptutor.services.skill.hub import ClawHubProvider, HubError, get_hub_provider
try:
provider = get_hub_provider(hub)
except HubError as exc:
raise HTTPException(status_code=400, detail=str(exc))
if not isinstance(provider, ClawHubProvider):
raise HTTPException(status_code=400, detail=f"Hub `{hub}` does not support browsing.")
try:
detail = await asyncio.to_thread(provider.detail, slug)
except HubError as exc:
raise HTTPException(status_code=502, detail=str(exc))
detail["web_url"] = f"{provider.web_origin}/skills/{slug}"
return detail
@router.get("/{name}")
async def get_skill(name: str) -> dict[str, object]:
service = get_skill_service()
try:
return service.get_detail(name).to_dict()
except SkillNotFoundError:
# User scope doesn't have it. Fall through to admin-assigned lookup
# below, which returns 403 if the user has no grant for it.
pass
except InvalidSkillNameError as exc:
raise HTTPException(status_code=400, detail=str(exc))
user = get_current_user()
if user.is_admin:
# Admin scope already checked above; nothing else to look at.
raise HTTPException(status_code=404, detail=f"Skill not found: {name}")
if name not in assigned_skill_ids(user.id):
raise HTTPException(status_code=403, detail="Skill is not assigned to you")
detail = assigned_skill_detail(name)
if detail is None:
raise HTTPException(status_code=404, detail=f"Skill not found: {name}")
return detail
@router.post("/create")
async def create_skill(payload: CreateSkillRequest) -> dict[str, object]:
service = get_skill_service()
try:
info = service.create(
name=payload.name,
description=payload.description,
content=payload.content,
tags=list(payload.tags or []),
)
return info.to_dict()
except SkillExistsError:
raise HTTPException(status_code=409, detail=f"Skill already exists: {payload.name}")
except InvalidSkillNameError as exc:
raise HTTPException(status_code=400, detail=str(exc))
except InvalidTagError as exc:
raise HTTPException(status_code=400, detail=str(exc))
@router.post("/install")
async def install_skill(payload: InstallSkillRequest) -> dict[str, object]:
"""Import a hub skill (e.g. from EduHub) into the caller's own skill layer.
Lands the package in the same per-user dir that ``/create`` writes to, so
the imported skill shows up in this user's Skills list. The install gate
(``suspicious`` verdict abort, safe extraction, ``always`` stripping)
lives in :func:`deeptutor.services.skill.hub.install_from_hub`.
"""
import asyncio
from deeptutor.services.skill.hub import HubError, install_from_hub
service = get_skill_service()
try:
outcome = await asyncio.to_thread(
install_from_hub,
payload.ref,
service=service,
rename_to=payload.name,
force=payload.force,
allow_unverified=payload.allow_unverified,
)
except SkillExistsError as exc:
raise HTTPException(status_code=409, detail=f"Skill already exists: {exc}")
except (SkillImportError, InvalidSkillNameError) as exc:
raise HTTPException(status_code=400, detail=str(exc))
except HubError as exc:
raise HTTPException(status_code=502, detail=str(exc))
return {
"skill": outcome.result.info.to_dict(),
"verdict": {"status": outcome.verdict.status, "detail": outcome.verdict.detail},
"version": outcome.ref.version,
}
@router.put("/{name}")
async def update_skill(name: str, payload: UpdateSkillRequest) -> dict[str, object]:
service = get_skill_service()
try:
info = service.update(
name,
description=payload.description,
content=payload.content,
rename_to=payload.rename_to,
tags=payload.tags,
)
return info.to_dict()
except SkillNotFoundError:
raise HTTPException(status_code=404, detail=f"Skill not found: {name}")
except SkillReadOnlyError as exc:
raise HTTPException(status_code=403, detail=str(exc))
except SkillExistsError as exc:
raise HTTPException(status_code=409, detail=str(exc))
except InvalidSkillNameError as exc:
raise HTTPException(status_code=400, detail=str(exc))
except InvalidTagError as exc:
raise HTTPException(status_code=400, detail=str(exc))
@router.delete("/{name}")
async def delete_skill(name: str) -> dict[str, str]:
service = get_skill_service()
try:
service.delete(name)
return {"status": "deleted", "name": name}
except SkillNotFoundError:
raise HTTPException(status_code=404, detail=f"Skill not found: {name}")
except SkillReadOnlyError as exc:
raise HTTPException(status_code=403, detail=str(exc))
except InvalidSkillNameError as exc:
raise HTTPException(status_code=400, detail=str(exc))
deeptutor/api/routers/subagents.py
+324
View File
@@ -0,0 +1,324 @@
"""Subagent connections API.
Backs the "My Agents → connected agents" feature: detect which local agent CLIs
(Claude Code / Codex) are installed on this machine, connect one as a pointer KB
the chat composer can select, and configure the consult budget. Connections are
stored as ``type: subagent`` knowledge bases (per-user, via the KB manager), so
they ride the same selection/persistence path as the other connected KB types —
the subagent capability drives them live, nothing is indexed.
The CLIs run on the host with the host user's own credentials, so detection is
machine-global; whether a connection is usable is simply "is the CLI installed
here". If it isn't, the UI just doesn't offer it.
"""
from __future__ import annotations
import asyncio
import json
import logging
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from deeptutor.api.routers.auth import require_admin
from deeptutor.knowledge.kb_types import SUBAGENT_KB_TYPE
from deeptutor.multi_user.knowledge_access import current_kb_manager
from deeptutor.multi_user.partner_access import assert_partner_allowed, visible_partner_cards
from deeptutor.services.rag.linked_kb import assert_path_allowed
from deeptutor.services.subagent import (
PARTNER_BACKEND_KIND,
detect_all,
list_backend_kinds,
load_subagent_settings,
save_subagent_settings,
settings_from_dict,
)
logger = logging.getLogger(__name__)
router = APIRouter()
class ConnectSubagentRequest(BaseModel):
name: str
agent_kind: str
cwd: str = ""
# For the partner backend (``agent_kind == "partner"``): which partner to
# consult. Ignored by the local-CLI backends, which use ``cwd`` instead.
partner_id: str = ""
class SubagentSettingsPayload(BaseModel):
consult_budget: int | None = None
backends: dict[str, dict] | None = Field(default=None)
class SubagentMessageRequest(BaseModel):
chat_session_id: str = ""
message: str
@router.get("/detect")
async def detect_subagents():
"""Report which agent CLIs are installed and usable on this machine."""
detections = await detect_all()
return {"backends": [d.to_dict() for d in detections]}
@router.get("/backends/options")
async def backend_options():
"""Synced model + reasoning-effort options per backend (settings page sync)."""
from deeptutor.services.subagent.models import list_backend_options
options = await list_backend_options()
return {"backends": [o.to_dict() for o in options]}
@router.post("/backends/{kind}/sync")
async def sync_backend(kind: str):
"""Re-pull one backend's model catalog (the settings "sync" button).
For Claude Code this scrapes its ``/model`` TUI live and caches the result;
for Codex it re-reads the CLI-maintained cache.
"""
from deeptutor.services.subagent import get_backend
from deeptutor.services.subagent.models import sync_backend_options
backend = get_backend(kind)
if backend is None or not getattr(backend, "local_cli", True):
# Only local CLIs have a model catalog to sync; partners run their own.
raise HTTPException(status_code=400, detail=f"Unknown agent kind: {kind!r}")
options = await sync_backend_options(kind)
return options.to_dict()
@router.get("/partners")
async def list_visible_partners():
"""Partners the current user can connect & consult.
Returns every partner for an admin, or just the ones an admin has assigned
for a non-admin. The partner CRUD API (``/api/v1/partners``) stays fully
admin-gated; this is the read surface the connect flow and the partner list
page use, so a non-admin sees their assigned partners without a 403.
"""
return {"partners": visible_partner_cards()}
@router.get("/connections")
async def list_connections():
"""List the current user's connected subagents."""
manager = current_kb_manager()
connections = []
for name in manager.list_knowledge_bases():
meta = manager.get_metadata(name)
if not isinstance(meta, dict) or meta.get("type") != SUBAGENT_KB_TYPE:
continue
connections.append(
{
"name": name,
"agent_kind": meta.get("agent_kind", ""),
"cwd": meta.get("cwd", ""),
"partner_id": meta.get("partner_id", ""),
"description": meta.get("description", ""),
"created_at": meta.get("created_at"),
"updated_at": meta.get("updated_at"),
}
)
return {"connections": connections}
@router.post("/connections")
async def create_connection(payload: ConnectSubagentRequest):
"""Connect a subagent (a local CLI, or one of the user's partners) as a selectable KB.
A partner connection (``agent_kind == "partner"``) binds a ``partner_id``
instead of a working directory: consulting it opens a fresh session on that
partner, exactly as if the user started one from the partner page. Every
consult within one DeepTutor chat lands in that one partner session.
"""
name = (payload.name or "").strip()
agent_kind = (payload.agent_kind or "").strip()
if not name or not agent_kind:
raise HTTPException(status_code=400, detail="Both name and agent_kind are required.")
if agent_kind not in list_backend_kinds():
raise HTTPException(status_code=400, detail=f"Unknown agent kind: {agent_kind!r}")
resolved_cwd = ""
partner_id = ""
if agent_kind == PARTNER_BACKEND_KIND:
partner_id = (payload.partner_id or "").strip()
if not partner_id:
raise HTTPException(
status_code=400, detail="A partner_id is required to connect a partner."
)
# Partners are admin-managed, but an admin can assign one to a user via
# the grant system. An admin may connect any partner; a non-admin only a
# partner assigned to them (403 otherwise). The partner still runs in its
# own isolated scope — connecting just lets the user consult it in chat.
assert_partner_allowed(partner_id)
from deeptutor.services.partners import get_partner_manager
if not get_partner_manager().partner_exists(partner_id):
raise HTTPException(status_code=400, detail=f"No partner named {partner_id!r}.")
else:
raw_cwd = (payload.cwd or "").strip()
if raw_cwd:
try:
resolved_cwd = str(assert_path_allowed(raw_cwd))
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
try:
manager = current_kb_manager()
entry = manager.register_subagent_connection(
name, agent_kind, cwd=resolved_cwd, partner_id=partner_id
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc: # pragma: no cover - defensive
logger.error("Error connecting subagent: %s", exc)
raise HTTPException(status_code=500, detail=str(exc)) from exc
return {
"status": "connected",
"name": name,
"agent_kind": entry["agent_kind"],
"cwd": entry["cwd"],
"partner_id": entry.get("partner_id", ""),
}
@router.delete("/connections/{name}")
async def delete_connection(name: str):
"""Disconnect a subagent (removes the pointer KB; touches no files)."""
manager = current_kb_manager()
meta = manager.get_metadata(name)
if not isinstance(meta, dict) or meta.get("type") != SUBAGENT_KB_TYPE:
raise HTTPException(status_code=404, detail=f"No connected subagent named {name!r}.")
try:
manager.delete_knowledge_base(name, confirm=True)
except Exception as exc: # pragma: no cover - defensive
logger.error("Error disconnecting subagent: %s", exc)
raise HTTPException(status_code=500, detail=str(exc)) from exc
# Drop any remembered live-session ids for this connection.
from deeptutor.services.subagent.sessions import forget_connection
forget_connection(name)
return {"status": "disconnected", "name": name}
def _ndjson(obj: dict) -> str:
return json.dumps(obj, ensure_ascii=False) + "\n"
@router.post("/connections/{name}/message")
async def message_connection(name: str, payload: SubagentMessageRequest):
"""Send a message straight to a connected subagent and stream its run.
This is the sidebar's "talk to the agent directly" path: it resumes the same
live session DeepTutor consults (shared via the cross-turn registry, keyed by
chat session + connection), so the agent keeps full context. Streams the
native run as newline-delimited JSON, in the same channel shape the chat WS
uses, so the sidebar transcript renders it identically.
"""
message = (payload.message or "").strip()
if not message:
raise HTTPException(status_code=400, detail="A non-empty 'message' is required.")
manager = current_kb_manager()
meta = manager.get_metadata(name)
if not isinstance(meta, dict) or meta.get("type") != SUBAGENT_KB_TYPE:
raise HTTPException(status_code=404, detail=f"No connected subagent named {name!r}.")
from deeptutor.services.subagent import get_backend
from deeptutor.services.subagent.sessions import get_session, remember_session, session_key
kind = str(meta.get("agent_kind") or "")
cwd = str(meta.get("cwd") or "")
partner_id = str(meta.get("partner_id") or "")
backend = get_backend(kind)
if backend is None:
raise HTTPException(status_code=400, detail=f"Unknown agent kind: {kind!r}")
config = load_subagent_settings().backend(kind)
skey = session_key(payload.chat_session_id, name) if payload.chat_session_id else ""
resume_id = get_session(skey) if skey else None
async def event_stream():
queue: asyncio.Queue = asyncio.Queue()
async def on_event(event) -> None:
await queue.put(("event", event))
async def run() -> None:
try:
res = await backend.consult(
message,
on_event=on_event,
cwd=cwd or None,
session_id=resume_id,
config=config,
partner_id=partner_id or None,
)
await queue.put(("done", res))
except Exception as exc: # pragma: no cover - defensive
logger.warning("subagent message failed: %s", exc, exc_info=True)
await queue.put(("fail", str(exc)))
task = asyncio.create_task(run())
# The user's own message heads the exchange.
yield _ndjson({"channel": "user_question", "text": message})
try:
while True:
kind_, item = await queue.get()
if kind_ == "event":
line = {"channel": item.kind, "text": item.text}
merge_id = (item.meta or {}).get("merge_id")
if merge_id:
# Namespace away from the chat turn's consult merge ids.
line["merge_id"] = f"side:{merge_id}"
yield _ndjson(line)
elif kind_ == "done":
if skey and item.session_id:
remember_session(skey, item.session_id, kind=kind, cwd=cwd)
yield _ndjson(
{"done": True, "success": item.success, "session_id": item.session_id or ""}
)
break
else: # fail
yield _ndjson({"channel": "error", "text": item})
yield _ndjson({"done": True, "success": False})
break
finally:
if not task.done():
await task
return StreamingResponse(event_stream(), media_type="application/x-ndjson")
@router.get("/settings")
async def get_settings():
"""Read the consult budget and per-backend run config."""
return load_subagent_settings().to_dict()
@router.put("/settings", dependencies=[Depends(require_admin)])
async def update_settings(payload: SubagentSettingsPayload):
"""Update the subagent settings (admin-gated; deployment-wide)."""
merged = load_subagent_settings().to_dict()
if payload.consult_budget is not None:
merged["consult_budget"] = payload.consult_budget
if payload.backends is not None:
# Merge per backend (and per field) so saving one backend's settings
# never clobbers the other's or any unsent field.
backends = dict(merged.get("backends") or {})
for kind, cfg in payload.backends.items():
backends[str(kind)] = {**(backends.get(str(kind)) or {}), **(cfg or {})}
merged["backends"] = backends
settings = settings_from_dict(merged)
save_subagent_settings(settings)
return settings.to_dict()
__all__ = ["router"]
deeptutor/api/routers/system.py
+322
View File
@@ -0,0 +1,322 @@
"""
System Status API Router
Manages system status checks and model connection tests
"""
from datetime import datetime
import time
from fastapi import APIRouter
from pydantic import BaseModel
from deeptutor.multi_user.context import get_current_user
from deeptutor.services.config import resolve_search_runtime_config
from deeptutor.services.embedding import get_embedding_client, get_embedding_config
from deeptutor.services.llm import complete as llm_complete
from deeptutor.services.llm import get_llm_config, get_token_limit_kwargs
from deeptutor.services.search import web_search
router = APIRouter()
class TestResponse(BaseModel):
success: bool
message: str
model: str | None = None
response_time_ms: float | None = None
error: str | None = None
@router.get("/runtime-topology")
async def get_runtime_topology():
"""
Describe the current execution topology.
This makes the unified runtime explicit for operators and frontend code:
interactive chat turns should prefer `/api/v1/ws`, while a few routers still
exist as compatibility or isolated subsystem endpoints.
"""
return {
"primary_runtime": {
"transport": "/api/v1/ws",
"manager": "TurnRuntimeManager",
"orchestrator": "ChatOrchestrator",
"session_store": "SQLiteSessionStore",
"capability_entry": "CapabilityRegistry",
"tool_entry": "ToolRegistry",
},
"compatibility_routes": [
{"router": "chat", "mode": "legacy_adapter_target"},
{"router": "solve", "mode": "legacy_adapter_target"},
{"router": "question", "mode": "legacy_specialized"},
{"router": "research", "mode": "legacy_specialized"},
],
"isolated_subsystems": [
{"router": "co_writer", "mode": "independent_subsystem"},
{"router": "plugins_api", "mode": "playground_transport"},
],
}
@router.get("/status")
async def get_system_status():
"""
Get overall system status including backend and model configurations
Returns:
Dictionary containing status of backend, LLM, embeddings, and search
"""
result = {
"backend": {"status": "online", "timestamp": datetime.now().isoformat()},
"llm": {"status": "unknown", "model": None, "testable": True},
"embeddings": {"status": "unknown", "model": None, "testable": True},
"search": {"status": "optional", "provider": None, "testable": True},
}
# Check backend status (this endpoint itself proves backend is online)
result["backend"]["status"] = "online"
# Check LLM configuration
try:
llm_config = get_llm_config()
result["llm"]["model"] = llm_config.model
result["llm"]["status"] = "configured"
except ValueError as e:
result["llm"]["status"] = "not_configured"
result["llm"]["error"] = str(e)
except Exception as e:
result["llm"]["status"] = "error"
result["llm"]["error"] = str(e)
# Check Embeddings configuration
try:
embedding_config = get_embedding_config()
result["embeddings"]["model"] = embedding_config.model
result["embeddings"]["status"] = "configured"
except ValueError as e:
result["embeddings"]["status"] = "not_configured"
result["embeddings"]["error"] = str(e)
except Exception as e:
result["embeddings"]["status"] = "error"
result["embeddings"]["error"] = str(e)
try:
search_config = resolve_search_runtime_config()
if search_config.requested_provider:
result["search"]["provider"] = search_config.provider
if search_config.unsupported_provider:
result["search"]["status"] = "unsupported"
result["search"]["error"] = (
f"{search_config.requested_provider} is deprecated/unsupported. "
"Switch to brave/tavily/jina/searxng/duckduckgo/perplexity."
)
elif search_config.deprecated_provider:
result["search"]["status"] = "deprecated"
result["search"]["error"] = (
f"{search_config.requested_provider} is deprecated. "
"Switch to brave/tavily/jina/searxng/duckduckgo/perplexity."
)
elif search_config.missing_credentials:
result["search"]["status"] = "not_configured"
result["search"]["error"] = (
f"{search_config.requested_provider} requires api_key. "
"Set profile.api_key in Settings > Catalog."
)
elif search_config.provider == "none":
result["search"]["status"] = "disabled"
result["search"]["testable"] = False
else:
result["search"]["status"] = "configured"
if search_config.fallback_reason:
result["search"]["status"] = "fallback"
result["search"]["error"] = search_config.fallback_reason
except Exception as e:
result["search"]["status"] = "error"
result["search"]["error"] = str(e)
# Non-admin users have no need to know which model the admin configured;
# exposing the name leaks operational detail and would let curious users
# fingerprint the deployment. Strip the identifying fields.
if not get_current_user().is_admin:
for section in ("llm", "embeddings"):
result[section].pop("model", None)
result["search"].pop("provider", None)
return result
@router.post("/test/llm", response_model=TestResponse)
async def test_llm_connection():
"""
Test LLM model connection by sending a simple completion request
Returns:
Test result with success status and response time
"""
start_time = time.time()
try:
llm_config = get_llm_config()
model = llm_config.model
base_url = llm_config.base_url.rstrip("/")
# Sanitize Base URL (remove /chat/completions suffix if present)
for suffix in ["/chat/completions", "/completions"]:
if base_url.endswith(suffix):
base_url = base_url[: -len(suffix)]
# Handle API Key (inject dummy if missing for local LLMs)
api_key = llm_config.api_key
if not api_key:
api_key = "sk-no-key-required"
# Send a minimal test request with a prompt that guarantees output
test_prompt = "Say 'OK' to confirm you are working. Do not produce long output."
token_kwargs = get_token_limit_kwargs(model, max_tokens=200)
response = await llm_complete(
model=model,
prompt=test_prompt,
system_prompt="You are a helpful assistant. Respond briefly.",
binding=llm_config.binding,
api_key=api_key,
base_url=base_url,
temperature=0.1,
**token_kwargs,
)
response_time = (time.time() - start_time) * 1000
if response and len(response.strip()) > 0:
return TestResponse(
success=True,
message="LLM connection successful",
model=model,
response_time_ms=round(response_time, 2),
)
return TestResponse(
success=False,
message="LLM connection failed: Empty response",
model=model,
error="Empty response from API",
)
except ValueError as e:
return TestResponse(success=False, message=f"LLM configuration error: {e!s}", error=str(e))
except Exception as e:
response_time = (time.time() - start_time) * 1000
return TestResponse(
success=False,
message=f"LLM connection failed: {e!s}",
response_time_ms=round(response_time, 2),
error=str(e),
)
@router.post("/test/embeddings", response_model=TestResponse)
async def test_embeddings_connection():
"""
Test Embeddings model connection by sending a simple embedding request
Returns:
Test result with success status and response time
"""
start_time = time.time()
try:
embedding_config = get_embedding_config()
embedding_client = get_embedding_client()
model = embedding_config.model
binding = embedding_config.binding
# Probe a tiny batch so "connection OK" also exercises the path RAG
# uses for multi-chunk indexing.
test_texts = ["test", "retrieval batch probe"]
embeddings = await embedding_client.embed(test_texts)
response_time = (time.time() - start_time) * 1000
if (
embeddings is not None
and len(embeddings) == len(test_texts)
and all(len(vector) > 0 for vector in embeddings)
and len({len(vector) for vector in embeddings}) == 1
):
return TestResponse(
success=True,
message=f"Embeddings connection successful ({binding} provider)",
model=model,
response_time_ms=round(response_time, 2),
)
return TestResponse(
success=False,
message="Embeddings connection failed: Invalid response",
model=model,
error="Embedding response must contain one non-empty vector per input",
)
except ValueError as e:
return TestResponse(
success=False, message=f"Embeddings configuration error: {e!s}", error=str(e)
)
except Exception as e:
response_time = (time.time() - start_time) * 1000
return TestResponse(
success=False,
message=f"Embeddings connection failed: {e!s}",
response_time_ms=round(response_time, 2),
error=str(e),
)
@router.post("/test/search", response_model=TestResponse)
async def test_search_connection():
start_time = time.time()
try:
search_config = resolve_search_runtime_config()
if search_config.provider == "none":
return TestResponse(
success=False,
message="Search is disabled",
error="Set a Search provider in Settings > Catalog.",
)
if search_config.unsupported_provider:
return TestResponse(
success=False,
message=(
f"Search provider `{search_config.requested_provider}` is deprecated/unsupported."
),
error="Switch to brave/tavily/jina/searxng/duckduckgo/perplexity",
)
if search_config.missing_credentials:
return TestResponse(
success=False,
message=f"Search provider `{search_config.requested_provider}` missing credentials.",
error="Set profile.api_key in Settings > Catalog.",
)
result = web_search("DeepTutor health check", provider=search_config.provider)
response_time = (time.time() - start_time) * 1000
answer = result.get("answer") or result.get("search_results")
if not answer:
raise ValueError("Search provider returned no content")
return TestResponse(
success=True,
message="Search connection successful",
model=search_config.provider,
response_time_ms=round(response_time, 2),
)
except ValueError as e:
return TestResponse(
success=False, message=f"Search configuration error: {e!s}", error=str(e)
)
except Exception as e:
response_time = (time.time() - start_time) * 1000
return TestResponse(
success=False,
message=f"Search connection check failed: {e!s}",
response_time_ms=round(response_time, 2),
error=str(e),
)
deeptutor/api/routers/tools.py
+222
View File
@@ -0,0 +1,222 @@
"""
Tools API Router
================
Read-only listing of the chat agent's built-in tools, used by the Settings UI
to render the "Tools" sub-page. Returns each tool's definition (name,
description, parameters) alongside its bilingual prompt hints, so the frontend
can show authoritative copy without duplicating the catalog.
"""
from __future__ import annotations
import logging
from typing import Any, Literal
from fastapi import APIRouter
from pydantic import BaseModel
from deeptutor.api.routers.settings import get_enabled_optional_tools
from deeptutor.core.tool_protocol import BaseTool, ToolDefinition, ToolPromptHints
from deeptutor.i18n.metadata_i18n import tool_description_i18n
from deeptutor.tools.builtin import (
BUILTIN_TOOL_TYPES,
COMING_SOON_TOOL_TYPES,
TOOL_ALIASES,
USER_TOGGLEABLE_TOOL_NAMES,
)
logger = logging.getLogger(__name__)
router = APIRouter()
class ToolParameterPayload(BaseModel):
name: str
type: str
description: str = ""
required: bool = True
default: Any = None
enum: list[str] | None = None
class ToolAliasPayload(BaseModel):
name: str
description: str = ""
input_format: str = ""
when_to_use: str = ""
phase: str = ""
class ToolHintsPayload(BaseModel):
short_description: str = ""
when_to_use: str = ""
input_format: str = ""
guideline: str = ""
note: str = ""
phase: str = ""
aliases: list[ToolAliasPayload] = []
class BuiltinToolPayload(BaseModel):
name: str
description: str
description_i18n: dict[Literal["en", "zh"], str] = {}
parameters: list[ToolParameterPayload]
hints: dict[Literal["en", "zh"], ToolHintsPayload]
aliases: list[str] = []
# True iff the user is allowed to switch this tool on/off from the
# /settings/tools UI. Locked-on tools (auto-mounted by the chat
# pipeline under context gates) report ``False`` and the UI renders
# them as informational entries only.
toggleable: bool = False
# Whether the tool is currently on. For toggleable tools this
# reflects the user's saved preference; for locked-on tools this is
# always ``True``.
enabled: bool = True
# ``coming_soon`` tools are NOT registered with the runtime — the chat
# agent cannot invoke them. They are listed here only so the settings
# page can render a placeholder card explaining the capability is on
# the roadmap. The frontend should lock the toggle and show a badge.
coming_soon: bool = False
# The capability that owns this tool (e.g. ``solve`` / ``mastery``), or
# ``None`` for a plain system built-in. Owned tools are reused by their
# capability on top of the shared built-in surface; the settings UI groups
# them under their owner, below the built-in section.
capability: str | None = None
class ToolsListResponse(BaseModel):
tools: list[BuiltinToolPayload]
enabled_optional_tools: list[str]
def _serialise_definition(
definition: ToolDefinition,
) -> tuple[str, str, list[ToolParameterPayload]]:
params = [
ToolParameterPayload(
name=p.name,
type=p.type,
description=p.description,
required=p.required,
default=p.default,
enum=p.enum,
)
for p in definition.parameters
]
return definition.name, definition.description, params
def _serialise_hints(hints: ToolPromptHints) -> ToolHintsPayload:
return ToolHintsPayload(
short_description=hints.short_description,
when_to_use=hints.when_to_use,
input_format=hints.input_format,
guideline=hints.guideline,
note=hints.note,
phase=hints.phase,
aliases=[
ToolAliasPayload(
name=alias.name,
description=alias.description,
input_format=alias.input_format,
when_to_use=alias.when_to_use,
phase=alias.phase,
)
for alias in hints.aliases
],
)
def _collect_aliases_for(tool_name: str) -> list[str]:
return sorted(alias for alias, (target, _) in TOOL_ALIASES.items() if target == tool_name)
def _build_tool_payload(
tool: BaseTool,
*,
enabled_optional: set[str],
coming_soon: bool = False,
capability: str | None = None,
) -> BuiltinToolPayload:
name, description, parameters = _serialise_definition(tool.get_definition())
descriptions = tool_description_i18n(name, description)
toggleable = (not coming_soon) and (name in USER_TOGGLEABLE_TOOL_NAMES)
if coming_soon:
enabled = False
elif toggleable:
enabled = name in enabled_optional
else:
enabled = True
return BuiltinToolPayload(
name=name,
description=descriptions.get("en") or description,
description_i18n=descriptions,
parameters=parameters,
hints={
"en": _serialise_hints(tool.get_prompt_hints(language="en")),
"zh": _serialise_hints(tool.get_prompt_hints(language="zh")),
},
aliases=_collect_aliases_for(name),
toggleable=toggleable,
enabled=enabled,
coming_soon=coming_soon,
capability=capability,
)
@router.get("", response_model=ToolsListResponse)
async def list_builtin_tools() -> ToolsListResponse:
"""Return all built-in tools the chat agent can invoke, plus any
coming-soon placeholders for the settings page."""
from deeptutor.capabilities import capability_tool_owners
enabled_optional = set(get_enabled_optional_tools())
owners = capability_tool_owners()
payloads: list[BuiltinToolPayload] = []
for tool_type in BUILTIN_TOOL_TYPES:
try:
instance = tool_type()
payloads.append(
_build_tool_payload(
instance,
enabled_optional=enabled_optional,
capability=owners.get(instance.name),
)
)
except Exception:
logger.exception("Failed to serialise tool %s", tool_type.__name__)
for tool_type in COMING_SOON_TOOL_TYPES:
try:
instance = tool_type()
payloads.append(
_build_tool_payload(
instance,
enabled_optional=enabled_optional,
coming_soon=True,
)
)
except Exception:
logger.exception("Failed to serialise coming-soon tool %s", tool_type.__name__)
# Guard against the unlikely case of name collision (e.g. someone
# accidentally registers the same tool both as built-in and coming-soon).
seen: set[str] = set()
deduped: list[BuiltinToolPayload] = []
for payload in payloads:
if payload.name in seen:
continue
seen.add(payload.name)
deduped.append(payload)
# Toggleable tools outside the user's admin grant don't exist for them:
# hidden here so the settings page and composer match what turn_runtime
# will actually allow.
from deeptutor.multi_user.tool_access import allowed_optional_tools
allowed = allowed_optional_tools()
if allowed is not None:
deduped = [p for p in deduped if not p.toggleable or p.name in allowed]
return ToolsListResponse(
tools=deduped,
enabled_optional_tools=sorted(enabled_optional),
)
deeptutor/api/routers/unified_ws.py
+324
View File
@@ -0,0 +1,324 @@
"""
Unified WebSocket Endpoint
==========================
Single ``/api/v1/ws`` endpoint for turn-based execution and replayable streaming.
Supported client message ``type`` values:
- ``message`` / ``start_turn`` — start a new turn from a payload.
- ``subscribe_turn`` — stream events of an existing turn (with ``after_seq``).
- ``subscribe_session`` — stream events of the active turn for a session.
- ``resume_from`` — resume an in-flight turn after reconnection.
- ``unsubscribe`` — stop a previously created subscription.
- ``cancel_turn`` — cancel a running turn.
- ``submit_user_reply`` — deliver the user's reply for an ``ask_user``
paused turn so the agentic loop can resume on the same turn.
- ``regenerate`` — re-run the last user message in the given session as a
brand-new turn. Replaces the trailing assistant message (if any) and
reuses the session's stored capability/tools/preferences. Optional
``overrides`` field accepts ``capability``, ``tools``, ``knowledge_bases``,
``language``, ``config``, ``notebook_references``, ``history_references``.
Errors: ``regenerate_busy`` (another turn is running) and
``nothing_to_regenerate`` (no prior user message).
- ``check_active_turn`` — report whether the session has a live running turn;
replies with ``active_turn_info`` (``turn_id``/``status``), marking stale
persisted "running" rows as cancelled when no live execution exists.
- ``user_input`` — deliver a learner answer to the turn's StreamBus
(resolves a pending ``wait_for_input``, e.g. an ``ask_user`` pause).
"""
from __future__ import annotations
import asyncio
import json
import logging
from typing import Any
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
router = APIRouter()
logger = logging.getLogger(__name__)
@router.websocket("/ws")
async def unified_websocket(ws: WebSocket) -> None:
from deeptutor.api.routers.auth import ws_auth_failed, ws_require_auth
from deeptutor.multi_user.context import reset_current_user
user_token = await ws_require_auth(ws)
if user_token is ws_auth_failed:
return
await ws.accept()
closed = False
subscription_tasks: dict[str, asyncio.Task[None]] = {}
async def safe_send(data: dict[str, Any]) -> None:
nonlocal closed
if closed:
return
try:
# default=str so one non-serializable value inside an event can
# never poison the push channel (send_json would raise, flag the
# socket as closed, and silently freeze the stream for the user).
await ws.send_text(json.dumps(data, ensure_ascii=False, default=str))
except Exception:
closed = True
async def stop_subscription(key: str) -> None:
task = subscription_tasks.pop(key, None)
if task is None:
return
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
async def subscribe_turn(turn_id: str, after_seq: int = 0) -> None:
from deeptutor.services.session import get_turn_runtime_manager
async def _forward() -> None:
runtime = get_turn_runtime_manager()
async for event in runtime.subscribe_turn(turn_id, after_seq=after_seq):
await safe_send(event)
await stop_subscription(turn_id)
subscription_tasks[turn_id] = asyncio.create_task(_forward())
async def subscribe_session(session_id: str, after_seq: int = 0) -> None:
from deeptutor.services.session import get_turn_runtime_manager
async def _forward() -> None:
runtime = get_turn_runtime_manager()
async for event in runtime.subscribe_session(session_id, after_seq=after_seq):
await safe_send(event)
key = f"session:{session_id}"
await stop_subscription(key)
subscription_tasks[key] = asyncio.create_task(_forward())
try:
while not closed:
raw = await ws.receive_text()
try:
msg = json.loads(raw)
except json.JSONDecodeError:
await safe_send({"type": "error", "content": "Invalid JSON."})
continue
msg_type = msg.get("type")
if msg_type in {"message", "start_turn"}:
from deeptutor.services.session import get_turn_runtime_manager
runtime = get_turn_runtime_manager()
try:
_, turn = await runtime.start_turn(msg)
except RuntimeError as exc:
await safe_send(
{
"type": "error",
"source": "unified_ws",
"stage": "",
"content": str(exc),
"metadata": {"turn_terminal": True, "status": "rejected"},
"session_id": str(msg.get("session_id") or ""),
"turn_id": "",
"seq": 0,
}
)
continue
await subscribe_turn(turn["id"], after_seq=0)
continue
if msg_type == "ping":
# Client-side heartbeat. Respond with a lightweight pong so
# the client knows the socket is alive; the client never
# consumes pong as a user-visible event (see unified-ws.ts
# filter below) but does refresh ``lastReceivedAt`` from it.
await safe_send({"type": "pong"})
continue
if msg_type == "subscribe_turn":
turn_id = str(msg.get("turn_id") or "").strip()
if not turn_id:
await safe_send({"type": "error", "content": "Missing turn_id."})
continue
await subscribe_turn(turn_id, after_seq=int(msg.get("after_seq") or 0))
continue
if msg_type == "subscribe_session":
session_id = str(msg.get("session_id") or "").strip()
if not session_id:
await safe_send({"type": "error", "content": "Missing session_id."})
continue
await subscribe_session(session_id, after_seq=int(msg.get("after_seq") or 0))
continue
if msg_type == "check_active_turn":
session_id = str(msg.get("session_id") or "").strip()
if not session_id:
await safe_send({"type": "error", "content": "Missing session_id."})
continue
from deeptutor.services.session import get_turn_runtime_manager
runtime = get_turn_runtime_manager()
active_turn = await runtime.store.get_active_turn(session_id)
if active_turn:
# Verify the turn has a live execution; stale persisted
# "running" rows (e.g. after server restart) have none.
turn_id = active_turn["id"]
has_live = await runtime.has_live_execution(turn_id)
if has_live:
await safe_send(
{
"type": "active_turn_info",
"turn_id": turn_id,
"status": active_turn.get("status", "running"),
}
)
else:
# Stale turn from a previous process — mark it terminal
# so create_turn won't reject the upcoming start_turn.
await runtime.store.update_turn_status(
turn_id, "cancelled", "Stale turn after restart"
)
await safe_send(
{"type": "active_turn_info", "turn_id": "", "status": "none"}
)
else:
await safe_send({"type": "active_turn_info", "turn_id": "", "status": "none"})
continue
if msg_type == "resume_from":
turn_id = str(msg.get("turn_id") or "").strip()
if not turn_id:
await safe_send({"type": "error", "content": "Missing turn_id."})
continue
await subscribe_turn(turn_id, after_seq=int(msg.get("seq") or 0))
continue
if msg_type == "unsubscribe":
turn_id = str(msg.get("turn_id") or "").strip()
if turn_id:
await stop_subscription(turn_id)
session_id = str(msg.get("session_id") or "").strip()
if session_id:
await stop_subscription(f"session:{session_id}")
continue
if msg_type == "cancel_turn":
turn_id = str(msg.get("turn_id") or "").strip()
if not turn_id:
await safe_send({"type": "error", "content": "Missing turn_id."})
continue
from deeptutor.services.session import get_turn_runtime_manager
runtime = get_turn_runtime_manager()
cancelled = await runtime.cancel_turn(turn_id)
if not cancelled:
await safe_send({"type": "error", "content": f"Turn not found: {turn_id}"})
continue
if msg_type == "submit_user_reply":
turn_id = str(msg.get("turn_id") or "").strip()
if not turn_id:
await safe_send({"type": "error", "content": "Missing turn_id."})
continue
# Accept either the legacy ``text`` (single free-form
# reply) or the v2 ``answers`` (list of {questionId, text}
# pairs). Empty text is allowed (lets the user signal "I
# have no answer" without typing).
text = msg.get("text")
text_str = str(text) if text is not None else None
answers_raw = msg.get("answers")
answers: list[dict[str, Any]] | None = None
if isinstance(answers_raw, list):
cleaned: list[dict[str, Any]] = []
for entry in answers_raw:
if not isinstance(entry, dict):
continue
qid = str(entry.get("questionId") or entry.get("id") or "").strip()
if not qid:
continue
cleaned.append({"questionId": qid, "text": str(entry.get("text") or "")})
answers = cleaned or None
from deeptutor.services.session import get_turn_runtime_manager
runtime = get_turn_runtime_manager()
accepted = await runtime.submit_user_reply(turn_id, text=text_str, answers=answers)
if not accepted:
await safe_send(
{
"type": "error",
"content": (f"Turn {turn_id} is not awaiting a user reply."),
}
)
continue
if msg_type == "regenerate":
session_id = str(msg.get("session_id") or "").strip()
if not session_id:
await safe_send({"type": "error", "content": "Missing session_id."})
continue
from deeptutor.services.session import get_turn_runtime_manager
runtime = get_turn_runtime_manager()
overrides = msg.get("overrides") if isinstance(msg.get("overrides"), dict) else None
try:
_, turn = await runtime.regenerate_last_turn(
session_id,
overrides=overrides,
)
except RuntimeError as exc:
await safe_send(
{
"type": "error",
"source": "unified_ws",
"stage": "",
"content": str(exc),
"metadata": {
"turn_terminal": True,
"status": "rejected",
"reason": str(exc),
},
"session_id": session_id,
"turn_id": "",
"seq": 0,
}
)
continue
await subscribe_turn(turn["id"], after_seq=0)
continue
if msg_type == "user_input":
turn_id = str(msg.get("turn_id") or "").strip()
if not turn_id:
await safe_send({"type": "error", "content": "Missing turn_id for user_input."})
continue
from deeptutor.core.stream_bus import get_bus
bus = get_bus(turn_id)
if bus is None:
await safe_send(
{"type": "error", "content": f"No active bus for turn: {turn_id}"}
)
continue
bus.submit_input(str(msg.get("content") or ""))
continue
await safe_send({"type": "error", "content": f"Unknown type: {msg_type}"})
except WebSocketDisconnect:
logger.debug("Client disconnected from /ws")
except Exception as exc:
logger.error("Unified WS error: %s", exc, exc_info=True)
await safe_send({"type": "error", "content": str(exc)})
finally:
closed = True
for key in list(subscription_tasks.keys()):
await stop_subscription(key)
if user_token is not None:
reset_current_user(user_token)
deeptutor/api/routers/voice.py
+130
View File
@@ -0,0 +1,130 @@
"""Voice endpoints — text-to-speech and speech-to-text.
These are thin HTTP surfaces over :mod:`deeptutor.services.voice`. Config comes
from the admin-managed model catalog (``services.tts`` / ``services.stt``), so
voice is shared infrastructure like embedding/search — any authenticated user
may call it; it is not gated by per-user LLM grants.
"""
from __future__ import annotations
import io
import logging
import wave
from fastapi import APIRouter, File, Form, HTTPException, Response, UploadFile, status
from pydantic import BaseModel, Field
from deeptutor.services.voice import (
VoiceProviderError,
synthesize_speech,
transcribe_audio,
)
logger = logging.getLogger(__name__)
router = APIRouter()
# Guard against pathological uploads (the providers cap well below this anyway).
_MAX_AUDIO_BYTES = 25 * 1024 * 1024 # 25 MB, matching OpenAI's limit.
_DEFAULT_PCM_SAMPLE_RATE = 24_000
_DEFAULT_PCM_CHANNELS = 1
_PCM16_SAMPLE_WIDTH = 2
class TTSRequest(BaseModel):
"""Text-to-speech request body."""
text: str = Field(..., min_length=1)
voice: str | None = None
format: str | None = None
def _parse_pcm_content_type(content_type: str) -> tuple[int, int] | None:
"""Return ``(sample_rate, channels)`` when a provider sent raw PCM audio."""
media_type, *params = (content_type or "").split(";")
if media_type.strip().lower() not in {"audio/pcm", "audio/x-pcm", "audio/l16"}:
return None
sample_rate = _DEFAULT_PCM_SAMPLE_RATE
channels = _DEFAULT_PCM_CHANNELS
for item in params:
key, sep, value = item.strip().partition("=")
if not sep:
continue
key = key.strip().lower()
value = value.strip().strip('"')
try:
parsed = int(value)
except ValueError:
continue
if key in {"rate", "sample-rate", "samplerate"} and parsed > 0:
sample_rate = parsed
elif key in {"channels", "channel"} and parsed > 0:
channels = parsed
return sample_rate, channels
def _pcm16_to_wav(audio: bytes, *, sample_rate: int, channels: int) -> bytes:
"""Wrap provider PCM16 bytes in a WAV container browsers can play."""
buffer = io.BytesIO()
with wave.open(buffer, "wb") as wav:
wav.setnchannels(channels)
wav.setsampwidth(_PCM16_SAMPLE_WIDTH)
wav.setframerate(sample_rate)
wav.writeframes(audio)
return buffer.getvalue()
@router.post("/tts")
async def text_to_speech(payload: TTSRequest) -> Response:
"""Synthesize ``text`` to audio using the active TTS provider."""
try:
audio, content_type = await synthesize_speech(
payload.text,
voice=payload.voice,
response_format=payload.format,
)
except ValueError as exc: # missing/invalid configuration
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
except VoiceProviderError as exc:
logger.warning("TTS provider error: %s", exc)
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
pcm_info = _parse_pcm_content_type(content_type)
if pcm_info:
sample_rate, channels = pcm_info
audio = _pcm16_to_wav(audio, sample_rate=sample_rate, channels=channels)
content_type = "audio/wav"
return Response(
content=audio,
media_type=content_type,
headers={"Cache-Control": "no-store"},
)
@router.post("/stt")
async def speech_to_text(
file: UploadFile = File(...),
language: str | None = Form(default=None),
) -> dict[str, str]:
"""Transcribe an uploaded audio clip using the active STT provider."""
audio = await file.read()
if not audio:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Empty audio upload.")
if len(audio) > _MAX_AUDIO_BYTES:
raise HTTPException(
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
detail="Audio exceeds the 25 MB limit.",
)
try:
text = await transcribe_audio(
audio,
filename=file.filename or "audio.webm",
content_type=file.content_type or "application/octet-stream",
language=language,
)
except ValueError as exc:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
except VoiceProviderError as exc:
logger.warning("STT provider error: %s", exc)
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
return {"text": text}
deeptutor/api/run_server.py
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python
"""
Uvicorn Server Startup Script
Uses Python API instead of command line to avoid Windows path parsing issues.
"""
import asyncio
import os
from pathlib import Path
import sys
from deeptutor.runtime.home import get_runtime_home
# Windows: uvicorn defaults to SelectorEventLoop which does not support
# asyncio.create_subprocess_exec. Switch to ProactorEventLoop so that
# child-process APIs (used by Math Animator renderer, etc.) work correctly.
if sys.platform == "win32":
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
import uvicorn
# Force unbuffered output
os.environ["PYTHONUNBUFFERED"] = "1"
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(line_buffering=True, encoding="utf-8", errors="replace")
if hasattr(sys.stderr, "reconfigure"):
sys.stderr.reconfigure(line_buffering=True, encoding="utf-8", errors="replace")
def main() -> None:
# Runtime workspace root owns data/user/settings and generated outputs.
project_root = get_runtime_home()
os.chdir(str(project_root))
# Get port from configuration
from deeptutor.logging import configure_logging
from deeptutor.runtime.mode import RunMode, set_mode
from deeptutor.services.setup import get_backend_port
set_mode(RunMode.SERVER)
configure_logging()
backend_port = get_backend_port(project_root)
# Configure reload_excludes to skip directories that shouldn't trigger reloads
# Use absolute paths to ensure they're properly resolved
reload_excludes = [
str(project_root / "venv"), # Virtual environment
str(project_root / ".venv"), # Virtual environment (alternative name)
str(project_root / "data"), # Data directory (includes knowledge_bases, user data, logs)
str(project_root / "node_modules"), # Node modules (if any at root)
str(project_root / "web" / "node_modules"), # Web node modules
str(project_root / "web" / ".next"), # Next.js build
str(project_root / ".git"), # Git directory
str(project_root / "scripts"), # Scripts directory - don't reload on launcher changes
]
# Filter out non-existent directories to avoid warnings
reload_excludes = [d for d in reload_excludes if Path(d).exists()]
# Start uvicorn server with reload enabled
uvicorn.run(
"deeptutor.api.main:app",
host="0.0.0.0",
port=backend_port,
reload=True,
reload_excludes=reload_excludes,
log_level="info",
access_log=False,
)
if __name__ == "__main__":
main()
deeptutor/api/utils/progress_broadcaster.py
+73
View File
@@ -0,0 +1,73 @@
"""
Progress Broadcaster - Manages WebSocket broadcasting of knowledge base progress
"""
import asyncio
import logging
from typing import Optional
from fastapi import WebSocket
logger = logging.getLogger(__name__)
class ProgressBroadcaster:
"""Manages WebSocket broadcasting of knowledge base progress"""
_instance: Optional["ProgressBroadcaster"] = None
_connections: dict[str, set[WebSocket]] = {} # kb_name -> Set[WebSocket]
_lock = asyncio.Lock()
@classmethod
def get_instance(cls) -> "ProgressBroadcaster":
"""Get singleton instance"""
if cls._instance is None:
cls._instance = cls()
return cls._instance
async def connect(self, kb_name: str, websocket: WebSocket):
"""Connect WebSocket to specified knowledge base"""
async with self._lock:
if kb_name not in self._connections:
self._connections[kb_name] = set()
self._connections[kb_name].add(websocket)
logger.debug(
f"Connected WebSocket for KB '{kb_name}' (total: {len(self._connections[kb_name])})"
)
async def disconnect(self, kb_name: str, websocket: WebSocket):
"""Disconnect WebSocket connection"""
async with self._lock:
if kb_name in self._connections:
self._connections[kb_name].discard(websocket)
if not self._connections[kb_name]:
del self._connections[kb_name]
logger.debug(f"Disconnected WebSocket for KB '{kb_name}'")
async def broadcast(self, kb_name: str, progress: dict):
"""Broadcast progress update to all WebSocket connections for specified knowledge base"""
async with self._lock:
if kb_name not in self._connections:
return
# Create list of connections to remove (closed connections)
to_remove = []
for websocket in self._connections[kb_name]:
try:
await websocket.send_json({"type": "progress", "data": progress})
except Exception as e:
# Connection closed or error, mark for removal
logger.debug(f"Error sending to WebSocket for KB '{kb_name}': {e}")
to_remove.append(websocket)
# Remove closed connections
for ws in to_remove:
self._connections[kb_name].discard(ws)
if not self._connections[kb_name]:
del self._connections[kb_name]
def get_connection_count(self, kb_name: str) -> int:
"""Get connection count for specified knowledge base"""
return len(self._connections.get(kb_name, set()))
deeptutor/api/utils/task_id_manager.py
+103
View File
@@ -0,0 +1,103 @@
"""
Task ID Manager - Assigns unique IDs to each background task
"""
from datetime import datetime, timedelta
import logging
import threading
from typing import Optional
import uuid
logger = logging.getLogger(__name__)
class TaskIDManager:
"""Singleton class for managing task IDs"""
_instance: Optional["TaskIDManager"] = None
_lock = threading.Lock()
_task_ids: dict[str, str] = {} # task_key -> task_id
_task_metadata: dict[str, dict] = {} # task_id -> metadata
@classmethod
def get_instance(cls) -> "TaskIDManager":
"""Get singleton instance"""
if cls._instance is None:
with cls._lock:
if cls._instance is None:
cls._instance = cls()
return cls._instance
def generate_task_id(self, task_type: str, task_key: str) -> str:
"""
Generate unique ID for task
Args:
task_type: Task type (e.g., 'kb_init', 'kb_upload', 'question_gen', 'solve', 'research')
task_key: Task unique identifier (e.g., knowledge base name, question ID, etc.)
Returns:
Task ID (format: {task_type}_{timestamp}_{uuid})
"""
with self._lock:
# If task already exists, return existing ID
if task_key in self._task_ids:
return self._task_ids[task_key]
# Generate new ID
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
unique_id = str(uuid.uuid4())[:8]
task_id = f"{task_type}_{timestamp}_{unique_id}"
# Save mapping and metadata
self._task_ids[task_key] = task_id
self._task_metadata[task_id] = {
"task_type": task_type,
"task_key": task_key,
"created_at": datetime.now().isoformat(),
"status": "running",
}
return task_id
def get_task_id(self, task_key: str) -> str | None:
"""Get task ID"""
with self._lock:
return self._task_ids.get(task_key)
def update_task_status(self, task_id: str, status: str, **kwargs):
"""Update task status"""
with self._lock:
if task_id in self._task_metadata:
self._task_metadata[task_id]["status"] = status
self._task_metadata[task_id].update(kwargs)
if status in ["completed", "error", "cancelled"]:
self._task_metadata[task_id]["finished_at"] = datetime.now().isoformat()
def get_task_metadata(self, task_id: str) -> dict | None:
"""Get task metadata"""
with self._lock:
return self._task_metadata.get(task_id, {}).copy()
def cleanup_old_tasks(self, max_age_hours: int = 24):
"""Clean up old tasks (completed tasks older than specified hours)"""
with self._lock:
cutoff = datetime.now() - timedelta(hours=max_age_hours)
to_remove = []
for task_id, metadata in self._task_metadata.items():
if metadata.get("status") in ["completed", "error", "cancelled"]:
finished_at = metadata.get("finished_at")
if finished_at:
try:
finished_time = datetime.fromisoformat(finished_at)
if finished_time < cutoff:
to_remove.append(task_id)
except Exception:
logger.warning("Failed to parse finished_at for task %s", task_id)
for task_id in to_remove:
metadata = self._task_metadata.pop(task_id, {})
task_key = metadata.get("task_key")
if task_key:
self._task_ids.pop(task_key, None)
deeptutor/api/utils/task_log_stream.py
+202
View File
@@ -0,0 +1,202 @@
import asyncio
from collections import deque
from collections.abc import AsyncGenerator
import contextlib
import importlib
import json
import logging
import threading
import time
from typing import Any
from deeptutor.logging import (
ProcessLogEvent,
bind_log_context,
capture_process_logs,
current_log_context,
)
def _format_sse(event: str, payload: dict[str, Any]) -> str:
return f"event: {event}\ndata: {json.dumps(payload, ensure_ascii=False, default=str)}\n\n"
class KnowledgeTaskStreamManager:
_instance: "KnowledgeTaskStreamManager | None" = None
_instance_lock = threading.Lock()
def __init__(self):
self._lock = threading.Lock()
self._buffers: dict[str, deque[dict[str, Any]]] = {}
self._subscribers: dict[str, list[tuple[asyncio.Queue, asyncio.AbstractEventLoop]]] = {}
@classmethod
def get_instance(cls) -> "KnowledgeTaskStreamManager":
if cls._instance is None:
with cls._instance_lock:
if cls._instance is None:
cls._instance = cls()
return cls._instance
def ensure_task(self, task_id: str):
with self._lock:
self._buffers.setdefault(task_id, deque(maxlen=500))
self._subscribers.setdefault(task_id, [])
def emit(self, task_id: str, event: str, payload: dict[str, Any]):
event_payload = {"event": event, "payload": payload}
with self._lock:
self._buffers.setdefault(task_id, deque(maxlen=500)).append(event_payload)
subscribers = list(self._subscribers.get(task_id, []))
for queue, loop in subscribers:
try:
loop.call_soon_threadsafe(self._queue_event, queue, event_payload)
except RuntimeError:
continue
def emit_process_log(self, task_id: str, event: ProcessLogEvent):
payload = event.to_dict()
payload.setdefault("context", {})["task_id"] = task_id
self.emit(task_id, "process_log", payload)
def emit_log(self, task_id: str, line: str):
event = ProcessLogEvent(
level="INFO",
message=line,
logger="deeptutor.knowledge.task",
timestamp=time.time(),
context={"task_id": task_id, "capability": "knowledge", "sink": "ui"},
)
self.emit_process_log(task_id, event)
def emit_complete(self, task_id: str, detail: str = "Task completed"):
self.emit(task_id, "complete", {"detail": detail, "task_id": task_id})
def emit_failed(self, task_id: str, detail: str, *, details: str | None = None):
payload: dict[str, Any] = {"detail": detail, "task_id": task_id}
if details:
payload["details"] = details
self.emit(task_id, "failed", payload)
def subscribe(
self, task_id: str
) -> tuple[asyncio.Queue[dict[str, Any]], list[dict[str, Any]], asyncio.AbstractEventLoop]:
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=200)
loop = asyncio.get_running_loop()
with self._lock:
self._buffers.setdefault(task_id, deque(maxlen=500))
self._subscribers.setdefault(task_id, []).append((queue, loop))
backlog = list(self._buffers[task_id])
return queue, backlog, loop
def unsubscribe(
self, task_id: str, queue: asyncio.Queue[dict[str, Any]], loop: asyncio.AbstractEventLoop
):
with self._lock:
subscribers = self._subscribers.get(task_id, [])
self._subscribers[task_id] = [
(subscriber_queue, subscriber_loop)
for subscriber_queue, subscriber_loop in subscribers
if subscriber_queue is not queue or subscriber_loop is not loop
]
async def stream(self, task_id: str) -> AsyncGenerator[str, None]:
queue, backlog, loop = self.subscribe(task_id)
try:
for item in backlog:
yield _format_sse(item["event"], item["payload"])
if backlog and backlog[-1]["event"] in {"complete", "failed"}:
return
while True:
item = await queue.get()
yield _format_sse(item["event"], item["payload"])
if item["event"] in {"complete", "failed"}:
break
finally:
self.unsubscribe(task_id, queue, loop)
@staticmethod
def _queue_event(queue: asyncio.Queue[dict[str, Any]], payload: dict[str, Any]):
try:
queue.put_nowait(payload)
except asyncio.QueueFull:
pass
class _TaskScopedLogHandler(logging.Handler):
"""Forward non-propagating library logs into one knowledge task stream."""
def __init__(self, task_id: str, manager: KnowledgeTaskStreamManager) -> None:
super().__init__(logging.INFO)
self._task_id = task_id
self._manager = manager
def emit(self, record: logging.LogRecord) -> None:
try:
context = current_log_context()
record_task_id = context.get("task_id")
if record_task_id and record_task_id != self._task_id:
return
context.setdefault("task_id", self._task_id)
context.setdefault("capability", "knowledge")
context.setdefault("sink", "ui")
self._manager.emit_process_log(
self._task_id,
ProcessLogEvent(
level=record.levelname,
message=record.getMessage(),
logger=record.name,
timestamp=record.created,
context=context,
),
)
except Exception:
self.handleError(record)
@contextlib.contextmanager
def _capture_non_propagating_task_logs(task_id: str, manager: KnowledgeTaskStreamManager):
"""Capture library loggers that intentionally do not propagate to root."""
logger_names = ("lightrag", "graphrag", "graphrag_llm")
handlers: list[tuple[logging.Logger, _TaskScopedLogHandler]] = []
for logger_name in logger_names:
if logger_name == "lightrag":
with contextlib.suppress(Exception):
importlib.import_module("lightrag.utils")
source_logger = logging.getLogger(logger_name)
if source_logger.propagate:
continue
handler = _TaskScopedLogHandler(task_id, manager)
source_logger.addHandler(handler)
handlers.append((source_logger, handler))
try:
yield
finally:
for source_logger, handler in handlers:
if handler in source_logger.handlers:
source_logger.removeHandler(handler)
handler.close()
@contextlib.contextmanager
def capture_task_logs(task_id: str):
"""Forward all logs bound to ``task_id`` into the task's SSE stream."""
manager = KnowledgeTaskStreamManager.get_instance()
manager.ensure_task(task_id)
def emit(event: ProcessLogEvent) -> None:
manager.emit_process_log(task_id, event)
with bind_log_context(task_id=task_id, capability="knowledge", sink="ui"):
with capture_process_logs(emit, task_id=task_id):
with _capture_non_propagating_task_logs(task_id, manager):
yield
def get_task_stream_manager() -> KnowledgeTaskStreamManager:
return KnowledgeTaskStreamManager.get_instance()
deeptutor/api/utils/tool_options.py
+87
View File
@@ -0,0 +1,87 @@
"""Configurable-tool surface shared by the partners and multi-user admin APIs.
``tools`` mirrors the user-toggleable system tools (the same pool the chat
composer / settings expose); ``builtin_tools`` lists the auto-mounted built-in
tools (rag / read_memory / web_fetch / …) a partner owner can selectively
allow or deny; ``mcp_tools`` lists every configured MCP tool that a whitelist
(partner config or user grant) could allow.
"""
from __future__ import annotations
import logging
from typing import Any
from deeptutor.core.i18n import current_language
from deeptutor.i18n.metadata_i18n import localized_description, tool_description_i18n
logger = logging.getLogger(__name__)
async def build_tool_options(
*, exclude_builtin: set[str] | None = None
) -> dict[str, list[dict[str, Any]]]:
"""Build the configurable-tool surface.
``exclude_builtin`` drops built-in tools from the ``builtin_tools`` list —
the partners API passes ``{"read_memory", "write_memory"}`` because partners
use the mandatory ``partner_*`` memory tools instead and cannot configure
chat's memory tools.
"""
from deeptutor.agents._shared.tool_composition import default_optional_tools
from deeptutor.runtime.registry.tool_registry import get_tool_registry
from deeptutor.tools.builtin import CONFIGURABLE_BUILTIN_TOOL_NAMES
exclude = exclude_builtin or set()
registry = get_tool_registry()
language = current_language()
try:
from deeptutor.services.mcp import get_mcp_manager
await get_mcp_manager().ensure_started()
except Exception:
logger.debug("MCP manager unavailable for tool options", exc_info=True)
def _describe(name: str) -> dict[str, Any]:
tool = registry.get(name)
description = ""
if tool is not None:
try:
description = tool.get_definition().description or ""
except Exception:
description = ""
descriptions = tool_description_i18n(name, description)
return {
"name": name,
"description": localized_description(descriptions, language),
"description_i18n": descriptions,
}
tools: list[dict[str, Any]] = [_describe(name) for name in default_optional_tools()]
builtin_tools: list[dict[str, Any]] = [
_describe(name) for name in CONFIGURABLE_BUILTIN_TOOL_NAMES if name not in exclude
]
mcp_tools: list[dict[str, Any]] = []
for tool in registry.deferred_tools():
try:
definition = tool.get_definition()
except Exception:
continue
mcp_tools.append(
{
"name": definition.name,
"server": str(getattr(tool, "server_name", "") or ""),
"description": definition.description or "",
"description_i18n": {
"en": definition.description or "",
"zh": definition.description or "",
},
}
)
return {"tools": tools, "builtin_tools": builtin_tools, "mcp_tools": mcp_tools}
__all__ = ["build_tool_options"]