-> split header and data
header, _, b64data = url.partition(",")
media_type = header.split(";")[0].replace("data:", "") or "image/jpeg"
anthropic_parts.append(
{
"type": "image",
"source": {
"type": "base64",
"media_type": media_type,
"data": b64data,
},
}
)
else:
# Remote URL -- Anthropic supports the url source type natively.
# See: https://docs.anthropic.com/en/docs/build-with-claude/vision#url-based-images
anthropic_parts.append(
{
"type": "image",
"source": {
"type": "url",
"url": url,
},
}
)
elif part.get("type") == "input_document":
# Studio's normalised PDF/doc type (file_data data-URI or
# file_url) -> Anthropic's native `document` block.
url = part.get("file_url") or ""
data_uri = part.get("file_data") or ""
title = part.get("filename")
# Treat a "data:" URI with no base64 payload as missing so
# the file_url branch can take over (matches OpenAI side).
data_uri_valid = False
b64data = ""
header = ""
if data_uri.startswith("data:"):
header, _, b64data = data_uri.partition(",")
data_uri_valid = bool(b64data.strip())
if data_uri_valid:
media_type = (
part.get("media_type")
or header.split(";")[0].replace("data:", "")
or "application/pdf"
)
doc_block: dict[str, Any] = {
"type": "document",
"source": {
"type": "base64",
"media_type": media_type,
"data": b64data,
},
# Opt into Anthropic's natural-citation
# pipeline; without this no citations_delta
# events fire. See
# https://platform.claude.com/docs/en/build-with-claude/citations
"citations": {"enabled": True},
}
if title:
doc_block["title"] = title
anthropic_parts.append(doc_block)
elif url:
doc_block = {
"type": "document",
"source": {
"type": "url",
"url": url,
},
"citations": {"enabled": True},
}
if title:
doc_block["title"] = title
anthropic_parts.append(doc_block)
# Assistant tool_calls -> Anthropic tool_use blocks appended to
# the same message. The native Messages API doesn't accept
# OpenAI's top-level `tool_calls` field; the call lives inside a
# content block `{type:"tool_use", id, name, input}`.
if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list):
for _tc in msg["tool_calls"]:
if not isinstance(_tc, dict):
continue
_fn = _tc.get("function") or {}
if not isinstance(_fn, dict) or not _fn.get("name"):
continue
_raw = _fn.get("arguments") or "{}"
try:
_input = _json.loads(_raw) if isinstance(_raw, str) else _raw
except Exception:
_input = {"_raw": _raw}
if not isinstance(_input, dict):
_input = {"value": _input}
anthropic_parts.append(
{
"type": "tool_use",
"id": _tc.get("id") or f"toolu_{time.time_ns()}",
"name": _fn["name"],
"input": _input,
}
)
# Skip whole-message append when nothing usable survived. An
# empty content array (e.g. user dropped only an unparseable
# `input_document`) would 400 with "messages.N.content: at
# least one block is required".
if anthropic_parts:
filtered.append({"role": msg["role"], "content": anthropic_parts})
else:
# role="tool" follow-up -> Anthropic native tool_result block
# on a `user` message. The OpenAI shape (role=tool,
# content=string, tool_call_id) is not a valid Anthropic role.
if msg.get("role") == "tool":
_tr_id = msg.get("tool_call_id") or ""
_tr_content = msg.get("content")
if _tr_content is None:
_tr_content = ""
filtered.append(
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": _tr_id,
"content": (
_tr_content
if isinstance(_tr_content, str)
else _json.dumps(_tr_content)
),
}
],
}
)
continue
# Assistant turn whose content is a plain string but also
# carries OpenAI `tool_calls`: convert into a content-array
# message with a text block + tool_use blocks. Without this,
# the top-level tool_calls leaks through unchanged.
if (
msg.get("role") == "assistant"
and isinstance(msg.get("tool_calls"), list)
and msg["tool_calls"]
):
_text_content = msg.get("content")
_blocks: list[dict[str, Any]] = []
if isinstance(_text_content, str) and _text_content:
_blocks.append({"type": "text", "text": _text_content})
for _tc in msg["tool_calls"]:
if not isinstance(_tc, dict):
continue
_fn = _tc.get("function") or {}
if not isinstance(_fn, dict) or not _fn.get("name"):
continue
_raw = _fn.get("arguments") or "{}"
try:
_input = _json.loads(_raw) if isinstance(_raw, str) else _raw
except Exception:
_input = {"_raw": _raw}
if not isinstance(_input, dict):
_input = {"value": _input}
_blocks.append(
{
"type": "tool_use",
"id": _tc.get("id") or f"toolu_{time.time_ns()}",
"name": _fn["name"],
"input": _input,
}
)
if _blocks:
filtered.append({"role": "assistant", "content": _blocks})
continue
filtered.append(msg)
# Claude 4.7 removed temperature/top_p/top_k entirely (400 "deprecated
# for this model"). Latch the match and reuse it wherever those are set,
# including the thinking-mode override below that used to force temp=1.
sampling_removed = bool(_ANTHROPIC_4_7_SAMPLING_REMOVED.match(model))
body: dict[str, Any] = {
"model": model,
"messages": filtered,
"max_tokens": max_tokens or 1024, # required by Anthropic
"stream": True,
}
if not sampling_removed:
body["temperature"] = temperature
if top_k is not None and top_k > 0 and not sampling_removed:
body["top_k"] = top_k
# Anthropic caches a prefix only with a cache_control marker. Treat None
# as True (frontend default); pass False to opt out.
prompt_caching_enabled = enable_prompt_caching is not False
# Optional 1h cache TTL is GA (no beta header). 1h writes cost 2x vs 5m's
# 1.25x but reads are 0.1x for both, so 1h wins after one extra hit.
cache_marker: dict[str, Any] = {"type": "ephemeral"}
if prompt_cache_ttl in ("5m", "1h"):
cache_marker["ttl"] = prompt_cache_ttl
if system:
if prompt_caching_enabled:
# System is the most stable cross-turn prefix; own breakpoint.
body["system"] = [
{
"type": "text",
"text": system,
"cache_control": dict(cache_marker),
}
]
else:
body["system"] = system
if prompt_caching_enabled and filtered:
# Second breakpoint on the latest message so turn N+1 rehydrates
# through turn N from cache. Covers the case where the system prompt
# is below Anthropic's ~1024-token cache floor. (Max 4 breakpoints;
# we use 2: system + tail.)
last_msg = filtered[-1]
content = last_msg.get("content")
if isinstance(content, str):
last_msg["content"] = [
{
"type": "text",
"text": content,
"cache_control": dict(cache_marker),
}
]
elif isinstance(content, list) and content:
# Rebuild the tail (don't mutate the caller's list) with
# cache_control on the final block.
head = list(content[:-1])
tail = content[-1]
if isinstance(tail, dict):
head.append({**tail, "cache_control": dict(cache_marker)})
else:
head.append(tail)
last_msg["content"] = head
thinking_spec = _anthropic_thinking_spec(model)
allowed_efforts = (
thinking_spec.efforts if thinking_spec else ("none", "low", "medium", "high")
)
effort = reasoning_effort if reasoning_effort in allowed_efforts else None
# Claude 4.6 takes top-tier adaptive effort as "max" only ("xhigh" is
# 4.7-only), so map "xhigh" -> "max" for 4.6 outbound requests.
if effort == "xhigh" and model.startswith(("claude-opus-4-6", "claude-sonnet-4-6")):
effort = "max"
if effort is None:
if enable_thinking is False:
effort = "none"
elif enable_thinking is True:
effort = "medium"
# Normalize one semantic Thinking control into Anthropic's two model-era
# APIs: adaptive effort on Claude 4.6/4.7, manual budget_tokens on 4.5.
if effort and effort != "none":
# Anthropic rejects top_k whenever thinking is enabled.
body.pop("top_k", None)
# 4.5/4.6 require temperature=1 with thinking and forbid top_p in the
# same request; 4.7 removed temperature entirely (any value 400s), so
# skip the override there.
if not sampling_removed:
body["temperature"] = 1
body.pop("top_p", None)
if thinking_spec and thinking_spec.kind == "adaptive":
# Force display="summarized": it defaults to "omitted" on Opus 4.7,
# which emits an empty thinking block and leaves the panel blank.
# Harmless no-op on 4.6.
body["thinking"] = {"type": "adaptive", "display": "summarized"}
# Adaptive effort lives under `output_config.effort`, not top-level
# (top-level 400s "Extra inputs are not permitted"). Allowed:
# low|medium|high|xhigh|max. See
# https://platform.claude.com/docs/en/api/messages
body["output_config"] = {"effort": effort}
elif thinking_spec and thinking_spec.kind == "manual":
budget_tokens = {"low": 1024, "medium": 2048, "high": 4096}[effort]
body["thinking"] = {
"type": "enabled",
"budget_tokens": budget_tokens,
}
# Anthropic requires max_tokens to be strictly greater than
# thinking.budget_tokens on the manual-thinking path.
if body.get("max_tokens", 0) <= budget_tokens:
body["max_tokens"] = budget_tokens + 1024
# tool_choice="none" or pinned-function suppresses hosted tools so a
# stale UI toggle can't fire server-side search/code-exec.
_anthropic_tool_choice_disabled = (
isinstance(tool_choice, str) and tool_choice.strip().lower() == "none"
)
_anthropic_tool_choice_forced_function = (
isinstance(tool_choice, dict)
and tool_choice.get("type") == "function"
and isinstance(tool_choice.get("function"), dict)
and bool(tool_choice["function"].get("name"))
)
_anthropic_hosted_builtins_allowed = (
not _anthropic_tool_choice_disabled and not _anthropic_tool_choice_forced_function
)
# Anthropic web_search (date-pinned per model family).
# https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool
if _anthropic_hosted_builtins_allowed and enabled_tools and "web_search" in enabled_tools:
anthropic_tools = list(body.get("tools") or [])
anthropic_tools.append(
{
"type": _anthropic_web_search_version(model),
"name": "web_search",
"max_uses": 5,
}
)
body["tools"] = anthropic_tools
# Anthropic web_fetch: only URLs already in conversation. Date-pinned.
# https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-fetch-tool
web_fetch_enabled = bool(
_anthropic_hosted_builtins_allowed and enabled_tools and "web_fetch" in enabled_tools
)
if web_fetch_enabled:
anthropic_tools = list(body.get("tools") or [])
anthropic_tools.append(
{
"type": _anthropic_web_fetch_version(model),
"name": "web_fetch",
"max_uses": 5,
}
)
body["tools"] = anthropic_tools
# Anthropic server-side code execution (date-pinned type per model, both
# unlocked by the same beta header set below). See
# https://platform.claude.com/docs/en/agents-and-tools/tool-use/code-execution-tool
code_execution_enabled = bool(
_anthropic_hosted_builtins_allowed
and enabled_tools
and "code_execution" in enabled_tools
)
if code_execution_enabled:
anthropic_tools = list(body.get("tools") or [])
anthropic_tools.append(
{
"type": _anthropic_code_execution_version(model),
"name": "code_execution",
}
)
body["tools"] = anthropic_tools
# Reuse the thread's prior container so filesystem state persists.
# Stale ids 4xx and clear via container_invalidated.
if anthropic_code_exec_container_id:
body["container"] = anthropic_code_exec_container_id
# Server-side compaction (beta `compact-2026-01-12`). Clamps below-min
# thresholds to 50K so the request doesn't 400.
# https://platform.claude.com/docs/en/build-with-claude/compaction
compaction_active = (
compaction_threshold is not None
and compaction_threshold > 0
and _anthropic_supports_compaction(model)
)
if compaction_active and compaction_threshold is not None:
trigger_value = max(
int(compaction_threshold),
_ANTHROPIC_COMPACTION_MIN,
)
body["context_management"] = {
"edits": [
{
"type": _ANTHROPIC_COMPACTION_TYPE,
"trigger": {
"type": "input_tokens",
"value": trigger_value,
},
}
]
}
# fast_mode is Opus 4.6/4.7 only; silently drop elsewhere. Incompatible
# with the Priority service_tier (frontend gate prevents both at once;
# backend lets Anthropic 400 if combined).
fast_mode_active = bool(fast_mode) and _anthropic_supports_fast_mode(model)
if fast_mode_active:
body["speed"] = "fast"
url = f"{self.base_url}/messages"
completion_id = f"chatcmpl-anthropic-{model.replace('/', '-')}"
# Log outgoing config keys (not messages) to prove which thinking /
# effort fields reached the wire.
logger.info(
"Anthropic request shape (model=%s, has_thinking=%s, thinking=%s, "
"output_config=%s, temperature=%s, has_top_p=%s, has_top_k=%s, "
"max_tokens=%s)",
model,
"thinking" in body,
body.get("thinking"),
body.get("output_config"),
body.get("temperature"),
"top_p" in body,
"top_k" in body,
body.get("max_tokens"),
)
# Anthropic stop_reason -> OpenAI finish_reason. `pause_turn` maps to
# None so the UI doesn't treat a paused server-tool turn as final.
# `refusal` -> "content_filter" (closest match).
# https://platform.claude.com/docs/en/api/messages#response-stop-reason
_finish_reason_map: dict[str, Optional[str]] = {
"end_turn": "stop",
"max_tokens": "length",
"stop_sequence": "stop",
"tool_use": "tool_calls",
"refusal": "content_filter",
"pause_turn": None,
}
logger.info("Proxying Anthropic Messages API to %s (model=%s)", url, model)
request_headers = self._auth_headers()
# Merge new beta flags onto whatever the registry contributed.
existing_beta = request_headers.get("anthropic-beta", "").strip()
beta_parts = (
[p.strip() for p in existing_beta.split(",") if p.strip()] if existing_beta else []
)
if code_execution_enabled and _ANTHROPIC_CODE_EXECUTION_BETA not in beta_parts:
beta_parts.append(_ANTHROPIC_CODE_EXECUTION_BETA)
if compaction_active and _ANTHROPIC_COMPACTION_BETA not in beta_parts:
beta_parts.append(_ANTHROPIC_COMPACTION_BETA)
if fast_mode_active and _ANTHROPIC_FAST_MODE_BETA not in beta_parts:
beta_parts.append(_ANTHROPIC_FAST_MODE_BETA)
if beta_parts:
request_headers["anthropic-beta"] = ",".join(beta_parts)
try:
async with _http_client.stream(
"POST",
url,
json = body,
headers = request_headers,
timeout = self._stream_timeout,
) as response:
if response.status_code != 200:
error_body = await response.aread()
error_text = error_body.decode("utf-8", errors = "replace")
logger.error(
"Anthropic returned %d: %s",
response.status_code,
error_text[:500],
)
# Stale container detection (mirrors the OpenAI path). When
# we sent a `container` field and the response is 4xx hinting
# the id is expired / missing, emit container_invalidated so
# the chat adapter clears the stored id and the next turn
# falls back to auto-create.
if anthropic_code_exec_container_id and 400 <= response.status_code < 500:
lowered = error_text.lower()
if "container" in lowered and (
"expired" in lowered
or "not_found" in lowered
or "not found" in lowered
or "no such container" in lowered
or "invalid" in lowered
):
yield (
f"data: "
f"{_json.dumps({'id': completion_id, 'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {}, 'finish_reason': None}], '_toolEvent': {'type': 'container_invalidated'}})}"
)
yield _error_sse_line(response.status_code, error_text, self.provider_type)
return
# NOTE: same manual __anext__ loop as stream_chat_completion — see comment there.
lines_gen = response.aiter_lines().__aiter__()
thinking_open = False
# Diagnostic counters for "no thinking content" reports —
# distinguish "Anthropic never sent thinking_delta" from
# "frontend didn't render the chunks".
event_counts: dict[str, int] = {}
# web_search state. Query streams via input_json_delta on a
# server_tool_use block; results land in a separate
# web_search_tool_result block. Per-call citations.
current_server_tool_use: Optional[dict[str, Any]] = None
current_result_block: Optional[dict[str, Any]] = None
web_search_calls: dict[str, dict[str, Any]] = {}
# code_execution state (bash / text_editor sub-tools); kept
# parallel to web_search so concurrent pills don't collide.
current_code_exec_use: Optional[dict[str, Any]] = None
current_code_exec_result: Optional[dict[str, Any]] = None
code_execution_calls: dict[str, dict[str, Any]] = {}
# web_fetch state. Same server_tool_use → *_tool_result block
# shape as web_search but server_tool_use carries
# name="web_fetch" and the result block is
# `web_fetch_tool_result` with content.type=`web_fetch_result`
# (success) or `web_fetch_tool_error` (failure). Kept separate
# from web_search state so a turn using both doesn't collide.
current_web_fetch_use: Optional[dict[str, Any]] = None
current_web_fetch_result: Optional[dict[str, Any]] = None
web_fetch_calls: dict[str, dict[str, Any]] = {}
# Compaction state. Server-side compaction emits a
# `{type:"compaction", content:"..."}` content block whenever it
# runs. The summary text can land on the start event AND/OR via
# text_delta events on the same block (Anthropic's wire format
# is permissive). Accumulate in `current_compaction["content"]`
# and emit on content_block_stop so the chat-adapter can persist
# it onto the assistant message for next-turn round-tripping.
current_compaction: Optional[dict[str, Any]] = None
compaction_blocks_seen = 0
# Document citations from ``citations_delta`` events. Deduped by
# type-specific anchor key; inline [N] is injected after each
# cited run, and the full list is forwarded as a synthetic
# document_citations tool_event on message_stop for the Sources
# panel.
document_citations: list[dict[str, Any]] = []
# Surfaced in the final log line. generated_files_count tracks
# file_id entries on bash_code_execution_result.content that v1
# drops, to gauge how often the future Files API PR would matter.
code_execution_generated_files = 0
# Container id captured from `message_start.message.container.id`
# when code_execution is enabled. Emit a `container_ready`
# _toolEvent on first sight so the chat adapter persists it
# on the thread record. Only emitted when the value differs
# from the inbound id — no churn on reuse.
latched_container_id: Optional[str] = None
container_id_emitted = False
# Cache usage from message_start (cache_creation/read_input_tokens)
# and message_delta (output_tokens), logged on stream complete so
# caching is verifiable per-request without the dashboard.
last_usage: dict[str, Any] = {}
def _content_chunk(text: str) -> str:
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {"content": text},
"finish_reason": None,
}
],
}
return f"data: {_json.dumps(chunk)}"
def _emit_tool_event(payload: dict[str, Any]) -> str:
_stamp_server_tool_marker(payload)
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": None,
}
],
"_toolEvent": payload,
}
return f"data: {_json.dumps(chunk)}"
def _format_web_search_results(results: list[Any]) -> str:
blocks: list[str] = []
for r in results:
if not isinstance(r, dict):
continue
if r.get("type") != "web_search_result":
continue
url = r.get("url", "")
title = r.get("title") or url
if not url:
continue
blocks.append(f"Title: {title}\nURL: {url}")
return "\n---\n".join(blocks)
def _format_web_fetch_result(inner: dict[str, Any]) -> str:
"""Render a `web_fetch_tool_result.content` payload as the
Title / URL / snippet block CodeExecutionToolUI and
parseSourcesFromResult expect from the web_search path.
Success shape (text):
{type: web_fetch_result, url, retrieved_at,
content: {type: document, source: {type: text,
media_type, data}, title?}}
Success shape (pdf): source.type=base64 + media_type=
application/pdf. We don't surface the base64 bytes; the
title + url is enough for the source pill, and the model
still sees the document contents.
Error shape: {type: web_fetch_tool_error, error_code}.
"""
inner_type = inner.get("type") or ""
if inner_type == "web_fetch_tool_error":
return f"Error: {inner.get('error_code', 'unknown')}"
url = inner.get("url", "")
document = inner.get("content") or {}
title = ""
snippet = ""
if isinstance(document, dict):
title = document.get("title") or ""
source = document.get("source") or {}
if isinstance(source, dict):
media_type = source.get("media_type") or ""
data = source.get("data") or ""
# Inline a short text preview so the source pill
# carries usable context; skip for PDFs (body is
# base64-encoded).
if media_type.startswith("text/") and isinstance(data, str) and data:
snippet = data[:240].strip()
# Frontend parseSourcesFromResult only emits a source pill
# when both `Title:` and `URL:` are present, so fall back to
# the URL when Anthropic omits the document title (matches
# the web_search formatter).
if not title and url:
title = url
parts: list[str] = []
if title:
parts.append(f"Title: {title}")
if url:
parts.append(f"URL: {url}")
if snippet:
parts.append(f"Snippet: {snippet}")
return "\n".join(parts) if parts else "(fetch complete)"
def _format_code_execution_result(inner: dict[str, Any]) -> str:
"""Render an Anthropic code-execution result block as the
preformatted text payload the frontend's
CodeExecutionToolUI displays inside a . Handles bash,
text_editor (view/create/str_replace), and the matching
error variants.
"""
inner_type = inner.get("type") or ""
if inner_type.endswith("_error"):
return f"Error: {inner.get('error_code', 'unknown')}"
if inner_type == "bash_code_execution_result":
stdout = inner.get("stdout") or ""
stderr = inner.get("stderr") or ""
return_code = inner.get("return_code")
parts: list[str] = []
if stdout:
parts.append(stdout)
if stderr:
parts.append(f"--- stderr ---\n{stderr}")
if isinstance(return_code, int) and return_code != 0:
parts.append(f"return_code: {return_code}")
return "\n".join(parts) if parts else "(no output)"
if inner_type == "text_editor_code_execution_result":
# view: file content; create: is_file_update flag;
# str_replace: diff `lines` list. The matching
# server_tool_use carries the command + path, already
# encoded into tool_start arguments — here we only
# format the result body.
if "lines" in inner and isinstance(inner.get("lines"), list):
return "\n".join(str(line) for line in inner["lines"])
if "is_file_update" in inner:
return "Updated" if inner.get("is_file_update") else "Created"
content_field = inner.get("content")
if isinstance(content_field, str):
return content_field
return "(file operation complete)"
return "(code execution complete)"
try:
while True:
try:
line = await lines_gen.__anext__()
except StopAsyncIteration:
break
if not line or line.startswith("event:"):
continue
if not line.startswith("data:"):
continue
data_str = line[len("data:") :].strip()
if not data_str:
continue
try:
event = _json.loads(data_str)
except _json.JSONDecodeError:
continue
event_type = event.get("type")
if event_type == "content_block_delta":
delta_kind = (event.get("delta") or {}).get("type")
key = f"{event_type}:{delta_kind}"
else:
key = event_type or ""
event_counts[key] = event_counts.get(key, 0) + 1
# Merge input-side usage from message_start with
# message_delta's output_tokens into last_usage.
if event_type == "message_start":
start_usage = (event.get("message") or {}).get("usage")
if isinstance(start_usage, dict):
last_usage.update(start_usage)
if event_type == "content_block_start":
content_block = event.get("content_block") or {}
block_type = content_block.get("type")
block_name = content_block.get("name")
if block_type == "server_tool_use" and block_name == "web_search":
tool_use_id = content_block.get("id", "") or (
f"ws_{len(web_search_calls)}"
)
current_server_tool_use = {
"id": tool_use_id,
"buffer": "",
}
web_search_calls[tool_use_id] = {
"query": "",
"results": [],
}
elif block_type == "web_search_tool_result":
tool_use_id = content_block.get("tool_use_id", "")
# Anthropic sometimes ships the full results list
# on the start event; sometimes deltas follow.
# Capture whatever is present and finalize on
# content_block_stop.
content = content_block.get("content") or []
current_result_block = {
"tool_use_id": tool_use_id,
"results": list(content) if isinstance(content, list) else [],
}
elif block_type == "server_tool_use" and block_name == "web_fetch":
tool_use_id = content_block.get("id", "") or (
f"wf_{len(web_fetch_calls)}"
)
current_web_fetch_use = {
"id": tool_use_id,
"buffer": "",
}
web_fetch_calls[tool_use_id] = {
"url": "",
"result": None,
}
elif block_type == "web_fetch_tool_result":
tool_use_id = content_block.get("tool_use_id", "")
inner = content_block.get("content") or {}
current_web_fetch_result = {
"tool_use_id": tool_use_id,
"inner": inner if isinstance(inner, dict) else {},
}
elif block_type == "server_tool_use" and block_name in (
"bash_code_execution",
"text_editor_code_execution",
):
tool_use_id = content_block.get("id", "") or (
f"ce_{len(code_execution_calls)}"
)
kind = (
"bash" if block_name == "bash_code_execution" else "text_editor"
)
current_code_exec_use = {
"id": tool_use_id,
"kind": kind,
"buffer": "",
}
code_execution_calls[tool_use_id] = {
"kind": kind,
"arguments": {},
"result": None,
}
elif block_type in (
"bash_code_execution_tool_result",
"text_editor_code_execution_tool_result",
):
# Code-exec result content arrives whole on the
# start event; finalize on content_block_stop to
# match the web_search ordering.
tool_use_id = content_block.get("tool_use_id", "")
inner = content_block.get("content") or {}
current_code_exec_result = {
"tool_use_id": tool_use_id,
"inner": inner if isinstance(inner, dict) else {},
}
elif block_type == "compaction":
# Summary may arrive on start AND/OR via
# text_delta. Capture both; emit on stop.
seed = content_block.get("content") or ""
current_compaction = {
"content": seed if isinstance(seed, str) else "",
}
elif event_type == "content_block_delta":
delta = event.get("delta", {})
delta_type = delta.get("type")
if delta_type == "thinking_delta":
# Wrap as ... for parseAssistantContent.
thinking_text = delta.get("thinking", "")
if thinking_text:
if not thinking_open:
thinking_text = f"{thinking_text}"
thinking_open = True
yield _content_chunk(thinking_text)
elif delta_type == "text_delta":
text = delta.get("text", "")
# text_deltas inside a compaction block carry
# summary chunks; route them into the compaction
# buffer and DON'T yield them to the
# user-visible stream -- the summary is opaque
# internal state, not assistant prose.
if current_compaction is not None:
if text:
current_compaction["content"] += text
else:
# First text after a thinking block closes the
# tag opened above. Anthropic emits a
# content_block_stop between blocks, but
# closing on the text_delta transition is more
# forgiving if events arrive out of order.
if thinking_open:
yield _content_chunk("")
thinking_open = False
if text:
yield _content_chunk(text)
# web_search citations: web_search_tool_result.
# User-doc citations: citations_delta below.
elif delta_type == "citations_delta":
# One citation per event; collapse onto a numbered
# footnote list and inject [N] inline. See
# https://platform.claude.com/docs/en/build-with-claude/citations
cit = delta.get("citation")
if isinstance(cit, dict):
key = _anthropic_citation_key(cit)
idx_for_marker: Optional[int] = None
for idx, existing in enumerate(document_citations, start = 1):
if existing.get("_key") == key:
idx_for_marker = idx
break
if idx_for_marker is None:
document_citations.append({**cit, "_key": key})
idx_for_marker = len(document_citations)
yield _content_chunk(f"[{idx_for_marker}]")
elif delta_type == "input_json_delta":
# partial_json carrying tool inputs (web_search
# query, code-exec command, etc.); route to
# whichever buffer is open.
partial = delta.get("partial_json", "")
if current_server_tool_use is not None:
current_server_tool_use["buffer"] += partial
elif current_code_exec_use is not None:
current_code_exec_use["buffer"] += partial
elif current_web_fetch_use is not None:
current_web_fetch_use["buffer"] += partial
# signature_delta and other delta types are skipped
# — they carry trust / verification metadata, not
# user-visible content.
elif event_type == "content_block_stop":
if current_server_tool_use is not None:
# End of the server_tool_use block — parse the
# accumulated input_json into a query and emit
# tool_start. The matching tool_end fires later
# when the web_search_tool_result block closes
# with the actual results.
buffer = current_server_tool_use["buffer"]
query = ""
if buffer:
try:
parsed = _json.loads(buffer)
if isinstance(parsed, dict):
q = parsed.get("query", "")
if isinstance(q, str):
query = q
except Exception:
query = ""
tool_use_id = current_server_tool_use["id"]
if tool_use_id in web_search_calls:
web_search_calls[tool_use_id]["query"] = query
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "web_search",
"tool_call_id": tool_use_id,
"arguments": ({"query": query} if query else {}),
}
)
current_server_tool_use = None
elif current_result_block is not None:
# End of a web_search_tool_result — emit tool_end
# carrying the search results as Title:/URL:
# blocks. The frontend's parseSourcesFromResult
# lifts these into source pills at message tail.
tool_use_id = current_result_block["tool_use_id"]
results = current_result_block["results"]
if tool_use_id in web_search_calls:
web_search_calls[tool_use_id]["results"] = results
result_text = _format_web_search_results(results)
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": tool_use_id,
"result": (result_text or "(search complete)"),
}
)
current_result_block = None
elif current_code_exec_use is not None:
# End of a code-execution server_tool_use — parse
# the buffered input_json into a {command, path,
# ...} dict and emit tool_start. The matching
# tool_end fires on the result block's
# content_block_stop.
buffer = current_code_exec_use["buffer"]
parsed_args: dict[str, Any] = {}
if buffer:
try:
parsed_obj = _json.loads(buffer)
if isinstance(parsed_obj, dict):
parsed_args = parsed_obj
except Exception:
parsed_args = {}
tool_use_id = current_code_exec_use["id"]
kind = current_code_exec_use["kind"]
emit_args = {"kind": kind, **parsed_args}
if tool_use_id in code_execution_calls:
code_execution_calls[tool_use_id]["arguments"] = emit_args
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "code_execution",
"tool_call_id": tool_use_id,
"arguments": emit_args,
}
)
current_code_exec_use = None
elif current_compaction is not None:
# End of a compaction block: emit a synthetic
# tool_event so the chat-adapter persists it onto
# the assistant message for next-turn round-trip.
compaction_blocks_seen += 1
yield _emit_tool_event(
{
"type": "compaction_block",
"content": current_compaction["content"],
}
)
current_compaction = None
elif current_code_exec_result is not None:
# End of a code-execution result block — format
# the inner result into the text payload
# CodeExecutionToolUI renders.
tool_use_id = current_code_exec_result["tool_use_id"]
inner = current_code_exec_result["inner"]
# Track generated-file count for the follow-up
# Files API PR. v1 drops them.
if isinstance(inner, dict):
file_blocks = inner.get("content")
if isinstance(file_blocks, list):
for entry in file_blocks:
if isinstance(entry, dict) and entry.get("file_id"):
code_execution_generated_files += 1
result_text = _format_code_execution_result(
inner if isinstance(inner, dict) else {}
)
if tool_use_id in code_execution_calls:
code_execution_calls[tool_use_id]["result"] = result_text
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": tool_use_id,
"result": result_text,
}
)
current_code_exec_result = None
elif current_web_fetch_use is not None:
# End of the web_fetch server_tool_use — parse the
# buffered input_json into the URL the model asked
# Anthropic to fetch and emit tool_start. The
# matching tool_end fires on the result block's
# content_block_stop just below.
buffer = current_web_fetch_use["buffer"]
url = ""
if buffer:
try:
parsed = _json.loads(buffer)
if isinstance(parsed, dict):
probe = parsed.get("url", "")
if isinstance(probe, str):
url = probe
except Exception:
logger.debug(
"Failed to parse web_fetch input_json",
buffer = buffer,
)
url = ""
tool_use_id = current_web_fetch_use["id"]
if tool_use_id in web_fetch_calls:
web_fetch_calls[tool_use_id]["url"] = url
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "web_fetch",
"tool_call_id": tool_use_id,
"arguments": ({"url": url} if url else {}),
}
)
current_web_fetch_use = None
elif current_web_fetch_result is not None:
# End of the web_fetch_tool_result — format the
# source pill and emit tool_end.
tool_use_id = current_web_fetch_result["tool_use_id"]
result_text = _format_web_fetch_result(
current_web_fetch_result["inner"]
)
if tool_use_id in web_fetch_calls:
web_fetch_calls[tool_use_id]["result"] = result_text
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": tool_use_id,
"result": result_text,
}
)
current_web_fetch_result = None
elif thinking_open:
# Close the tag when the thinking block
# ends, in case no text_delta follows (e.g.
# display=omitted on Claude 4.7, or
# thinking-only turns).
yield _content_chunk("")
thinking_open = False
elif event_type == "message_delta":
delta_usage = event.get("usage")
if isinstance(delta_usage, dict):
last_usage.update(delta_usage)
# Compaction iterations aren't in top-level
# input/output_tokens; fold them into
# compaction_{input,output}_tokens for billing.
iterations = delta_usage.get("iterations")
if isinstance(iterations, list):
c_in = 0
c_out = 0
for it in iterations:
if isinstance(it, dict) and it.get("type") == "compaction":
c_in += int(it.get("input_tokens") or 0)
c_out += int(it.get("output_tokens") or 0)
if c_in or c_out:
last_usage["compaction_input_tokens"] = c_in
last_usage["compaction_output_tokens"] = c_out
# Container id is on message_delta.delta.container
# (not message_start; not provisioned yet there).
# Emit container_ready only when it differs from the
# inbound id so reuse doesn't re-write it every turn.
delta_obj = event.get("delta") or {}
container_obj = delta_obj.get("container")
if isinstance(container_obj, dict) and latched_container_id is None:
probe = container_obj.get("id")
if isinstance(probe, str) and probe:
latched_container_id = probe
if (
latched_container_id
and not container_id_emitted
and latched_container_id != anthropic_code_exec_container_id
):
yield _emit_tool_event(
{
"type": "container_ready",
"container_id": latched_container_id,
}
)
container_id_emitted = True
stop_reason = event.get("delta", {}).get("stop_reason")
if stop_reason:
if thinking_open:
yield _content_chunk("")
thinking_open = False
# `pause_turn` is in-progress, not terminal: the
# SSE stream still ends with [DONE] via
# message_stop but we skip emitting a
# finish_reason="stop" chunk that would truncate
# the rendered message in the UI.
mapped = _finish_reason_map.get(stop_reason, "stop")
# Streaming refusal: emit a visible notice plus an
# out-of-band _toolEvent so the frontend can prune
# the refused turn. The mapped finish_reason is
# "content_filter" per OpenAI spec.
# https://platform.claude.com/docs/en/test-and-evaluate/strengthen-guardrails/handle-streaming-refusals
if stop_reason == "refusal":
logger.warning(
"Anthropic refusal stop_reason (model=%s)",
model,
)
# Drop signal rides _toolEvent (not text) so
# assistant content can't spoof a context
# reset.
yield _content_chunk(
"\n\n_The response was stopped by "
"Anthropic's safety classifier. Edit "
"or remove the previous turn and try "
"again._"
)
yield _emit_tool_event({"type": "anthropic_refusal"})
if mapped is not None:
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": mapped,
}
],
}
yield f"data: {_json.dumps(chunk)}"
elif event_type == "message_stop":
if thinking_open:
yield _content_chunk("")
thinking_open = False
# Forward document_citations so the Sources panel
# can render the inline [N] footnotes. ``cited_text``
# is truncated server-side to keep SSE bytes bounded
# on long spans.
if document_citations:
clean_cits = []
for c in document_citations:
entry = {k: v for k, v in c.items() if k != "_key"}
cited = entry.get("cited_text")
if isinstance(cited, str) and len(cited) > _CITED_TEXT_MAX_LEN:
entry["cited_text"] = cited[:_CITED_TEXT_MAX_LEN] + "…"
clean_cits.append(entry)
yield _emit_tool_event(
{
"type": "document_citations",
"citations": clean_cits,
}
)
# Final include_usage-style chunk so callers see
# cache_creation / cache_read without scraping the
# server log.
usage_line = _build_usage_chunk(
completion_id,
"anthropic",
last_usage,
)
if usage_line:
yield usage_line
yield "data: [DONE]"
await response.aclose() # set PoolByteStream._closed=True FIRST
break
except GeneratorExit:
await response.aclose() # set PoolByteStream._closed=True FIRST
await lines_gen.aclose() # now safe — aclose() is a no-op
raise
finally:
# Per-event-type counts + web_search summary for triage.
web_search_requested = bool(enabled_tools and "web_search" in enabled_tools)
web_search_invocations = len(web_search_calls)
total_results = sum(
len(sc.get("results") or []) for sc in web_search_calls.values()
)
queries = [sc["query"] for sc in web_search_calls.values() if sc.get("query")]
# cache_read_input_tokens > 0 proves the cache_control marker
# works (turn 1 shows cache_creation instead).
code_execution_invocations = len(code_execution_calls)
code_execution_results = sum(
1 for c in code_execution_calls.values() if c.get("result") is not None
)
web_fetch_requested = web_fetch_enabled
web_fetch_invocations = len(web_fetch_calls)
web_fetch_urls = [wf["url"] for wf in web_fetch_calls.values() if wf.get("url")]
logger.info(
"Anthropic stream complete (model=%s, "
"web_search_requested=%s, web_search_invocations=%s, "
"results=%s, queries=%s, "
"web_fetch_requested=%s, web_fetch_invocations=%s, "
"web_fetch_urls=%s, "
"code_execution_requested=%s, "
"code_execution_invocations=%s, "
"code_execution_results=%s, "
"code_execution_generated_files=%s, "
"container_id_in=%s, container_id_out=%s, "
"input_tokens=%s, output_tokens=%s, "
"cache_creation_input_tokens=%s, "
"cache_read_input_tokens=%s, "
"compaction_input_tokens=%s, "
"compaction_output_tokens=%s, "
"compaction_blocks_seen=%s, events=%s)",
model,
web_search_requested,
web_search_invocations,
total_results,
queries,
web_fetch_requested,
web_fetch_invocations,
web_fetch_urls,
code_execution_enabled,
code_execution_invocations,
code_execution_results,
code_execution_generated_files,
anthropic_code_exec_container_id,
latched_container_id,
last_usage.get("input_tokens"),
last_usage.get("output_tokens"),
last_usage.get("cache_creation_input_tokens"),
last_usage.get("cache_read_input_tokens"),
last_usage.get("compaction_input_tokens"),
last_usage.get("compaction_output_tokens"),
compaction_blocks_seen,
event_counts,
)
await response.aclose()
await lines_gen.aclose()
except httpx.ConnectError as exc:
logger.error("Connection error to %s: %s", self.provider_type, exc)
yield _error_sse_line(
502,
f"Failed to connect to {self.provider_type}: {exc}",
self.provider_type,
)
except httpx.ReadTimeout as exc:
logger.error("Read timeout from %s: %s", self.provider_type, exc)
yield _error_sse_line(
504,
f"Timeout waiting for {self.provider_type} response",
self.provider_type,
)
except httpx.HTTPError as exc:
logger.error("HTTP error from %s: %s", self.provider_type, exc)
yield _error_sse_line(
502,
f"Error communicating with {self.provider_type}: {exc}",
self.provider_type,
)
async def _stream_gemini(
self,
messages: list[dict[str, Any]],
model: str,
temperature: float,
top_p: float,
max_tokens: Optional[int],
top_k: Optional[int] = None,
presence_penalty: float = 0.0,
enabled_tools: Optional[list[str]] = None,
enable_prompt_caching: Optional[Any] = None,
enable_thinking: Optional[bool] = None,
reasoning_effort: Optional[str] = None,
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[Any] = None,
) -> AsyncGenerator[str, None]:
"""
Call Google's native Gemini API and translate its streaming
``streamGenerateContent`` response into OpenAI Chat Completions chunks.
Gemini does NOT speak the OpenAI Chat Completions contract on its
primary endpoint. The wire shape is:
POST /v1beta/models/{model}:streamGenerateContent?alt=sse
{
"contents": [{"role": "user|model", "parts": [{"text": "..."}]}],
"systemInstruction": {"parts": [{"text": "..."}]},
"generationConfig": {"temperature": 0.7, "topP": 0.95, "topK": 40,
"maxOutputTokens": 1024},
"tools": [{"googleSearch": {}}, {"codeExecution": {}}],
"cachedContent": "" // optional, see caching docs
}
Streamed responses are SSE frames carrying partial
``GenerateContentResponse`` objects:
{"candidates": [{"content": {"parts": [{"text": "Hello"}]},
"finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 3}}
Image generation uses the same endpoint with model
``gemini-2.5-flash-image`` (Nano Banana); the response carries an
``inlineData`` part with base64 PNG bytes and a ``mimeType``. We surface
that through the same ``tool_start`` / ``tool_end`` ``image_b64``
envelope the OpenAI image_generation path uses, so the chat UI renders
the image inline with no extra plumbing.
References:
- https://ai.google.dev/gemini-api/docs/text-generation
- https://ai.google.dev/gemini-api/docs/function-calling
- https://ai.google.dev/gemini-api/docs/grounding
- https://ai.google.dev/gemini-api/docs/caching
- https://ai.google.dev/gemini-api/docs/image-generation
"""
import json as _json
# Validate the user-controlled model id first: `../cachedContents/x` is
# path traversal, and rejecting early avoids attacker-triggered outbound
# image fetches on a doomed request. Catalog ids match `[A-Za-z0-9._-]+`.
if not re.fullmatch(r"[A-Za-z0-9._-]+", model):
yield _error_sse_line(
400,
f"Invalid Gemini model id: {model!r}",
self.provider_type,
)
return
# Translate OpenAI messages -> Gemini contents. system role promotes to
# top-level systemInstruction.
system_text_parts: list[str] = []
contents: list[dict[str, Any]] = []
# OpenAI may drop `name` from role="tool" follow-ups. Remember prior
# function names so functionResponse isn't sent name-less (Gemini 400s
# on empty names).
tool_call_names: dict[str, str] = {}
# tool_call_ids whose assistant card was dropped (synthetic builtin) or
# already replayed as native parts. Their role="tool" follow-up must be
# skipped to avoid orphan/duplicate responses.
_gemini_skip_tool_result_ids: set[str] = set()
# Per-request image caps. The byte cap counts DECODED bytes; set to
# ~14 MB because base64 expansion + prompt overhead must fit Gemini's
# ~20 MB request limit.
_GEMINI_REMOTE_IMAGE_MAX_COUNT = 8
_GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES = 14 * 1024 * 1024
_remote_image_count = 0
_remote_image_total_bytes = 0
for msg in messages:
role = msg.get("role")
content = msg.get("content", "")
if role == "system":
if isinstance(content, str):
if content:
system_text_parts.append(content)
elif isinstance(content, list):
for part in content:
if (
isinstance(part, dict)
and part.get("type") == "text"
and part.get("text")
):
system_text_parts.append(part["text"])
continue
# Map OpenAI roles to Gemini's two-role contract.
gemini_role = "model" if role == "assistant" else "user"
parts: list[dict[str, Any]] = []
if isinstance(content, str):
if content:
parts.append({"text": content})
elif isinstance(content, list):
for part in content:
if not isinstance(part, dict):
continue
ptype = part.get("type")
if ptype == "text":
text = part.get("text", "")
if text:
parts.append({"text": text})
elif ptype == "image_url":
url = part.get("image_url", {}).get("url", "")
if url.startswith("data:"):
header, _, b64data = url.partition(",")
media_type = (
header.split(";")[0].replace("data:", "").strip().lower()
or "image/jpeg"
)
# Reject non-image data URLs (e.g. data:text/html);
# they'd 400 the request as inlineData. Mirrors the
# remote-fetch path's Content-Type check.
if not media_type.startswith("image/"):
logger.info(
"Gemini inlineData: refusing non-image data URL media_type=%s",
media_type,
)
elif b64data:
# data: URLs share the same caps as fetched
# URLs so inline payloads don't bypass them.
_data_approx_bytes = (len(b64data) * 3) // 4
if _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:
logger.info(
"Gemini inlineData: per-request count cap %d reached, dropping image",
_GEMINI_REMOTE_IMAGE_MAX_COUNT,
)
elif (
_remote_image_total_bytes + _data_approx_bytes
> _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES
):
logger.info(
"Gemini inlineData: per-request byte cap reached, dropping image",
)
else:
_remote_image_count += 1
_remote_image_total_bytes += _data_approx_bytes
parts.append(
{
"inlineData": {
"mimeType": media_type,
"data": b64data,
}
}
)
elif url:
# fileData.fileUri only accepts Files-API URIs and
# YouTube; everything else is downloaded and inlined.
# Parse host explicitly so e.g.
# https://evil.com/youtube.com/x isn't mis-detected.
try:
_parsed_image_url = urlparse(url)
except (ValueError, UnicodeError):
_parsed_image_url = None
if _parsed_image_url is None:
_img_scheme = ""
_img_host = ""
_img_path = ""
else:
_img_scheme = (_parsed_image_url.scheme or "").lower()
_img_host = (_parsed_image_url.hostname or "").lower()
_img_path = _parsed_image_url.path or ""
_is_native_uri = (
_img_scheme == "https"
and _img_host == "generativelanguage.googleapis.com"
and _img_path.startswith("/v1beta/files/")
)
_is_youtube = _img_scheme == "https" and (
_img_host == "youtu.be"
or _img_host == "youtube.com"
or _img_host.endswith(".youtube.com")
)
_guessed, _ = mimetypes.guess_type(_img_path)
_media_type = (
_guessed
if isinstance(_guessed, str) and _guessed.startswith("image/")
else "image/jpeg"
)
if _is_youtube:
# YouTube URIs must use video/mp4; the
# default image/jpeg yields a 400.
parts.append(
{
"fileData": {
"fileUri": url,
"mimeType": "video/mp4",
}
}
)
elif _is_native_uri:
parts.append(
{
"fileData": {
"fileUri": url,
"mimeType": _media_type,
}
}
)
elif _remote_image_count >= _GEMINI_REMOTE_IMAGE_MAX_COUNT:
logger.info(
"Gemini image fetch: per-request count cap %d reached, dropping image",
_GEMINI_REMOTE_IMAGE_MAX_COUNT,
)
else:
# Refuse pre-fetch when the per-request byte
# budget is spent; pass the remainder so
# over-budget URLs reject on Content-Length.
_remaining_bytes = (
_GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES - _remote_image_total_bytes
)
if _remaining_bytes <= 0:
logger.info(
"Gemini image fetch: per-request byte cap already reached, dropping image",
)
else:
# Count attempts before awaiting so slow URLs
# don't each burn the timeout.
_remote_image_count += 1
_fetched = await _safe_fetch_image_for_gemini(
url,
_media_type,
max_bytes = _remaining_bytes,
)
if _fetched is not None:
_final_mime, _b64 = _fetched
# base64 expands ~4/3 — recover bytes from len(_b64).
_approx_bytes = (len(_b64) * 3) // 4
if (
_remote_image_total_bytes + _approx_bytes
> _GEMINI_REMOTE_IMAGE_MAX_TOTAL_BYTES
):
logger.info(
"Gemini image fetch: per-request byte cap reached, dropping image",
)
else:
_remote_image_total_bytes += _approx_bytes
parts.append(
{
"inlineData": {
"mimeType": _final_mime,
"data": _b64,
}
}
)
# Gemini 3 strict function-calling requires text-part
# thoughtSignatures to be replayed on history; the frontend stows
# the latest one as extra_content.google.thought_signature on the
# assistant message and we pin it onto the last text part here.
if role == "assistant" and parts:
_msg_extra = msg.get("extra_content") if isinstance(msg, dict) else None
if isinstance(_msg_extra, dict):
_msg_g = _msg_extra.get("google") or {}
if isinstance(_msg_g, dict):
_msg_sig = _msg_g.get("thought_signature") or _msg_g.get("thoughtSignature")
if isinstance(_msg_sig, str) and _msg_sig:
for _idx in range(len(parts) - 1, -1, -1):
if "text" in parts[_idx]:
parts[_idx] = {
**parts[_idx],
"thoughtSignature": _msg_sig,
}
break
# Translate OpenAI tool_calls into Gemini functionCall parts.
# code_execution / image_generation replay their native parts
# (executableCode / codeExecutionResult / inlineData) stowed on
# extra_content.google.native_part.
tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else None
if isinstance(tool_calls, list):
for tc in tool_calls:
if not isinstance(tc, dict):
continue
fn = tc.get("function") or {}
if not isinstance(fn, dict):
continue
args_raw = fn.get("arguments") or "{}"
if isinstance(args_raw, str):
try:
args = _json.loads(args_raw)
except Exception:
args = {"_raw": args_raw}
elif isinstance(args_raw, dict):
args = args_raw
else:
args = {}
fn_name = fn.get("name", "")
tc_id = tc.get("id")
if fn_name and isinstance(tc_id, str) and tc_id:
tool_call_names[tc_id] = fn_name
# Replay native Gemini code_execution / image_generation
# parts from extra_content.google.native_part, falling back
# to args.google.native_part for OAI-compat round-trips.
_extra = tc.get("extra_content")
_native_part = None
_google_extra: dict[str, Any] = {}
if isinstance(_extra, dict):
_ge = _extra.get("google") or {}
if isinstance(_ge, dict):
_google_extra = _ge
_native_part = _ge.get("native_part")
if _native_part is None and isinstance(args, dict):
_args_google = args.get("google")
if isinstance(_args_google, dict):
_args_np = _args_google.get("native_part")
if isinstance(_args_np, dict):
_native_part = _args_np
if not _google_extra:
_google_extra = _args_google
# Synthetic builtin cards (web_search/web_fetch) must not
# become fake functionCalls; drop them. Native
# code_execution / image_generation replay below.
_name_lc = fn_name.lower() if isinstance(fn_name, str) else ""
_is_synthetic_server_builtin = (
_name_lc
in (
"web_search",
"web_fetch",
"code_execution",
"image_generation",
)
and isinstance(args, dict)
and (
args.get("_server_tool") is True
or isinstance((args.get("google") or {}).get("native_part"), dict)
)
)
if _is_synthetic_server_builtin and not (
_name_lc in ("code_execution", "image_generation")
and isinstance(_native_part, dict)
):
# No replayable Gemini native part -- skip entirely
# rather than send a fake functionCall. Also remember
# this tool_call_id so a matching role="tool" follow-up
# doesn't become an orphan functionResponse below.
if isinstance(tc_id, str) and tc_id:
_gemini_skip_tool_result_ids.add(tc_id)
tool_call_names.pop(tc_id, None)
continue
if fn_name in ("code_execution", "image_generation") and isinstance(
_native_part, dict
):
# code_execution/image_generation history is replayed as
# native parts; the matching role="tool" must be skipped
# or Gemini sees a functionResponse with no declared
# function name and 400s the turn.
if isinstance(tc_id, str) and tc_id:
_gemini_skip_tool_result_ids.add(tc_id)
# New shape: `native_part.parts` is an ordered list of
# full part wrappers, each carrying its own
# `thoughtSignature`. Preserves Gemini 3's strict
# per-part replay requirement when the frontend merged
# executableCode + codeExecutionResult + inlineData into
# the same tool-call card.
_native_parts_list = _native_part.get("parts")
if isinstance(_native_parts_list, list):
for _entry in _native_parts_list:
if isinstance(_entry, dict):
parts.append(_entry)
continue
# Legacy single-object native_part: fan the shared
# thoughtSignature only when one subpart exists; for
# code+result, prefer executableCode and drop the
# signature elsewhere.
_legacy_sig = _native_part.get("thoughtSignature") or _native_part.get(
"thought_signature"
)
_legacy_subparts = [
_k
for _k in (
"executableCode",
"codeExecutionResult",
"inlineData",
)
if isinstance(_native_part.get(_k), dict)
]
for _native_key in (
"executableCode",
"codeExecutionResult",
"inlineData",
):
_sub = _native_part.get(_native_key)
if not isinstance(_sub, dict):
continue
_replay_part: dict[str, Any] = {_native_key: _sub}
if isinstance(_legacy_sig, str) and _legacy_sig:
if len(_legacy_subparts) == 1:
_replay_part["thoughtSignature"] = _legacy_sig
elif _native_key == "executableCode":
_replay_part["thoughtSignature"] = _legacy_sig
parts.append(_replay_part)
continue
# Forward the OpenAI tool_call id into Gemini's
# functionCall.id so a follow-up turn issuing multiple calls
# to the same function (different args, same name) can be
# disambiguated on the response side. Gemini accepts the
# field per
# https://ai.google.dev/gemini-api/docs/function-calling.
function_call_part: dict[str, Any] = {
"name": fn_name,
"args": args,
}
if isinstance(tc_id, str) and tc_id:
function_call_part["id"] = tc_id
# Gemini 3 function-calling requires the prior
# thoughtSignature echoed back as a sibling of the
# functionCall part. The translator stows it on the
# assistant tool_call via
# `extra_content.google.thought_signature` (see the inbound
# emit below).
fc_part: dict[str, Any] = {"functionCall": function_call_part}
sig = _google_extra.get("thought_signature") or _google_extra.get(
"thoughtSignature"
)
if isinstance(sig, str) and sig:
fc_part["thoughtSignature"] = sig
parts.append(fc_part)
if role == "tool":
# Drop the follow-up if its assistant-side tool_call was dropped
# or already replayed as native parts; else it would be an
# orphan/duplicate functionResponse.
_tc_id_for_skip = msg.get("tool_call_id")
if (
isinstance(_tc_id_for_skip, str)
and _tc_id_for_skip in _gemini_skip_tool_result_ids
):
continue
# OpenAI's role="tool" follow-up carries the function result.
# Gemini's matching shape is a role="user" turn with a
# functionResponse part. When the caller dropped ``name``,
# recover it from the matching assistant tool_call so Gemini
# doesn't 400 on an empty name.
tool_name = msg.get("name") or msg.get("tool_name") or ""
if not tool_name:
tc_id = msg.get("tool_call_id")
if isinstance(tc_id, str) and tc_id in tool_call_names:
tool_name = tool_call_names[tc_id]
response_payload: Any
if isinstance(content, list):
# Flatten list-form tool content to text so the
# functionResponse result matches the string-content path.
_flat_parts: list[str] = []
for _cpart in content:
if (
isinstance(_cpart, dict)
and _cpart.get("type") == "text"
and isinstance(_cpart.get("text"), str)
):
_flat_parts.append(_cpart["text"])
_flat_text = "".join(_flat_parts)
try:
response_payload = _json.loads(_flat_text)
except Exception:
response_payload = {"result": _flat_text}
elif isinstance(content, str):
try:
response_payload = _json.loads(content)
except Exception:
response_payload = {"result": content}
else:
response_payload = content or {}
function_response_part: dict[str, Any] = {
"name": tool_name,
"response": (
response_payload
if isinstance(response_payload, dict)
else {"result": response_payload}
),
}
# Mirror tool_call_id onto functionResponse.id so Gemini can
# match the result to the originating functionCall when multiple
# parallel calls were made.
tc_id = msg.get("tool_call_id")
if isinstance(tc_id, str) and tc_id:
function_response_part["id"] = tc_id
parts = [{"functionResponse": function_response_part}]
gemini_role = "user"
if parts:
# Merge consecutive functionResponse-only user blocks: Gemini
# wants parallel tool responses grouped into one user turn.
if (
role == "tool"
and contents
and contents[-1].get("role") == "user"
and all(
isinstance(p, dict) and "functionResponse" in p
for p in (contents[-1].get("parts") or [])
)
):
contents[-1]["parts"].extend(parts)
else:
contents.append({"role": gemini_role, "parts": parts})
body: dict[str, Any] = {"contents": contents}
if system_text_parts:
body["systemInstruction"] = {"parts": [{"text": "\n\n".join(system_text_parts)}]}
# Generation config -- temperature / topP / topK / maxOutputTokens map
# straight across. The frontend capability matrix restricts the sliders
# the UI exposes for Gemini to this set.
gen_config: dict[str, Any] = {}
if temperature is not None:
gen_config["temperature"] = temperature
if top_p is not None:
gen_config["topP"] = top_p
if top_k is not None and top_k > 0:
gen_config["topK"] = top_k
# Gemini accepts ``presencePenalty`` on generationConfig with the same
# sign convention as the OpenAI knob (positive discourages repetition).
# Forward when the caller sets it.
if presence_penalty:
gen_config["presencePenalty"] = presence_penalty
if max_tokens is not None:
gen_config["maxOutputTokens"] = max_tokens
# Nano Banana image generation: only image-capable models (id contains
# `-image`/`nano-banana`) accept responseModalities=["TEXT","IMAGE"];
# text models 400 on it, so a stale image_generation pill is ignored.
# https://ai.google.dev/gemini-api/docs/image-generation
model_lc = model.lower()
is_image_picker_model = "-image" in model_lc or "nano-banana" in model_lc
# tool_choice="none"/forced-function also suppresses implicit image
# generation, else an explicit opt-out still bills for image output.
_tool_choice_disabled = (
isinstance(tool_choice, str) and tool_choice.strip().lower() == "none"
)
_tool_choice_forced_function = (
isinstance(tool_choice, dict)
and tool_choice.get("type") == "function"
and isinstance(tool_choice.get("function"), dict)
and bool(tool_choice["function"].get("name"))
)
_hosted_builtins_allowed = not _tool_choice_disabled and not _tool_choice_forced_function
# Image-tier models reject text-only tools and thinkingConfig regardless
# of the pill (model-level constraint); the pill only controls image
# output. Decouple the two so Images-off + Code/Search-on doesn't 400.
image_tool_requested = bool(
_hosted_builtins_allowed and enabled_tools and "image_generation" in enabled_tools
)
# Strict tool / thinking strip uses the model-id check.
is_image_model_strict = is_image_picker_model
# The actual modality flip only happens when the user opted in.
is_image_model = is_image_picker_model and image_tool_requested
if is_image_model:
gen_config["responseModalities"] = ["TEXT", "IMAGE"]
elif is_image_picker_model:
# Force TEXT-only so an image-capable model with Images OFF doesn't
# still bill for image output.
gen_config["responseModalities"] = ["TEXT"]
# Thinking control. Gemini 3 uses thinkingLevel (str), 2.5 uses
# thinkingBudget (int). Gemini 3 has no full-off; minimum is
# "minimal" on Flash, "low" on Pro.
# https://ai.google.dev/gemini-api/docs/thinking
_GEMINI3_THINKING_PREFIXES = (
"gemini-3.5-",
"gemini-3.1-",
"gemini-3-",
"gemini-pro-latest",
"gemini-flash-latest",
"gemini-flash-lite-latest",
)
_GEMINI3_PRO_PREFIXES = (
"gemini-3.5-pro",
"gemini-3.1-pro",
"gemini-3-pro",
"gemini-pro-latest",
)
_PRO_THINKING_PREFIXES = ("gemini-2.5-pro",)
is_gemini3_thinking = any(model_lc.startswith(p) for p in _GEMINI3_THINKING_PREFIXES)
is_gemini3_pro = any(model_lc.startswith(p) for p in _GEMINI3_PRO_PREFIXES)
_is_pro_thinking_only = any(
model_lc == p or model_lc.startswith(p + "-") for p in _PRO_THINKING_PREFIXES
)
effort_lc = (reasoning_effort or "").strip().lower()
if not is_image_model_strict and is_gemini3_thinking:
# Gemini 3.x thinkingLevel matrix:
# 3.1+ Pro: low/medium/high
# 3 Pro: low/high (deprecated 2026-03-09)
# 3.x Flash*: minimal/low/medium/high
# Coerce minimal->low on Pro; medium->high on legacy 3-Pro.
_G3_LEVELS = {"minimal", "low", "medium", "high"}
level: Optional[str] = None
if effort_lc in ("none", "off"):
level = "low" if is_gemini3_pro else "minimal"
elif effort_lc == "max":
level = "high"
elif effort_lc in _G3_LEVELS:
# Coerce legacy 3-Pro (low/high only) inputs.
_is_legacy_gemini3_pro = model_lc.startswith(
("gemini-3-pro-preview", "gemini-3-pro")
) and not model_lc.startswith(("gemini-3.1-pro", "gemini-3.5-pro"))
if is_gemini3_pro and effort_lc == "minimal":
level = "low"
elif _is_legacy_gemini3_pro and effort_lc == "medium":
level = "high"
else:
level = effort_lc
elif enable_thinking is True:
level = "high"
elif enable_thinking is False:
level = "low" if is_gemini3_pro else "minimal"
if level is not None:
gen_config["thinkingConfig"] = {"thinkingLevel": level}
elif not is_image_model_strict:
# Gemini 2.5 / older: thinkingBudget int. Effort -> budget mirrors
# the OpenAI minimal/low/medium/high ladder so the frontend picker
# maps cleanly.
# NOTE: gemini-2.5-flash-lite rejects positive budgets below 512
# with HTTP 400, so minimal=512 sits at that floor.
_EFFORT_TO_BUDGET: dict[str, int] = {
"minimal": 512,
"low": 2048,
"medium": 8192,
"high": 24576,
"xhigh": -1,
"max": -1,
}
thinking_budget: Optional[int] = None
if effort_lc == "none" or enable_thinking is False:
# Pro-tier 2.5 rejects budget=0 (400 "only works in thinking
# mode"), so coerce to a small positive value.
thinking_budget = 128 if _is_pro_thinking_only else 0
elif effort_lc in _EFFORT_TO_BUDGET:
thinking_budget = _EFFORT_TO_BUDGET[effort_lc]
elif enable_thinking is True:
thinking_budget = -1
if thinking_budget is not None:
gen_config["thinkingConfig"] = {
"thinkingBudget": thinking_budget,
}
if gen_config:
body["generationConfig"] = gen_config
# Hosted tools: googleSearch (grounding) and codeExecution.
# Image-mode rejects codeExecution; only Gemini 3 image models
# accept googleSearch.
# https://ai.google.dev/gemini-api/docs/grounding
# https://ai.google.dev/gemini-api/docs/code-execution
def _gemini_image_model_allows_google_search(_m: str) -> bool:
return (
_m.startswith("gemini-3-pro-image")
or _m.startswith("gemini-3.1-flash-image")
or _m.startswith("nano-banana-pro")
or _m.startswith("nano-banana-2")
)
google_search_allowed = (
not is_image_model_strict or _gemini_image_model_allows_google_search(model_lc)
)
code_execution_allowed = not is_image_model_strict
text_tools_allowed = not is_image_model_strict
# tool_choice="none" / forced-function suppresses hosted builtins too,
# matching the Anthropic / OpenRouter gates.
tools_array: list[dict[str, Any]] = []
if (
_hosted_builtins_allowed
and enabled_tools
and "web_search" in enabled_tools
and google_search_allowed
):
tools_array.append({"googleSearch": {}})
if (
_hosted_builtins_allowed
and enabled_tools
and "code_execution" in enabled_tools
and code_execution_allowed
):
tools_array.append({"codeExecution": {}})
# OpenAI function declarations -> Gemini functionDeclarations
# (https://ai.google.dev/gemini-api/docs/function-calling#step_1). Gemini's
# Schema accepts only the OpenAPI 3.0 subset; OpenAI strict tools include
# keys (additionalProperties, $schema, $defs, ...) that 400 as
# INVALID_ARGUMENT, so strip recursively. Ref:
# https://ai.google.dev/api/caching#Schema
_GEMINI_ALLOWED_SCHEMA_KEYS = frozenset(
{
"type",
"format",
"title",
"description",
"nullable",
"enum",
"maxItems",
"minItems",
"properties",
"required",
"minProperties",
"maxProperties",
"items",
"minimum",
"maximum",
"minLength",
"maxLength",
"pattern",
"default",
"anyOf",
"propertyOrdering",
}
)
def _resolve_local_schema_ref(root: Optional[dict[str, Any]], ref: str) -> Optional[Any]:
# Walk a `#/foo/bar` JSON pointer against the schema root. Returns
# None if the pointer doesn't resolve to a dict, so the caller can
# fall back to the unresolved node.
if not isinstance(root, dict) or not isinstance(ref, str):
return None
if not ref.startswith("#/"):
return None
node: Any = root
for raw_part in ref[2:].split("/"):
if not raw_part:
continue
part = raw_part.replace("~1", "/").replace("~0", "~")
if not isinstance(node, dict) or part not in node:
return None
node = node[part]
return node
def _sanitize_gemini_schema(
node: Any,
root: Optional[dict[str, Any]] = None,
_seen_refs: Optional[frozenset[str]] = None,
) -> Any:
# Recursively filter to Gemini's OpenAPI 3.0 subset (drop non-allowlist
# keys) and translate JSON Schema `type:[X,"null"]` into
# `type:X` + `nullable:true`.
if root is None and isinstance(node, dict):
root = node
if _seen_refs is None:
_seen_refs = frozenset()
if isinstance(node, dict):
# Inline `$ref` targets (Gemini's subset has no $ref), with local
# siblings overriding and a cycle guard.
_ref = node.get("$ref")
if isinstance(_ref, str):
if _ref in _seen_refs:
return {}
_target = _resolve_local_schema_ref(root, _ref)
if isinstance(_target, dict):
_merged = {
**_target,
**{k: v for k, v in node.items() if k != "$ref"},
}
return _sanitize_gemini_schema(_merged, root, _seen_refs | {_ref})
cleaned: dict[str, Any] = {}
_nullable_from_union = False
_flattened_type: Optional[str] = None
_union_any_of: Optional[list[dict[str, Any]]] = None
_raw_type = node.get("type")
if isinstance(_raw_type, list):
_non_null = [t for t in _raw_type if t != "null"]
if len(_non_null) < len(_raw_type):
_nullable_from_union = True
if len(_non_null) == 1:
_flattened_type = _non_null[0]
elif len(_non_null) > 1:
# Preserve multi-type unions as anyOf; flattening to the
# first non-null type silently drops the other branches
# and changes the tool contract.
_union_any_of = [{"type": _t} for _t in _non_null if isinstance(_t, str)]
for _k, _v in node.items():
if _k == "type" and isinstance(_v, list):
# Handled below via _flattened_type.
continue
if _k not in _GEMINI_ALLOWED_SCHEMA_KEYS:
continue
if _k == "properties" and isinstance(_v, dict):
cleaned[_k] = {
_name: _sanitize_gemini_schema(_subschema, root, _seen_refs)
for _name, _subschema in _v.items()
}
elif _k == "items":
cleaned[_k] = _sanitize_gemini_schema(_v, root, _seen_refs)
elif _k == "anyOf" and isinstance(_v, list):
# Optional[X] / Union[A, B, None]: Pydantic emits
# `anyOf: [..., {"type":"null"}]`. Gemini's OpenAPI
# subset rejects `"type": "null"` inside anyOf, so drop
# the null variant and surface it via `nullable: true`.
# If exactly one non-null branch remains, collapse it
# inline; otherwise keep the slim anyOf and mark the
# field nullable.
_saw_null = any(
isinstance(_entry, dict) and _entry.get("type") == "null"
for _entry in _v
)
_non_null_entries = [
_entry
for _entry in _v
if not (isinstance(_entry, dict) and _entry.get("type") == "null")
]
if len(_non_null_entries) == 1 and _saw_null:
_inner = _sanitize_gemini_schema(_non_null_entries[0], root, _seen_refs)
if isinstance(_inner, dict):
for _ik, _iv in _inner.items():
cleaned.setdefault(_ik, _iv)
cleaned.setdefault("nullable", True)
else:
cleaned[_k] = [
_sanitize_gemini_schema(_entry, root, _seen_refs)
for _entry in _non_null_entries
]
if _saw_null:
cleaned.setdefault("nullable", True)
elif _k in ("required", "enum", "propertyOrdering"):
# Lists of plain strings; copy verbatim.
cleaned[_k] = _v
else:
cleaned[_k] = _v
if _union_any_of is not None and "anyOf" not in cleaned:
cleaned["anyOf"] = [
_sanitize_gemini_schema(_s, root, _seen_refs) for _s in _union_any_of
]
elif _flattened_type is not None:
cleaned["type"] = _flattened_type
if _nullable_from_union and "nullable" not in cleaned:
cleaned["nullable"] = True
return cleaned
return node
function_declarations: list[dict[str, Any]] = []
if tools and text_tools_allowed and not _tool_choice_disabled:
for _tool in tools:
if not isinstance(_tool, dict) or _tool.get("type") != "function":
continue
_fn = _tool.get("function")
if not isinstance(_fn, dict) or not _fn.get("name"):
continue
_decl: dict[str, Any] = {
"name": _fn["name"],
"description": _fn.get("description") or "",
}
_params = _fn.get("parameters")
if isinstance(_params, dict):
_decl["parameters"] = _sanitize_gemini_schema(_params)
function_declarations.append(_decl)
if function_declarations:
tools_array.append({"functionDeclarations": function_declarations})
if tools_array:
body["tools"] = tools_array
# Tool-choice mapping: OpenAI "auto"/"none"/"required"/{name=...} ->
# Gemini toolConfig.functionCallingConfig.mode + allowedFunctionNames.
if tool_choice is not None and function_declarations and text_tools_allowed:
_mode: Optional[str] = None
_allowed: Optional[list[str]] = None
if isinstance(tool_choice, str):
_tc_lc = tool_choice.strip().lower()
if _tc_lc == "auto":
_mode = "AUTO"
elif _tc_lc == "none":
_mode = "NONE"
elif _tc_lc in ("required", "any"):
_mode = "ANY"
elif isinstance(tool_choice, dict) and tool_choice.get("type") == "function":
_fn_pick = tool_choice.get("function") or {}
_name = _fn_pick.get("name") if isinstance(_fn_pick, dict) else None
if isinstance(_name, str) and _name:
_mode = "ANY"
_allowed = [_name]
if _mode is not None:
_fcc: dict[str, Any] = {"mode": _mode}
if _allowed:
_fcc["allowedFunctionNames"] = _allowed
body["toolConfig"] = {"functionCallingConfig": _fcc}
# Prompt caching. The Gemini contract is "create a CachedContent
# resource, then pass its name on `cachedContent`". The cache is created
# out of band by the caller via POST /cachedContents; here we forward an
# explicit cache id when the dispatcher hands us one (a string value on
# enable_prompt_caching means "use this cache name").
# https://ai.google.dev/gemini-api/docs/caching
if isinstance(enable_prompt_caching, str) and enable_prompt_caching:
body["cachedContent"] = enable_prompt_caching
# Model id is already validated at the top of _stream_gemini so we never
# reach a path-traversed URL segment here.
url = f"{self.base_url}/models/{model}:streamGenerateContent?alt=sse"
completion_id = f"chatcmpl-gemini-{model.replace('/', '-')}"
logger.info(
"Proxying Gemini streamGenerateContent to %s (model=%s, tools=%s, image=%s)",
url,
model,
[list(t.keys())[0] for t in tools_array] if tools_array else [],
is_image_model,
)
def _emit_tool_event(payload: dict[str, Any]) -> str:
_stamp_server_tool_marker(payload)
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": None,
}
],
"_toolEvent": payload,
}
return f"data: {_json.dumps(chunk)}"
def _text_chunk(text: str, extra_content: Optional[dict[str, Any]] = None) -> str:
delta: dict[str, Any] = {"content": text}
if extra_content:
delta["extra_content"] = extra_content
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": delta,
"finish_reason": None,
}
],
}
return f"data: {_json.dumps(chunk)}"
def _gemini_part_extra(part: dict[str, Any]) -> Optional[dict[str, Any]]:
"""Return ``{"google": {"thought_signature": ...}}`` when the Gemini
stream part carries a `thoughtSignature` we must replay on a
follow-up turn (Gemini 3 image editing + tool contexts both require
an exact signature echo)."""
sig = part.get("thoughtSignature") or part.get("thought_signature")
if isinstance(sig, str) and sig:
return {"google": {"thought_signature": sig}}
return None
# Gemini finish reasons -> OpenAI vocabulary. Reference:
# https://ai.google.dev/api/rest/v1beta/Candidate#FinishReason
_finish_reason_map: dict[str, Optional[str]] = {
"STOP": "stop",
"MAX_TOKENS": "length",
"SAFETY": "content_filter",
"RECITATION": "content_filter",
"PROHIBITED_CONTENT": "content_filter",
"BLOCKLIST": "content_filter",
"MALFORMED_FUNCTION_CALL": "stop",
"OTHER": "stop",
"FINISH_REASON_UNSPECIFIED": None,
}
last_usage: Optional[dict[str, Any]] = None
emitted_function_call_ids: set[str] = set()
# True once any Gemini functionCall part has been emitted so the final
# finish_reason swaps STOP -> tool_calls (matches the OpenAI Chat
# Completions contract; an OAI client that sees a tool_calls delta
# followed by finish_reason="stop" never executes the tool).
emitted_any_function_call = False
# Keyed on whether `googleSearch` was actually forwarded (not caller
# intent) so image-mode turns don't show a phantom "search complete".
web_search_active = any("googleSearch" in t for t in tools_array)
web_search_tool_id = "gemini_web_search"
web_search_tool_started = False
web_search_tool_ended = False
web_search_citations: list[dict[str, str]] = []
# tool_call_id minted on the most recent executableCode part so the
# matching codeExecutionResult closes out the same envelope. None
# between rounds.
gemini_code_exec_pending_id: Optional[str] = None
# The most recently emitted code_execution id + result text. Kept *after*
# the tool_end so a following inline image (matplotlib plot from
# codeExecution) can attach to the same card via a `__IMAGES__:` marker
# instead of spawning a separate image_generation event.
last_code_exec_tool_id: Optional[str] = None
last_code_exec_result_text: str = ""
try:
async with _http_client.stream(
"POST",
url,
json = body,
headers = self._auth_headers(),
timeout = self._stream_timeout,
) as response:
if response.status_code != 200:
error_body = await response.aread()
error_text = error_body.decode("utf-8", errors = "replace")
logger.error(
"Gemini returned %d: %s",
response.status_code,
error_text[:500],
)
yield _error_sse_line(response.status_code, error_text, self.provider_type)
return
if web_search_active:
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "web_search",
"tool_call_id": web_search_tool_id,
"arguments": {},
}
)
web_search_tool_started = True
# NOTE: same manual __anext__ loop as the other streaming
# helpers (see stream_chat_completion for the Python 3.13 +
# httpcore 1.0.x GeneratorExit ordering).
lines_gen = response.aiter_lines().__aiter__()
final_finish_reason: Optional[str] = None
try:
while True:
try:
line = await lines_gen.__anext__()
except StopAsyncIteration:
break
if not line.strip():
continue
if not line.startswith("data:"):
continue
data_str = line[len("data:") :].strip()
if not data_str or data_str == "[DONE]":
continue
try:
event = _json.loads(data_str)
except Exception:
logger.warning(
"Gemini: failed to parse SSE chunk: %s",
data_str[:200],
)
continue
if not isinstance(event, dict):
continue
# Latch usageMetadata across deltas -- the final fragment
# carries the complete totals.
usage_meta = event.get("usageMetadata")
if isinstance(usage_meta, dict):
last_usage = usage_meta
# Prompt-level safety block (zero candidates +
# promptFeedback.blockReason): surface as an error so the
# client doesn't see an empty successful response.
prompt_feedback = event.get("promptFeedback")
if isinstance(prompt_feedback, dict) and prompt_feedback.get("blockReason"):
block_reason = str(prompt_feedback.get("blockReason"))
# Close out the synthetic web_search start so the UI
# doesn't show a spinner stuck on "searching..."
# after the error toast lands.
if (
web_search_active
and web_search_tool_started
and not web_search_tool_ended
):
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": web_search_tool_id,
"result": (
"(search aborted: Gemini blocked "
f"prompt: {block_reason})"
),
}
)
web_search_tool_ended = True
yield _error_sse_line(
400,
f"Gemini blocked prompt: {block_reason}",
self.provider_type,
)
return
candidates = event.get("candidates") or []
if not isinstance(candidates, list):
continue
for cand in candidates:
if not isinstance(cand, dict):
continue
# Citations / grounding metadata.
# `groundingMetadata.groundingChunks[].web` carries
# `uri` + `title`. Collect for the tool_end emission
# at stream close.
gm = cand.get("groundingMetadata")
if isinstance(gm, dict) and web_search_active:
chunks_list = gm.get("groundingChunks") or []
if isinstance(chunks_list, list):
for ch in chunks_list:
if not isinstance(ch, dict):
continue
web = ch.get("web") or {}
if not isinstance(web, dict):
continue
u = web.get("uri") or ""
if not u or not isinstance(u, str):
continue
if any(c["url"] == u for c in web_search_citations):
continue
web_search_citations.append(
{
"url": u,
"title": (web.get("title") or u),
"snippet": "",
}
)
content_obj = cand.get("content") or {}
parts = (
content_obj.get("parts") if isinstance(content_obj, dict) else None
)
if isinstance(parts, list):
for part in parts:
if not isinstance(part, dict):
continue
# Text delta. Stow part-level
# `thoughtSignature` on the delta so Gemini 3
# turns needing an exact signature echo
# round-trip cleanly.
text = part.get("text")
_part_extra = _gemini_part_extra(part)
if isinstance(text, str) and text:
yield _text_chunk(
text,
extra_content = _part_extra,
)
elif _part_extra is not None and not any(
k in part
for k in (
"functionCall",
"executableCode",
"codeExecutionResult",
"inlineData",
)
):
# Empty-content part carrying a
# thoughtSignature: emit an empty delta to
# preserve the signature.
yield _text_chunk(
"",
extra_content = _part_extra,
)
# functionCall -> OpenAI tool_calls
# delta envelope.
fc = part.get("functionCall")
if isinstance(fc, dict):
fc_name = fc.get("name") or ""
fc_args = fc.get("args") or {}
fc_id = fc.get("id") or f"call_{fc_name}_{time.time_ns()}"
if fc_id in emitted_function_call_ids:
continue
emitted_function_call_ids.add(fc_id)
# Each functionCall needs its own
# tool_calls[*].index, else index-based
# consumers collapse parallel calls.
tc_index = len(emitted_function_call_ids) - 1
tool_call_delta: dict[str, Any] = {
"index": tc_index,
"id": fc_id,
"type": "function",
"function": {
"name": fc_name,
"arguments": _json.dumps(fc_args),
},
}
# Gemini 3 requires the part-level
# thoughtSignature echoed next turn; stow
# it on extra_content.google for replay.
thought_sig = part.get("thoughtSignature") or part.get(
"thought_signature"
)
if isinstance(thought_sig, str) and thought_sig:
tool_call_delta["extra_content"] = {
"google": {
"thought_signature": thought_sig,
}
}
emitted_any_function_call = True
tool_chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {"tool_calls": [tool_call_delta]},
"finish_reason": None,
}
],
}
yield f"data: {_json.dumps(tool_chunk)}"
# executableCode + codeExecutionResult parts
# surface as the standard code_execution
# tool_start/tool_end envelope (same shape
# OpenAI and Anthropic emit) so the chat
# adapter renders Gemini sandbox output
# through CodeExecutionToolUI.
# https://ai.google.dev/gemini-api/docs/code-execution
exec_code = part.get("executableCode")
if isinstance(exec_code, dict):
code_str = exec_code.get("code") or ""
if code_str:
code_tool_id = (
exec_code.get("id")
or f"gemini_code_exec_{time.time_ns()}"
)
gemini_code_exec_pending_id = code_tool_id
# Stow the raw Gemini part so
# follow-up turns can replay the
# native `executableCode` (Gemini
# rejects a generic functionCall echo
# for code execution history).
_exec_thought_sig = part.get(
"thoughtSignature"
) or part.get("thought_signature")
# Per-part thoughtSignature stays
# bound to its own part (Gemini 3
# rejects shared signatures).
_exec_part_entry: dict[str, Any] = {
"executableCode": exec_code,
}
if (
isinstance(_exec_thought_sig, str)
and _exec_thought_sig
):
_exec_part_entry["thoughtSignature"] = (
_exec_thought_sig
)
_exec_native: dict[str, Any] = {
"parts": [_exec_part_entry],
}
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "code_execution",
"tool_call_id": code_tool_id,
"arguments": {
"kind": "code_execution",
"language": (
(
exec_code.get("language")
or "PYTHON"
).lower()
),
"code": code_str,
"google": {
"native_part": _exec_native,
},
},
}
)
exec_result = part.get("codeExecutionResult")
if isinstance(exec_result, dict):
outcome = exec_result.get("outcome") or ""
output = exec_result.get("output") or ""
# Gemini returns OUTCOME_OK /
# OUTCOME_FAILED /
# OUTCOME_DEADLINE_EXCEEDED. Treat non-OK
# outcomes as stderr so the UI surfaces
# the error.
if outcome and outcome != "OUTCOME_OK":
result_text = f"[{outcome}]\n{output}".rstrip()
else:
result_text = output
# Pair tool_end with the most recent
# executableCode tool_start; else
# exec_result.id, then a fresh id.
pair_id = (
gemini_code_exec_pending_id
or exec_result.get("id")
or f"gemini_code_exec_{time.time_ns()}"
)
if gemini_code_exec_pending_id is None:
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "code_execution",
"tool_call_id": pair_id,
"arguments": {
"kind": "code_execution",
"code": "",
},
}
)
_result_thought_sig = part.get(
"thoughtSignature"
) or part.get("thought_signature")
_result_part_entry: dict[str, Any] = {
"codeExecutionResult": exec_result,
}
if (
isinstance(_result_thought_sig, str)
and _result_thought_sig
):
_result_part_entry["thoughtSignature"] = (
_result_thought_sig
)
_result_native: dict[str, Any] = {
"parts": [_result_part_entry],
}
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": pair_id,
"result": result_text,
"google": {
"native_part": _result_native,
},
}
)
last_code_exec_tool_id = pair_id
last_code_exec_result_text = result_text
gemini_code_exec_pending_id = None
# inlineData: either a Nano Banana generation
# (own card) or a sandbox plot attached to
# the code_execution card via the __IMAGES__:
# marker.
inline = part.get("inlineData")
if isinstance(inline, dict):
b64 = inline.get("data") or ""
mime = inline.get("mimeType") or "image/png"
if b64:
image_uri = f"data:{mime};base64,{b64}"
attached_to_code_exec = (
not is_image_model
and last_code_exec_tool_id is not None
and bool(enabled_tools)
and "code_execution" in (enabled_tools or [])
)
if attached_to_code_exec:
updated_result = (
last_code_exec_result_text
+ "\n__IMAGES__:"
+ _json.dumps([image_uri])
)
# Stow inlineData so a follow-up
# turn replays the plot with its
# per-part thoughtSignature.
_plot_thought_sig = part.get(
"thoughtSignature"
) or part.get("thought_signature")
_plot_part_entry: dict[str, Any] = {
"inlineData": {
"mimeType": mime,
"data": b64,
},
}
if (
isinstance(_plot_thought_sig, str)
and _plot_thought_sig
):
_plot_part_entry["thoughtSignature"] = (
_plot_thought_sig
)
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": (last_code_exec_tool_id),
"result": updated_result,
"google": {
"native_part": {
"parts": [_plot_part_entry],
},
},
}
)
last_code_exec_result_text = updated_result
else:
img_id = f"img_{time.time_ns()}"
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "image_generation",
"tool_call_id": img_id,
"arguments": {
"kind": "image",
"prompt": "",
},
}
)
# Gemini 3 image edit needs
# the prior thoughtSignature
# echoed on the inline image part.
_img_thought_sig = part.get(
"thoughtSignature"
) or part.get("thought_signature")
_img_tool_end: dict[str, Any] = {
"type": "tool_end",
"tool_call_id": img_id,
"result": "",
"image_b64": b64,
"image_mime": mime,
}
# Stow inlineData so multi-turn
# edits replay the original
# image as native history.
_img_part_entry: dict[str, Any] = {
"inlineData": {
"mimeType": mime,
"data": b64,
},
}
if (
isinstance(_img_thought_sig, str)
and _img_thought_sig
):
_img_part_entry["thoughtSignature"] = (
_img_thought_sig
)
_img_native: dict[str, Any] = {
"parts": [_img_part_entry],
}
_img_google: dict[str, Any] = {
"native_part": _img_native,
}
if (
isinstance(_img_thought_sig, str)
and _img_thought_sig
):
_img_google["thought_signature"] = (
_img_thought_sig
)
_img_tool_end["google"] = _img_google
yield _emit_tool_event(_img_tool_end)
finish_reason = cand.get("finishReason")
if isinstance(finish_reason, str):
mapped = _finish_reason_map.get(finish_reason, "stop")
if mapped is not None:
final_finish_reason = mapped
# End-of-stream order: web_search tool_end -> finish_reason ->
# usage -> [DONE], matching the Anthropic/OpenAI helpers.
if web_search_active and web_search_tool_started and not web_search_tool_ended:
blocks: list[str] = []
for cit in web_search_citations:
line_out = f"Title: {cit['title']}\nURL: {cit['url']}"
if cit.get("snippet"):
line_out += f"\nSnippet: {cit['snippet']}"
blocks.append(line_out)
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": web_search_tool_id,
"result": (
"\n---\n".join(blocks) if blocks else "(search complete)"
),
}
)
web_search_tool_ended = True
if final_finish_reason:
# Gemini emits "STOP" even for a pure functionCall turn;
# override to "tool_calls" so OAI clients run the tool.
if emitted_any_function_call and final_finish_reason == "stop":
final_finish_reason = "tool_calls"
finish_chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": final_finish_reason,
}
],
}
yield f"data: {_json.dumps(finish_chunk)}"
# Map Gemini usageMetadata onto OpenAI include_usage.
# thoughtsTokenCount is billed output too — fold it in so
# cost calculators don't undercount.
if isinstance(last_usage, dict):
thought_tokens = last_usage.get("thoughtsTokenCount") or 0
candidate_tokens = last_usage.get("candidatesTokenCount") or 0
prompt_tokens = last_usage.get("promptTokenCount") or 0
# Gemini bills tool-call prompt slices separately via
# `toolUsePromptTokenCount`. Fold into input so
# total_tokens doesn't undercount tool turns.
tool_use_prompt_tokens = last_usage.get("toolUsePromptTokenCount") or 0
translated_usage = {
"input_tokens": prompt_tokens + tool_use_prompt_tokens,
"output_tokens": candidate_tokens + thought_tokens,
"input_tokens_details": {
"cached_tokens": (last_usage.get("cachedContentTokenCount") or 0),
"tool_use_prompt_tokens": tool_use_prompt_tokens,
},
"output_tokens_details": {
"reasoning_tokens": thought_tokens,
},
}
usage_line = _build_usage_chunk(completion_id, "openai", translated_usage)
if usage_line:
yield usage_line
yield "data: [DONE]"
finally:
# Close response first so lines_gen.aclose() is a no-op.
await response.aclose()
await lines_gen.aclose()
except httpx.ConnectError as exc:
logger.error("Connection error to %s: %s", self.provider_type, exc)
if web_search_tool_started and not web_search_tool_ended:
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": web_search_tool_id,
"result": f"(search aborted: connection error: {exc})",
}
)
web_search_tool_ended = True
yield _error_sse_line(
502,
f"Failed to connect to {self.provider_type}: {exc}",
self.provider_type,
)
except httpx.ReadTimeout as exc:
logger.error("Read timeout from %s: %s", self.provider_type, exc)
if web_search_tool_started and not web_search_tool_ended:
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": web_search_tool_id,
"result": "(search aborted: read timeout)",
}
)
web_search_tool_ended = True
yield _error_sse_line(
504,
f"Timeout waiting for {self.provider_type} response",
self.provider_type,
)
except httpx.HTTPError as exc:
logger.error("HTTP error from %s: %s", self.provider_type, exc)
if web_search_tool_started and not web_search_tool_ended:
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": web_search_tool_id,
"result": f"(search aborted: transport error: {exc})",
}
)
web_search_tool_ended = True
yield _error_sse_line(
502,
f"Error communicating with {self.provider_type}: {exc}",
self.provider_type,
)
async def _stream_openai_responses(
self,
messages: list[dict[str, Any]],
model: str,
temperature: float,
top_p: float,
max_tokens: Optional[int],
enable_thinking: Optional[bool],
reasoning_effort: Optional[str],
enabled_tools: Optional[list[str]] = None,
enable_prompt_caching: Optional[bool] = None,
openai_code_exec_container_id: Optional[str] = None,
compaction_threshold: Optional[int] = None,
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[Any] = None,
) -> AsyncGenerator[str, None]:
"""
Call OpenAI's /v1/responses endpoint and translate its SSE stream back
into OpenAI Chat Completions chunk format.
The Responses API uses a different request shape (``input`` not
``messages``, ``instructions`` for system prompts, ``max_output_tokens``
for the budget) and emits event-typed SSE frames (e.g.
``response.output_text.delta``) rather than chat-completion chunks.
``presence_penalty`` / ``top_k`` aren't part of the Responses contract
and are dropped here.
"""
import json as _json
is_openai_cloud = _is_openai_family_cloud(self.base_url)
image_generation_requested = bool(
enabled_tools and "image_generation" in enabled_tools and is_openai_cloud
)
# Split system messages into a single `instructions` string and
# translate user/assistant messages into the Responses input shape.
instructions_parts: list[str] = []
input_items: list[dict[str, Any]] = []
# When we drop a server-side builtin `function_call` here, the matching
# `role="tool"` follow-up must also be dropped -- otherwise the outbound
# body has an orphan `function_call_output` with no matching
# `function_call`, which OpenAI Responses can reject or mis-associate.
skipped_server_builtin_call_ids: set[str] = set()
openai_replay_items: list[dict[str, Any]] = []
previous_response_id: Optional[str] = None
for msg in messages:
role = msg.get("role")
content = msg.get("content", "")
if role == "system":
if isinstance(content, str):
if content:
instructions_parts.append(content)
elif isinstance(content, list):
for part in content:
if part.get("type") == "text" and part.get("text"):
instructions_parts.append(part["text"])
continue
# Responses uses item-shape history: each assistant call is a
# `function_call` item and each role="tool" follow-up a
# `function_call_output` keyed by call_id (Chat Completions shape 400s).
if role == "tool":
_call_id = msg.get("tool_call_id") or ""
# If the matching assistant `function_call` was a server-side
# builtin we already dropped, drop the follow-up too to avoid
# an orphan `function_call_output`.
if _call_id and _call_id in skipped_server_builtin_call_ids:
continue
if isinstance(content, list):
_flat_parts: list[str] = []
for part in content:
if part.get("type") == "text" and part.get("text"):
_flat_parts.append(part["text"])
_output_text = "".join(_flat_parts)
else:
_output_text = content if isinstance(content, str) else ""
if _call_id:
input_items.append(
{
"type": "function_call_output",
"call_id": _call_id,
"output": _output_text,
}
)
continue
# Translate assistant tool_calls into `function_call` items, skipping
# server-side builtin cards (builtin name + `_server_tool` marker).
_tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else None
if role == "assistant" and isinstance(_tool_calls, list):
# Emit assistant text before its function_call items to preserve
# the original response.output ordering.
if isinstance(content, str) and content:
input_items.append({"role": "assistant", "content": content})
elif isinstance(content, list):
_asst_parts: list[dict[str, Any]] = []
for _part in content:
if not isinstance(_part, dict):
continue
_pt = _part.get("type")
if _pt == "text" and _part.get("text"):
_asst_parts.append(
{
"type": "input_text",
"text": _part.get("text", ""),
}
)
elif _pt == "image_url":
_u = _part.get("image_url", {}).get("url", "")
if _u:
_asst_parts.append({"type": "input_image", "image_url": _u})
if _asst_parts:
input_items.append({"role": "assistant", "content": _asst_parts})
for _tc in _tool_calls:
if not isinstance(_tc, dict):
continue
_fn = _tc.get("function") or {}
if not isinstance(_fn, dict) or not _fn.get("name"):
continue
_args_raw = _fn.get("arguments") or ""
if not isinstance(_args_raw, str):
try:
_args_raw = _json.dumps(_args_raw)
except Exception:
_args_raw = ""
_fn_name_lc = (_fn.get("name") or "").lower()
_is_server_builtin = False
if _fn_name_lc in _SERVER_SIDE_BUILTIN_TOOL_NAMES:
try:
_args_obj = _json.loads(_args_raw) if _args_raw else {}
except Exception:
_args_obj = None
if isinstance(_args_obj, dict):
if _args_obj.get("_server_tool") is True:
_is_server_builtin = True
else:
_g = _args_obj.get("google")
if isinstance(_g, dict) and isinstance(_g.get("native_part"), dict):
_is_server_builtin = True
_call_id_out = _tc.get("id") or f"call_{time.time_ns()}"
if _is_server_builtin:
skipped_server_builtin_call_ids.add(_call_id_out)
continue
input_items.append(
{
"type": "function_call",
"call_id": _call_id_out,
"name": _fn["name"],
"arguments": _args_raw,
}
)
# Assistant text already emitted above (in order) so we don't
# fall through to the generic content branches.
continue
if isinstance(content, str):
input_items.append({"role": role, "content": content})
continue
if isinstance(content, list):
translated_parts: list[dict[str, Any]] = []
used_previous_response_id = False
for part in content:
part_type = part.get("type")
if part_type == "text":
translated_parts.append(
{"type": "input_text", "text": part.get("text", "")}
)
elif part_type == "image_url":
url = part.get("image_url", {}).get("url", "")
if url:
# Responses takes image_url as a flat string (both
# https:// URLs and data: URLs are accepted).
translated_parts.append({"type": "input_image", "image_url": url})
elif (
part_type == "reasoning"
and role == "assistant"
and image_generation_requested
):
replay_item = _sanitize_openai_reasoning_replay_item(part)
if replay_item:
openai_replay_items.append(replay_item)
elif (
part_type == "image_generation_call"
and role == "assistant"
and image_generation_requested
):
response_id = (
part.get("response_id")
or part.get("openai_response_id")
or part.get("previous_response_id")
)
call_id = part.get("id") or part.get("image_generation_call_id")
if isinstance(call_id, str) and call_id:
if isinstance(response_id, str) and response_id:
previous_response_id = response_id
input_items = []
translated_parts = []
used_previous_response_id = True
else:
previous_response_id = None
openai_replay_items.append(
{"type": "image_generation_call", "id": call_id}
)
elif part_type == "input_document":
# Map Studio's `input_document` onto Responses' `input_file`.
# https://developers.openai.com/api/docs/guides/images-vision
file_url = part.get("file_url")
file_data = part.get("file_data")
filename = part.get("filename")
# Treat a "data:" URI with no base64 payload as missing
# (else file_data="" 400s) and fall back to file_url.
file_data_valid = bool(
isinstance(file_data, str)
and file_data
and (
not file_data.startswith("data:")
or file_data.partition(",")[2].strip()
)
)
block: dict[str, Any] = {"type": "input_file"}
if file_data_valid:
block["file_data"] = file_data
elif file_url:
block["file_url"] = file_url
else:
continue
if filename:
block["filename"] = filename
translated_parts.append(block)
if translated_parts and not used_previous_response_id:
input_items.append({"role": role, "content": translated_parts})
if previous_response_id:
# OpenAI's documented multi-turn image generation path can use
# `previous_response_id` to carry the prior generated image and
# paired reasoning state. Prefer that over manual item replay when we
# captured the response id; replay below is a fallback for older
# stored turns that only have an image_generation_call id.
openai_replay_items = []
elif (
_openai_image_replay_requires_reasoning(model)
and reasoning_effort != "none"
and enable_thinking is not False
):
filtered_replay_items: list[dict[str, Any]] = []
has_reasoning_replay = False
dropped_image_replay_without_reasoning = False
for item in openai_replay_items:
if item.get("type") == "reasoning":
has_reasoning_replay = True
filtered_replay_items.append(item)
elif item.get("type") == "image_generation_call":
if has_reasoning_replay:
filtered_replay_items.append(item)
else:
dropped_image_replay_without_reasoning = True
else:
filtered_replay_items.append(item)
openai_replay_items = filtered_replay_items
if dropped_image_replay_without_reasoning:
yield _error_sse_line(
400,
"OpenAI image edit reference is missing paired reasoning state. "
"Regenerate the image, then retry the edit.",
self.provider_type,
)
return
image_generation_has_reference = bool(
previous_response_id
or any(
isinstance(item, dict) and item.get("type") == "image_generation_call"
for item in openai_replay_items
)
)
if openai_replay_items:
insert_at = len(input_items)
for index in range(len(input_items) - 1, -1, -1):
if input_items[index].get("role") == "user":
insert_at = index
break
input_items[insert_at:insert_at] = openai_replay_items
# gpt-5.x / o3 / gpt-4.5 reject temperature/top_p (400 "Unsupported
# parameter"); the openai allowlist scopes the picker to these families,
# so never forward sampling knobs.
del temperature, top_p # accepted for API symmetry, not forwarded.
body: dict[str, Any] = {
"model": model,
"input": input_items,
"stream": True,
}
if previous_response_id:
body["previous_response_id"] = previous_response_id
# `summary: "auto"` is what makes /v1/responses emit reasoning summary
# events; without it the reasoning panel stays blank. Pair it with any
# explicit effort except "none".
summary_unsupported = bool(
_OPENAI_REASONING_SUMMARY_UNSUPPORTED.match(model.strip().lower())
)
if reasoning_effort in (
"minimal",
"low",
"medium",
"high",
"max",
"xhigh",
):
body["reasoning"] = {"effort": reasoning_effort}
if not summary_unsupported:
body["reasoning"]["summary"] = "auto"
elif reasoning_effort == "none" or enable_thinking is False:
body["reasoning"] = {"effort": "none"}
elif enable_thinking is True:
body["reasoning"] = {"effort": "medium"}
if not summary_unsupported:
body["reasoning"]["summary"] = "auto"
if instructions_parts:
body["instructions"] = "\n\n".join(instructions_parts)
if max_tokens is not None:
body["max_output_tokens"] = max_tokens
# Opt into 24h prompt-cache retention (free, vs the default ~5-10 min).
# Gated on the OpenAI cloud host because ollama / llama.cpp / "custom"
# presets reach this path too and would 400 on the unknown field.
if is_openai_cloud and enable_prompt_caching is not False:
body["prompt_cache_retention"] = "24h"
# Server-side context compaction (OpenAI cloud only).
# https://developers.openai.com/api/docs/guides/compaction
if is_openai_cloud and compaction_threshold is not None and compaction_threshold > 0:
body["context_management"] = [
{
"type": "compaction",
"compact_threshold": int(compaction_threshold),
}
]
# Map enabled_tools onto Responses-API server tools (cloud only;
# local OAI-compat backends 400 on these).
# https://developers.openai.com/api/docs/guides/tools
code_execution_enabled_openai = bool(
enabled_tools and "code_execution" in enabled_tools and is_openai_cloud
)
image_generation_enabled_openai = bool(
enabled_tools and "image_generation" in enabled_tools and is_openai_cloud
)
def _openai_image_generation_tool() -> dict[str, Any]:
tool: dict[str, Any] = {"type": "image_generation"}
if image_generation_has_reference:
# Force edit mode so the prior call id is used as context.
tool["action"] = "edit"
return tool
# Translate Chat-Completions function tools into the Responses
# function-tool shape (flat name/description/parameters).
responses_user_function_tools: list[dict[str, Any]] = []
if tools:
for _tool in tools:
if not isinstance(_tool, dict) or _tool.get("type") != "function":
continue
_fn = _tool.get("function")
if not isinstance(_fn, dict) or not _fn.get("name"):
continue
_entry: dict[str, Any] = {
"type": "function",
"name": _fn["name"],
}
if _fn.get("description"):
_entry["description"] = _fn["description"]
if isinstance(_fn.get("parameters"), dict):
_entry["parameters"] = _fn["parameters"]
responses_user_function_tools.append(_entry)
# Translate tool_choice into the Responses shape.
_responses_tc_string: Optional[str] = None
if isinstance(tool_choice, str):
_tc_lc = tool_choice.strip().lower()
if _tc_lc in ("auto", "none", "required"):
_responses_tc_string = _tc_lc
responses_tool_choice: Optional[Any] = None
_has_responses_tools = bool(enabled_tools or responses_user_function_tools)
if _responses_tc_string is not None and _has_responses_tools:
responses_tool_choice = _responses_tc_string
elif (
tool_choice is not None
and responses_user_function_tools
and isinstance(tool_choice, dict)
and tool_choice.get("type") == "function"
):
_fn_pick = tool_choice.get("function") or {}
_name = _fn_pick.get("name") if isinstance(_fn_pick, dict) else None
if isinstance(_name, str) and _name:
responses_tool_choice = {"type": "function", "name": _name}
_responses_tool_choice_none = _responses_tc_string == "none"
# A pinned user function suppresses hosted builtins (privacy +
# billing), matching the Gemini / Anthropic / OpenRouter gates.
_responses_tool_choice_forced_function = (
isinstance(tool_choice, dict)
and tool_choice.get("type") == "function"
and isinstance(tool_choice.get("function"), dict)
and bool(tool_choice["function"].get("name"))
)
_responses_hosted_builtins_allowed = (
not _responses_tool_choice_none and not _responses_tool_choice_forced_function
)
if (enabled_tools or responses_user_function_tools) and not _responses_tool_choice_none:
tools_array: list[dict[str, Any]] = list(responses_user_function_tools)
if (
_responses_hosted_builtins_allowed
and enabled_tools
and "web_search" in enabled_tools
):
tools_array.append({"type": "web_search"})
if _responses_hosted_builtins_allowed and code_execution_enabled_openai:
# Reuse the thread's container so filesystem state persists;
# auto-create when there isn't one yet. Stale ids 400 and are
# cleared via container_invalidated.
shell_env: dict[str, Any]
if openai_code_exec_container_id:
shell_env = {
"type": "container_reference",
"container_id": openai_code_exec_container_id,
}
else:
shell_env = {"type": "container_auto"}
tools_array.append({"type": "shell", "environment": shell_env})
if _responses_hosted_builtins_allowed and image_generation_enabled_openai:
tools_array.append(_openai_image_generation_tool())
if tools_array:
body["tools"] = tools_array
if responses_tool_choice is not None:
body["tool_choice"] = responses_tool_choice
url = f"{self.base_url}/responses"
completion_id = f"chatcmpl-openai-{model.replace('/', '-')}"
logger.info("Proxying OpenAI Responses API to %s (model=%s)", url, model)
def _build_body(container_id_for_this_attempt: Optional[str]) -> dict[str, Any]:
"""Snapshot of the request body. Called once for the initial attempt
and again with ``None`` for the post-expiry retry. Returns a fresh
dict so the retry doesn't share state with the first attempt.
"""
attempt_body = dict(body)
if (enabled_tools or responses_user_function_tools) and not _responses_tool_choice_none:
tools_array_attempt: list[dict[str, Any]] = list(responses_user_function_tools)
if (
_responses_hosted_builtins_allowed
and enabled_tools
and "web_search" in enabled_tools
):
tools_array_attempt.append({"type": "web_search"})
if _responses_hosted_builtins_allowed and code_execution_enabled_openai:
if container_id_for_this_attempt:
env_attempt: dict[str, Any] = {
"type": "container_reference",
"container_id": container_id_for_this_attempt,
}
else:
env_attempt = {"type": "container_auto"}
tools_array_attempt.append({"type": "shell", "environment": env_attempt})
if _responses_hosted_builtins_allowed and image_generation_enabled_openai:
tools_array_attempt.append(_openai_image_generation_tool())
if tools_array_attempt:
attempt_body["tools"] = tools_array_attempt
else:
attempt_body.pop("tools", None)
if responses_tool_choice is not None:
attempt_body["tool_choice"] = responses_tool_choice
return attempt_body
def _is_openai_container_expired_error(error_text: str) -> bool:
"""Substring-match OpenAI's expired/missing code-exec container errors
(no official error code exists)."""
lowered = error_text.lower()
if "container" not in lowered:
return False
return (
"expired" in lowered
or "not_found" in lowered
or "not found" in lowered
or "no such container" in lowered
)
try:
retried = False
attempt_container_id = openai_code_exec_container_id
while True:
attempt_body = _build_body(attempt_container_id)
async with _http_client.stream(
"POST",
url,
json = attempt_body,
headers = self._auth_headers(),
timeout = self._stream_timeout,
) as response:
if response.status_code != 200:
error_body = await response.aread()
error_text = error_body.decode("utf-8", errors = "replace")
logger.error(
"OpenAI Responses returned %d: %s",
response.status_code,
error_text[:500],
)
expired_container_4xx = (
attempt_container_id
and 400 <= response.status_code < 500
and _is_openai_container_expired_error(error_text)
)
if expired_container_4xx and not retried:
yield (
f"data: "
f"{_json.dumps({'id': completion_id, 'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {}, 'finish_reason': None}], '_toolEvent': {'type': 'container_invalidated'}})}"
)
retried = True
attempt_container_id = None
continue
yield _error_sse_line(response.status_code, error_text, self.provider_type)
return
# NOTE: same manual __anext__ loop as stream_chat_completion —
# see comment there for the GeneratorExit / aclose ordering.
lines_gen = response.aiter_lines().__aiter__()
done_emitted = False
reasoning_open = False
reasoning_emitted = False
# Per-call function-tool indexing; distinct slots so
# parallel calls don't collide on delta.tool_calls[].index.
saw_function_call = False
function_call_index = 0
# Latched from response.completed/incomplete; surfaces
# input_tokens_details.cached_tokens to prove cache hits.
last_usage: Optional[dict[str, Any]] = None
# web_search state. Citations are emitted on text deltas (not
# per call), so the aggregate list is shared and applied to
# the LAST web_search tool_end (parseSourcesFromResult
# flatmaps every call, one non-empty is enough).
web_search_calls: dict[str, dict[str, Any]] = {}
all_url_citations: list[dict[str, Any]] = []
# shell_calls (code execution): {call_id -> {commands, output}}.
# shell_call <-> shell_call_output match by call_id; emit
# tool_start/tool_end like the Anthropic UX.
shell_calls: dict[str, dict[str, Any]] = {}
# Container id latched from response.container_id or
# item.environment.container_id; emit container_ready when it
# differs from the inbound id.
latched_container_id: Optional[str] = None
container_id_emitted = False
current_openai_response_id: Optional[str] = None
last_openai_reasoning_replay_item: Optional[dict[str, Any]] = None
openai_reasoning_replay_items: dict[str, dict[str, Any]] = {}
image_generation_calls_started: set[str] = set()
# Buffer for a citation marker straddling two delta events;
# prepended onto the next delta. See _split_pending_citation_tail.
pending_marker_tail: str = ""
# Segments deferred while their markers reference unseen
# source_ids; held in arrival order so output never leapfrogs
# an earlier deferred segment. Flushed on annotation events
# and force-flushed at end-of-stream with leftover
# private-use codepoints stripped.
pending_citation_segments: list[str] = []
def _record_openai_response_id(payload: dict[str, Any]) -> None:
nonlocal current_openai_response_id
response_obj = payload.get("response")
candidates: list[Any] = []
if isinstance(response_obj, dict):
candidates.append(response_obj.get("id"))
candidates.append(payload.get("response_id"))
for candidate in candidates:
if isinstance(candidate, str) and candidate:
current_openai_response_id = candidate
return
def _drain_pending_segments(force: bool) -> str:
"""Re-attempt resolution on buffered segments in order.
Stops at the first still-unresolved segment unless
``force`` (end-of-stream), where lingering markers drop."""
out: list[str] = []
while pending_citation_segments:
seg = pending_citation_segments[0]
rewritten, unresolved = _rewrite_citation_markers_partial(
seg,
all_url_citations,
)
if unresolved and not force:
pending_citation_segments[0] = rewritten
break
if unresolved and force:
rewritten = _replace_openai_citation_markers(
rewritten,
all_url_citations,
)
pending_citation_segments.pop(0)
if rewritten:
out.append(rewritten)
return "".join(out)
def _flush_pending_marker_tail(tail: str) -> str:
"""Render any leftover citation tail at end-of-stream.
Unterminated tails drop (no annotation to bind to). If the
close byte arrived concatenated, rewrite then scrub any
residual private-use bytes and any orphan ``cite``
literal so the renderer never sees raw markup. url_citations
are aggregated separately and applied to web_search tool_end.
"""
if not tail:
return ""
if _OPENAI_CITE_STOP not in tail:
# Unterminated: drop the whole tail, else the residual
# ``cite`` would leak as plain text.
return ""
rendered = _replace_openai_citation_markers(tail, all_url_citations)
# Scrub residual private-use bytes (e.g. a partial opener).
for ch in ("", "", ""):
rendered = rendered.replace(ch, "")
# Drop any orphan ``cite`` literal -- meaningless
# without its closing byte and matching url_citation.
rendered = re.sub(r"^cite\S*", "", rendered)
return rendered
def _emit_tool_event(payload: dict[str, Any]) -> str:
_stamp_server_tool_marker(payload)
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": None,
}
],
"_toolEvent": payload,
}
return f"data: {_json.dumps(chunk)}"
def _format_shell_output(output: Any) -> str:
"""Render `shell_call_output.output` (stdout/stderr/outcome
per entry) as the preformatted text CodeExecutionToolUI
shows; append return_code/(timeout) only when informative."""
if not isinstance(output, list):
return ""
parts: list[str] = []
for entry in output:
if not isinstance(entry, dict):
continue
stdout = entry.get("stdout") or ""
stderr = entry.get("stderr") or ""
outcome = entry.get("outcome") or {}
chunk_parts: list[str] = []
if stdout:
chunk_parts.append(stdout)
if stderr:
chunk_parts.append(f"--- stderr ---\n{stderr}")
if isinstance(outcome, dict):
outcome_type = outcome.get("type")
if outcome_type == "exit":
exit_code = outcome.get("exit_code")
if isinstance(exit_code, int) and exit_code != 0:
chunk_parts.append(f"return_code: {exit_code}")
elif outcome_type == "timeout":
chunk_parts.append("(timeout)")
if chunk_parts:
parts.append("\n".join(chunk_parts))
return "\n--- next command ---\n".join(parts) if parts else "(no output)"
def _record_url_citation(payload: dict[str, Any]) -> None:
"""Append a url_citation, deduped by URL: collect every
source_id alias onto the entry's ``source_ids`` so the
rewriter can resolve any alias. The id lives under
source_id/id/locator across API revisions."""
if payload.get("type") != "url_citation":
return
url = payload.get("url", "")
if not url:
return
source_id = (
payload.get("source_id")
or payload.get("id")
or payload.get("locator")
or ""
)
# Single pass: either backfill aliases onto an existing
# URL entry (and return) or fall through to append a
# fresh one.
for c in all_url_citations:
if c["url"] != url:
continue
if source_id:
aliases = c.setdefault("source_ids", [])
if source_id not in aliases:
aliases.append(source_id)
return
title = payload.get("title") or url
snippet = payload.get("snippet") or payload.get("quote") or ""
all_url_citations.append(
{
"url": url,
"title": title,
"snippet": snippet,
"source_ids": [source_id] if source_id else [],
}
)
def _record_openai_reasoning_replay_item(
payload: Any,
) -> Optional[dict[str, Any]]:
if not isinstance(payload, dict):
return None
item_id = payload.get("id") or payload.get("item_id")
if not isinstance(item_id, str) or not item_id:
return None
existing = openai_reasoning_replay_items.setdefault(
item_id,
{
"type": "reasoning",
"id": item_id,
"summary": [],
"status": "completed",
},
)
if payload.get("type") == "reasoning":
sanitized = _sanitize_openai_reasoning_replay_item(payload)
if sanitized:
existing.update(sanitized)
return existing
summary_text = ""
part = payload.get("part")
if isinstance(part, dict) and part.get("type") == "summary_text":
text = part.get("text")
if isinstance(text, str):
summary_text = text
elif payload.get("type") == "response.reasoning_summary_text.done":
text = payload.get("text")
if isinstance(text, str):
summary_text = text
if summary_text:
summary_index = payload.get("summary_index")
summary = existing.setdefault("summary", [])
if isinstance(summary, list):
summary_part = {
"type": "summary_text",
"text": summary_text,
}
if isinstance(summary_index, int) and summary_index >= 0:
while len(summary) <= summary_index:
summary.append({"type": "summary_text", "text": ""})
summary[summary_index] = summary_part
else:
summary.append(summary_part)
return existing
def _image_generation_arguments(
prompt: str, raw_item_id: Any
) -> dict[str, Any]:
arguments: dict[str, Any] = {"kind": "image", "prompt": prompt}
if isinstance(raw_item_id, str) and raw_item_id:
arguments["openai_image_generation_call_id"] = raw_item_id
if current_openai_response_id:
arguments["openai_response_id"] = current_openai_response_id
if last_openai_reasoning_replay_item:
arguments["openai_reasoning_item"] = last_openai_reasoning_replay_item
return arguments
def _extract_reasoning_text(payload: Any) -> str:
if payload is None:
return ""
if isinstance(payload, str):
return payload
if isinstance(payload, list):
out: list[str] = []
for item in payload:
text = _extract_reasoning_text(item)
if text:
out.append(text)
return "".join(out)
if isinstance(payload, dict):
# OpenAI responses carry reasoning summaries in
# different envelope fields across event variants.
for key in ("text", "delta", "content", "summary"):
if key in payload:
text = _extract_reasoning_text(payload.get(key))
if text:
return text
if payload.get("type") == "summary_text":
return _extract_reasoning_text(payload.get("text"))
return ""
def _chunk_with_text(text: str) -> str:
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {"content": text},
"finish_reason": None,
}
],
}
return f"data: {_json.dumps(chunk)}"
try:
while True:
try:
line = await lines_gen.__anext__()
except StopAsyncIteration:
break
if not line or line.startswith("event:"):
continue
if not line.startswith("data:"):
continue
data_str = line[len("data:") :].strip()
if not data_str:
continue
if data_str == "[DONE]":
# Flush any held-over partial marker; strip
# private-use bytes so garbled glyphs don't leak.
if pending_marker_tail:
flushed = _flush_pending_marker_tail(pending_marker_tail)
pending_marker_tail = ""
if flushed:
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
yield _chunk_with_text(flushed)
# Force-drain any segment still awaiting an
# annotation; lingering codepoints drop.
tail_flushed = _drain_pending_segments(
force = True,
)
if tail_flushed:
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
yield _chunk_with_text(tail_flushed)
if not done_emitted:
yield "data: [DONE]"
done_emitted = True
break
try:
event = _json.loads(data_str)
except _json.JSONDecodeError:
continue
event_type = event.get("type")
_record_openai_response_id(event)
if event_type == "response.output_text.delta":
delta_text = event.get("delta", "")
# Process inline annotations first so source_ids
# referenced by same-delta markers are in the
# lookup before the rewriter runs. Some API
# versions inline url citations on the delta event.
for ann in event.get("annotations") or []:
if isinstance(ann, dict):
_record_url_citation(ann)
if delta_text or pending_marker_tail:
# Prepend any held-over tail so a marker
# straddling two SSE events resolves cleanly.
combined = pending_marker_tail + delta_text
head, pending_marker_tail = _split_pending_citation_tail(
combined
)
if head:
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
# Re-attempt earlier deferred segments
# first so output stays in order; the
# needed annotation may have arrived
# inline above.
flushed = _drain_pending_segments(
force = False,
)
if flushed:
yield _chunk_with_text(flushed)
head_rewritten, has_unresolved = (
_rewrite_citation_markers_partial(
head,
all_url_citations,
)
)
if has_unresolved or pending_citation_segments:
pending_citation_segments.append(head_rewritten)
elif head_rewritten:
yield _chunk_with_text(head_rewritten)
elif event_type == "response.output_text.annotation.added":
ann = event.get("annotation")
if isinstance(ann, dict):
_record_url_citation(ann)
flushed = _drain_pending_segments(
force = False,
)
if flushed:
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
yield _chunk_with_text(flushed)
elif event_type == "response.output_item.added":
item = event.get("item", {})
if isinstance(item, dict) and item.get("type") == "web_search_call":
item_id = item.get("id", "") or (f"ws_{len(web_search_calls)}")
web_search_calls.setdefault(item_id, {"query": ""})
# Register shell_call eagerly so out-of-order
# output links back. Probe env.container_id to
# emit container_ready before response.completed.
if isinstance(item, dict) and item.get("type") == "shell_call":
item_id = item.get("id", "") or (f"sc_{len(shell_calls)}")
shell_calls.setdefault(
item_id,
{"commands": [], "output": None},
)
env = item.get("environment")
if isinstance(env, dict):
probe = env.get("container_id") or env.get("id")
if (
isinstance(probe, str)
and probe.startswith("cntr_")
and latched_container_id is None
):
latched_container_id = probe
if (
isinstance(item, dict)
and item.get("type") == "image_generation_call"
):
raw_item_id = item.get("id")
if isinstance(raw_item_id, str) and raw_item_id:
arguments = _image_generation_arguments(
"",
raw_item_id,
)
image_generation_calls_started.add(raw_item_id)
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "image_generation",
"tool_call_id": raw_item_id,
"arguments": arguments,
}
)
elif event_type == "response.output_item.done":
item = event.get("item", {})
if not isinstance(item, dict):
continue
if item.get("type") == "reasoning":
last_openai_reasoning_replay_item = (
_record_openai_reasoning_replay_item(item)
)
summary_text = _extract_reasoning_text(item.get("summary"))
if summary_text and not reasoning_emitted:
if not reasoning_open:
summary_text = f"{summary_text}"
reasoning_open = True
yield _chunk_with_text(summary_text)
reasoning_emitted = True
elif item.get("type") == "web_search_call":
# done carries the query; emit tool_start +
# tool_end here. Citations are aggregated and
# the last call's result is overwritten at
# response.completed.
item_id = item.get("id", "") or (f"ws_{len(web_search_calls)}")
action = item.get("action")
query = (
action.get("query", "") if isinstance(action, dict) else ""
)
web_search_calls[item_id] = {"query": query}
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "web_search",
"tool_call_id": item_id,
"arguments": ({"query": query} if query else {}),
}
)
# Per-card text; last call gets overwritten
# with citations at response.completed.
per_call_result = f"Searching: {query}" if query else ""
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": item_id,
"result": per_call_result,
}
)
elif item.get("type") == "shell_call":
# Join the action.commands array into one
# newline-separated string (the card renderer,
# shared with Anthropic bash, wants a single
# `command`).
item_id = item.get("id", "") or (f"sc_{len(shell_calls)}")
action = item.get("action") or {}
commands = (
action.get("commands") if isinstance(action, dict) else None
) or []
joined_command = (
"\n".join(str(c) for c in commands)
if isinstance(commands, list)
else ""
)
shell_calls.setdefault(
item_id,
{
"commands": [],
"output": None,
"tool_end_emitted": False,
},
)
shell_calls[item_id]["commands"] = (
list(commands) if isinstance(commands, list) else []
)
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "code_execution",
"tool_call_id": item_id,
"arguments": {
"kind": "bash",
"command": joined_command,
},
}
)
# Fallback: output may be bundled on the
# shell_call done event itself.
embedded_output = item.get("output")
if isinstance(embedded_output, list) and embedded_output:
shell_calls[item_id]["output"] = embedded_output
shell_calls[item_id]["tool_end_emitted"] = True
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": item_id,
"result": _format_shell_output(embedded_output),
}
)
elif item.get("type") == "shell_call_output":
# `call_id` links back to the shell_call's
# `id`, used as the tool_call_id on
# tool_start. Match on call_id when present so
# the matching card transitions to complete.
call_id = item.get("call_id") or item.get("id") or ""
output = item.get("output") or []
# Skip if bundled-output path already
# finalised this card.
if shell_calls.get(call_id, {}).get("tool_end_emitted"):
continue
if call_id in shell_calls:
shell_calls[call_id]["output"] = output
shell_calls[call_id]["tool_end_emitted"] = True
result_text = _format_shell_output(output)
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": call_id,
"result": result_text,
}
)
elif item.get("type") == "image_generation_call":
# Base64 image on `result` (or `b64_json`),
# `revised_prompt` for the rewritten prompt.
# ns-resolution id so concurrent gens are unique.
raw_item_id = item.get("id")
item_id = raw_item_id or f"img_{time.time_ns()}"
prompt_in = (
item.get("revised_prompt") or item.get("prompt") or ""
)
done_arguments = _image_generation_arguments(
prompt_in,
raw_item_id,
)
if item_id not in image_generation_calls_started:
yield _emit_tool_event(
{
"type": "tool_start",
"tool_name": "image_generation",
"tool_call_id": item_id,
"arguments": done_arguments,
}
)
b64 = item.get("result") or item.get("b64_json") or ""
output_format = item.get("output_format") or "png"
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": item_id,
"result": "",
"arguments": done_arguments,
"image_b64": b64,
"image_mime": (f"image/{output_format}"),
"size": item.get("size"),
"quality": item.get("quality"),
"background": item.get("background"),
"prompt": prompt_in,
}
)
elif item.get("type") == "function_call":
# Translate to Chat-Completions delta.tool_calls.
# https://platform.openai.com/docs/guides/function-calling?api-mode=responses
fn_call_id = (
item.get("call_id")
or item.get("id")
or f"call_{time.time_ns()}"
)
fn_name = item.get("name") or ""
fn_args = item.get("arguments") or ""
if not isinstance(fn_args, str):
try:
fn_args = _json.dumps(fn_args)
except Exception:
fn_args = ""
_tc_index = function_call_index
function_call_index += 1
yield (
"data: "
+ _json.dumps(
{
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{
"index": _tc_index,
"id": fn_call_id,
"type": "function",
"function": {
"name": fn_name,
"arguments": (fn_args),
},
}
],
},
"finish_reason": None,
}
],
}
)
)
saw_function_call = True
elif isinstance(event_type, str) and "reasoning" in event_type:
recorded_reasoning = _record_openai_reasoning_replay_item(event)
if recorded_reasoning:
last_openai_reasoning_replay_item = recorded_reasoning
reasoning_delta = _extract_reasoning_text(event)
if reasoning_delta:
if not reasoning_open:
reasoning_delta = f"{reasoning_delta}"
reasoning_open = True
yield _chunk_with_text(reasoning_delta)
reasoning_emitted = True
elif event_type == "response.completed":
completed_usage = (event.get("response") or {}).get("usage")
if isinstance(completed_usage, dict):
last_usage = completed_usage
# Flush any unterminated citation tail; by now all
# annotations are recorded, else private-use bytes
# are stripped.
if pending_marker_tail:
flushed = _flush_pending_marker_tail(pending_marker_tail)
pending_marker_tail = ""
if flushed:
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
yield _chunk_with_text(flushed)
# Force-drain segments still awaiting an annotation.
tail_flushed = _drain_pending_segments(
force = True,
)
if tail_flushed:
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
yield _chunk_with_text(tail_flushed)
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
# Scan response.container_id and response.container.id
# (docs don't pin the field); emit container_ready
# only when it differs from the inbound id.
response_obj = event.get("response") or {}
if isinstance(response_obj, dict):
probe_id = response_obj.get("container_id")
if not probe_id:
container_field = response_obj.get("container")
if isinstance(container_field, dict):
probe_id = container_field.get("id")
if (
isinstance(probe_id, str)
and probe_id.startswith("cntr_")
and latched_container_id is None
):
latched_container_id = probe_id
if (
latched_container_id
and not container_id_emitted
and latched_container_id != openai_code_exec_container_id
):
yield _emit_tool_event(
{
"type": "container_ready",
"container_id": latched_container_id,
}
)
container_id_emitted = True
# Overwrite the last web_search card with the
# citation list (the extractor flatMaps cards).
if web_search_calls and all_url_citations:
last_id = list(web_search_calls.keys())[-1]
blocks: list[str] = []
for cit in all_url_citations:
line = f"Title: {cit['title']}\nURL: {cit['url']}"
if cit.get("snippet"):
line += f"\nSnippet: {cit['snippet']}"
blocks.append(line)
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": last_id,
"result": "\n---\n".join(blocks),
}
)
# Final flush: finalise any orphan shell_call
# so the card stops spinning.
for sc_id, sc_state in shell_calls.items():
if sc_state.get("tool_end_emitted"):
continue
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": sc_id,
"result": _format_shell_output(
sc_state.get("output") or []
),
}
)
sc_state["tool_end_emitted"] = True
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": (
"tool_calls" if saw_function_call else "stop"
),
}
],
}
yield f"data: {_json.dumps(chunk)}"
# Emit include_usage-style chunk after the
# finish_reason so callers can surface
# cached_tokens in their UI.
usage_line = _build_usage_chunk(
completion_id,
"openai",
last_usage,
)
if usage_line:
yield usage_line
elif event_type == "response.incomplete":
incomplete_usage = (event.get("response") or {}).get("usage")
if isinstance(incomplete_usage, dict):
last_usage = incomplete_usage
# Same flush as response.completed -- truncated
# streams can leave a half-marker in the buffer.
if pending_marker_tail:
flushed = _flush_pending_marker_tail(pending_marker_tail)
pending_marker_tail = ""
if flushed:
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
yield _chunk_with_text(flushed)
# Force-drain any segment still awaiting an
# annotation; lingering codepoints drop.
tail_flushed = _drain_pending_segments(
force = True,
)
if tail_flushed:
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
yield _chunk_with_text(tail_flushed)
if reasoning_open:
yield _chunk_with_text("")
reasoning_open = False
# Same citation backfill as response.completed.
if web_search_calls and all_url_citations:
last_id = list(web_search_calls.keys())[-1]
blocks = []
for cit in all_url_citations:
line = f"Title: {cit['title']}\nURL: {cit['url']}"
if cit.get("snippet"):
line += f"\nSnippet: {cit['snippet']}"
blocks.append(line)
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": last_id,
"result": "\n---\n".join(blocks),
}
)
# Mirror the response.completed flush so
# truncated streams also finalise orphan
# shell_calls.
for sc_id, sc_state in shell_calls.items():
if sc_state.get("tool_end_emitted"):
continue
yield _emit_tool_event(
{
"type": "tool_end",
"tool_call_id": sc_id,
"result": _format_shell_output(
sc_state.get("output") or []
),
}
)
sc_state["tool_end_emitted"] = True
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": "length",
}
],
}
yield f"data: {_json.dumps(chunk)}"
# Emit include_usage-style chunk after the
# length-truncated finish_reason too, so
# incomplete responses still report cached_tokens.
usage_line = _build_usage_chunk(
completion_id,
"openai",
last_usage,
)
if usage_line:
yield usage_line
elif event_type in ("response.failed", "error"):
# Surface the failure to the client; the outer
# route emits [DONE] as part of its cleanup.
error_payload = event.get("response", {}).get("error", {}) or {
"message": event.get("message", "Unknown error"),
"code": event.get("code"),
}
yield _error_sse_line(
502,
_json.dumps(error_payload),
self.provider_type,
)
break
except GeneratorExit:
await response.aclose()
await lines_gen.aclose()
raise
finally:
# Per-turn tool summary for triage.
web_search_requested = bool(enabled_tools and "web_search" in enabled_tools)
web_search_invocations = len(web_search_calls)
total_citations = len(all_url_citations)
queries = [
sc["query"] for sc in web_search_calls.values() if sc.get("query")
]
# On /v1/responses cached tokens live at
# usage.input_tokens_details.cached_tokens (not
# prompt_tokens_details, the chat/completions shape).
cached_input_tokens = None
if isinstance(last_usage, dict):
details = last_usage.get("input_tokens_details")
if isinstance(details, dict):
cached_input_tokens = details.get("cached_tokens")
code_execution_requested = code_execution_enabled_openai
code_execution_invocations = len(shell_calls)
code_execution_results = sum(
1 for sc in shell_calls.values() if sc.get("output") is not None
)
logger.info(
"OpenAI Responses stream complete (model=%s, "
"web_search_requested=%s, web_search_invocations=%s, "
"citations=%s, queries=%s, reasoning_emitted=%s, "
"code_execution_requested=%s, "
"code_execution_invocations=%s, "
"code_execution_results=%s, "
"container_id_in=%s, container_id_out=%s, "
"input_tokens=%s, output_tokens=%s, "
"cached_input_tokens=%s)",
model,
web_search_requested,
web_search_invocations,
total_citations,
queries,
reasoning_emitted,
code_execution_requested,
code_execution_invocations,
code_execution_results,
openai_code_exec_container_id,
latched_container_id,
(last_usage or {}).get("input_tokens"),
(last_usage or {}).get("output_tokens"),
cached_input_tokens,
)
await response.aclose()
await lines_gen.aclose()
return
except httpx.ConnectError as exc:
logger.error("Connection error to %s: %s", self.provider_type, exc)
yield _error_sse_line(
502,
f"Failed to connect to {self.provider_type}: {exc}",
self.provider_type,
)
except httpx.ReadTimeout as exc:
logger.error("Read timeout from %s: %s", self.provider_type, exc)
yield _error_sse_line(
504,
f"Timeout waiting for {self.provider_type} response",
self.provider_type,
)
except httpx.HTTPError as exc:
logger.error("HTTP error from %s: %s", self.provider_type, exc)
yield _error_sse_line(
502,
f"Error communicating with {self.provider_type}: {exc}",
self.provider_type,
)
async def chat_completion(
self,
messages: list[dict[str, Any]],
model: str,
temperature: float = 0.7,
top_p: float = 0.95,
max_tokens: Optional[int] = None,
presence_penalty: float = 0.0,
) -> dict[str, Any]:
"""Non-streaming chat completion. Returns the full response dict.
Only valid for OpenAI-compatible providers. Anthropic requires its own
Messages API; use stream_chat_completion (with stream=False) if a
non-streaming Anthropic path is needed later.
"""
body: dict[str, Any] = {
"model": model,
"messages": messages,
"stream": False,
"temperature": temperature,
"top_p": top_p,
"presence_penalty": presence_penalty,
}
if max_tokens is not None:
if self.provider_type == "openai":
body["max_completion_tokens"] = max_tokens
else:
body["max_tokens"] = max_tokens
response = await _http_client.post(
f"{self.base_url}/chat/completions",
json = body,
headers = self._auth_headers(),
timeout = self._timeout,
)
response.raise_for_status()
return response.json()
async def list_models(self) -> list[dict[str, Any]]:
"""GET /models to discover available models.
Returns dicts with at least 'id'. All providers expose /models with the
OpenAI {"data": [...]} shape (Anthropic included:
https://api.anthropic.com/v1/models).
"""
try:
response = await _http_client.get(
f"{self.base_url}/models",
headers = self._auth_headers(),
timeout = self._timeout,
)
response.raise_for_status()
data = response.json()
# Some local servers (Ollama with no models) return data: null.
models: list[dict[str, Any]] = []
if isinstance(data, dict):
raw_models = data.get("data") or []
if isinstance(raw_models, list):
models = [model for model in raw_models if isinstance(model, dict)]
if not models and self.provider_type == "ollama":
models = await self._list_ollama_native_models()
# Gemini's native /v1beta/models uses a different shape; repackage
# into the OpenAI-compatible one Studio expects.
if not models and self.provider_type == "gemini":
models = self._parse_gemini_models(data)
return models
except httpx.HTTPError as exc:
logger.error("Failed to list models from %s: %s", self.provider_type, exc)
raise
@staticmethod
def _parse_gemini_models(payload: Any) -> list[dict[str, Any]]:
"""Translate Gemini's native /v1beta/models payload to OpenAI shape,
keeping only entries advertising generateContent / streamGenerateContent
so embedding-only models don't reach the chat picker.
"""
if not isinstance(payload, dict):
return []
entries = payload.get("models") or []
if not isinstance(entries, list):
return []
out: list[dict[str, Any]] = []
for entry in entries:
if not isinstance(entry, dict):
continue
methods = entry.get("supportedGenerationMethods") or []
if (
isinstance(methods, list)
and methods
and not any(m in methods for m in ("generateContent", "streamGenerateContent"))
):
continue
base_id = entry.get("baseModelId")
name = entry.get("name") or ""
# ``name`` arrives as ``"models/gemini-2.5-flash"``; the chat path
# uses the bare id.
short_id = (
base_id
if isinstance(base_id, str) and base_id
else (name.split("/", 1)[1] if "/" in name else name)
)
if not short_id:
continue
out.append(
{
"id": short_id,
"owned_by": "google",
"display_name": entry.get("displayName") or short_id,
}
)
return out
async def _list_ollama_native_models(self) -> list[dict[str, Any]]:
"""Fallback when Ollama's /v1/models returns an empty or null catalog."""
root = self.base_url.removesuffix("/v1").rstrip("/")
response = await _http_client.get(
f"{root}/api/tags",
headers = self._auth_headers(),
timeout = self._timeout,
)
response.raise_for_status()
payload = response.json()
if not isinstance(payload, dict):
return []
raw_models = payload.get("models") or []
if not isinstance(raw_models, list):
return []
return [
{"id": entry.get("name", "").strip(), "owned_by": "ollama"}
for entry in raw_models
if isinstance(entry, dict) and entry.get("name", "").strip()
]
async def verify_models_endpoint_lightweight(self) -> None:
"""
Confirm GET /models returns 200 without buffering the full response body.
Used for providers with enormous catalogs (e.g. OpenRouter, Hugging Face
router) where downloading the full JSON would be prohibitive.
"""
url = f"{self.base_url}/models"
try:
async with _http_client.stream(
"GET",
url,
headers = self._auth_headers(),
timeout = self._timeout,
) as response:
if response.status_code != 200:
response.raise_for_status()
async for _chunk in response.aiter_bytes(chunk_size = 2048):
break
except httpx.HTTPError as exc:
logger.error(
"Lightweight /models check failed for %s: %s",
self.provider_type,
exc,
)
raise
def _container_headers(self) -> dict[str, str]:
"""Auth headers plus the required ``OpenAI-Beta: containers=v1`` opt-in;
without it DELETE silently no-ops (returns deleted:true but keeps the
container, verified 2026-05-15)."""
headers = self._auth_headers()
headers["OpenAI-Beta"] = "containers=v1"
return headers
async def list_openai_containers(self) -> list[dict[str, Any]]:
"""GET /v1/containers; returns raw container records (the route reshapes
them). Only valid against api.openai.com (caller guards is_openai_cloud).
"""
response = await _http_client.get(
f"{self.base_url}/containers",
headers = self._container_headers(),
timeout = self._timeout,
)
response.raise_for_status()
data = response.json()
containers = data.get("data") if isinstance(data, dict) else None
result = list(containers) if isinstance(containers, list) else []
logger.info(
"openai_container_list.response count=%s items=%s",
len(result),
[{"id": c.get("id"), "status": c.get("status")} for c in result if isinstance(c, dict)],
)
return result
async def create_openai_container(self, name: str, ttl_minutes: int) -> dict[str, Any]:
"""
POST /v1/containers with ``expires_after.anchor="last_active_at"``.
``ttl_minutes`` is the idle timeout — every API call touching the
container resets the timer.
"""
body = {
"name": name,
"expires_after": {
"anchor": "last_active_at",
"minutes": ttl_minutes,
},
}
response = await _http_client.post(
f"{self.base_url}/containers",
json = body,
headers = self._container_headers(),
timeout = self._timeout,
)
response.raise_for_status()
return response.json()
async def delete_openai_container(self, container_id: str) -> None:
"""DELETE /v1/containers/{id}. 404s surface as HTTPError.
Uses a fresh httpx client (shared-pool DELETEs returned deleted:true but
left the container alive). Also verifies the body reports deleted:true,
since OpenAI 2xx-returns that even when silently rejecting the request.
"""
url = f"{self.base_url}/containers/{container_id}"
headers = self._container_headers()
logger.info(
"openai_container_delete.outbound url=%s has_auth=%s openai_beta=%s",
url,
"Authorization" in headers,
headers.get("OpenAI-Beta"),
)
async with httpx.AsyncClient(timeout = self._timeout) as fresh_client:
response = await fresh_client.delete(url, headers = headers)
logger.info(
"openai_container_delete.response status=%s cf_ray=%s "
"request_id=%s organization=%s project=%s processing_ms=%s body=%s",
response.status_code,
response.headers.get("cf-ray"),
response.headers.get("x-request-id"),
response.headers.get("openai-organization"),
response.headers.get("openai-project"),
response.headers.get("openai-processing-ms"),
response.text[:300],
)
response.raise_for_status()
try:
payload = response.json()
except ValueError:
payload = None
if not (isinstance(payload, dict) and payload.get("deleted") is True):
raise httpx.HTTPError(
f"OpenAI did not confirm container deletion: {response.text[:200]}"
)
async def close(self) -> None:
"""No-op — the underlying client is shared across requests."""
def _provider_display_name(provider_type: str) -> str:
from core.inference.providers import get_provider_info
info = get_provider_info(provider_type) or {}
return str(info.get("display_name") or provider_type)
def _friendly_provider_error_text(
provider_type: str,
status_code: int,
raw_message: str,
*,
model: str | None = None,
) -> str:
"""Rewrite common provider errors into actionable Studio copy."""
if status_code == 404 and model:
lowered = raw_message.lower()
if "not found" in lowered or "not_found" in lowered:
if provider_type == "ollama":
label = _provider_display_name(provider_type)
return (
f"Model '{model}' is not installed in {label}. "
f"Run `ollama pull {model}` in a terminal, then retry."
)
if provider_type in ("vllm", "llama_cpp"):
label = _provider_display_name(provider_type)
return (
f"Model '{model}' is not available on the {label} server. "
"Check that the server is running and the model is loaded, "
"then retry."
)
return raw_message
def _error_sse_line(status_code: int, message: str, provider_type: str) -> str:
"""Format an error as an SSE data line in OpenAI error format."""
import json
error_obj = {
"error": {
"message": message,
"type": "provider_error",
"code": str(status_code),
"provider": provider_type,
}
}
return f"data: {json.dumps(error_obj)}"
def _build_usage_chunk(
completion_id: str, provider: Literal["anthropic", "openai"], last_usage: Optional[dict]
) -> Optional[str]:
"""Build an OpenAI ``include_usage``-style SSE chunk carrying upstream
prompt-cache accounting back to the client.
Emits the standard chunk shape (``choices: []`` + ``usage`` block) so
``stream_options={"include_usage": true}`` clients keep working, plus the
Anthropic-native counts as extra keys:
usage.prompt_tokens_details.cached_tokens (both providers)
usage.cache_creation_input_tokens (Anthropic-only)
usage.cache_read_input_tokens (Anthropic-only)
Anthropic's ``input_tokens`` excludes the cache buckets, so prompt_tokens
sums all three (OpenAI Responses already folds cached tokens in).
Returns ``None`` when there are no usage numbers to report.
"""
if not isinstance(last_usage, dict):
return None
completion_tokens = last_usage.get("output_tokens") or 0
if provider == "anthropic":
uncached_input = last_usage.get("input_tokens") or 0
cache_creation = last_usage.get("cache_creation_input_tokens") or 0
cache_read = last_usage.get("cache_read_input_tokens") or 0
prompt_tokens = uncached_input + cache_creation + cache_read
if not (prompt_tokens or completion_tokens):
return None
usage_block: dict[str, Any] = {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
"prompt_tokens_details": {"cached_tokens": cache_read},
"cache_creation_input_tokens": cache_creation,
"cache_read_input_tokens": cache_read,
}
# Forward 5m/1h cache-write breakdown so cost calc applies the 2x 1h
# premium instead of defaulting to 5m on chat-style.
cc_breakdown = last_usage.get("cache_creation")
if isinstance(cc_breakdown, dict) and cc_breakdown:
usage_block["cache_creation"] = cc_breakdown
# Propagate fast-mode `usage.speed` so the cost ledger applies the 6x
# multiplier without re-derivation (Anthropic falls back to "standard"
# when fast-mode is unsupported or rate-limited).
speed = last_usage.get("speed")
if speed in ("fast", "standard"):
usage_block["speed"] = speed
else:
prompt_tokens = last_usage.get("input_tokens") or 0
cached = 0
details = last_usage.get("input_tokens_details")
if isinstance(details, dict):
cached = details.get("cached_tokens") or 0
if not (prompt_tokens or completion_tokens or cached):
return None
usage_block = {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
"prompt_tokens_details": {"cached_tokens": cached},
}
# Surface OpenAI Responses / Gemini reasoning-token detail. The caller
# pre-populates last_usage["output_tokens_details"] with at least
# {"reasoning_tokens": ...}; mirror it into the OAI
# `completion_tokens_details` shape so SDKs can render the
# hidden-thoughts slice.
out_details = last_usage.get("output_tokens_details")
if isinstance(out_details, dict) and out_details:
usage_block["completion_tokens_details"] = {
"reasoning_tokens": out_details.get("reasoning_tokens") or 0,
}
usage_block["output_tokens_details"] = out_details
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"choices": [],
"usage": usage_block,
}
return f"data: {_json.dumps(chunk)}"