chore: import upstream snapshot with attribution

runtime/node/agent/providers/__init__.py

@@ -0,0 +1,8 @@
+from .base import ModelProvider, ProviderRegistry
+from .response import ModelResponse
+
+__all__ = [
+    "ModelProvider",
+    "ProviderRegistry",
+    "ModelResponse",
+]

runtime/node/agent/providers/base.py

@@ -0,0 +1,116 @@
+"""Abstract base classes for agent providers."""
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from entity.configs import AgentConfig
+from entity.messages import Message
+from schema_registry import register_model_provider_schema
+from entity.tool_spec import ToolSpec
+from runtime.node.agent.providers.response import ModelResponse
+from utils.token_tracker import TokenUsage
+from utils.registry import Registry
+
+
+class ModelProvider(ABC):
+    """Abstract base class for all agent providers."""
+    
+    def __init__(self, config: AgentConfig):
+        """
+        Initialize the agent provider with configuration.
+        
+        Args:
+            config: Agent configuration instance
+        """
+        self.config = config
+        self.base_url = config.base_url
+        self.api_key = config.api_key
+        self.model_name = config.name if isinstance(config.name, str) else str(config.name)
+        self.provider = config.provider
+        self.params = config.params or {}
+
+    @abstractmethod
+    def create_client(self):
+        """
+        Create and return the appropriate client for this provider.
+        
+        Returns:
+            Client instance for making API calls
+        """
+        pass
+
+    @abstractmethod
+    def call_model(
+        self,
+        client,
+        conversation: List[Message],
+        timeline: List[Any],
+        tool_specs: Optional[List[ToolSpec]] = None,
+        **kwargs,
+    ) -> ModelResponse:
+        """
+        Call the model with the given messages and parameters.
+        
+        Args:
+            client: Provider-specific client instance
+            conversation: List of messages in the conversation
+            tool_specs: Tool specifications available for this call
+            **kwargs: Additional parameters for the model call
+            
+        Returns:
+            ModelResponse containing content and potentially tool calls
+        """
+        pass
+
+    @abstractmethod
+    def extract_token_usage(self, response: Any) -> TokenUsage:
+        """
+        Extract token usage from the API response.
+        
+        Args:
+            response: Raw API response from the model call
+            
+        Returns:
+            TokenUsage instance with token counts
+        """
+        pass
+
+
+_provider_registry = Registry("agent_provider")
+
+
+class ProviderRegistry:
+    """Registry facade for agent providers."""
+
+    @classmethod
+    def register(
+        cls,
+        name: str,
+        provider_class: type,
+        *,
+        label: str | None = None,
+        summary: str | None = None,
+    ) -> None:
+        metadata = {
+            "label": label,
+            "summary": summary,
+        }
+        # Drop None values so schema consumers don't need to filter.
+        metadata = {key: value for key, value in metadata.items() if value is not None}
+        _provider_registry.register(name, target=provider_class, metadata=metadata)
+        register_model_provider_schema(name, label=label, summary=summary)
+
+    @classmethod
+    def get_provider(cls, name: str) -> type | None:
+        try:
+            entry = _provider_registry.get(name)
+        except Exception:
+            return None
+        return entry.load()
+
+    @classmethod
+    def list_providers(cls) -> List[str]:
+        return list(_provider_registry.names())
+
+    @classmethod
+    def iter_metadata(cls) -> Dict[str, Dict[str, Any]]:
+        return {name: dict(entry.metadata or {}) for name, entry in _provider_registry.items()}

runtime/node/agent/providers/builtin_providers.py

@@ -0,0 +1,27 @@
+"""Register built-in agent providers."""
+
+from runtime.node.agent.providers.base import ProviderRegistry
+
+from runtime.node.agent.providers.openai_provider import OpenAIProvider
+
+ProviderRegistry.register(
+    "openai",
+    OpenAIProvider,
+    label="OpenAI",
+    summary="OpenAI models via the official OpenAI SDK (responses API)",
+)
+
+try:
+    from runtime.node.agent.providers.gemini_provider import GeminiProvider
+except ImportError:
+    GeminiProvider = None
+
+if GeminiProvider is not None:
+    ProviderRegistry.register(
+        "gemini",
+        GeminiProvider,
+        label="Google Gemini",
+        summary="Google Gemini models via google-genai",
+    )
+else:
+    print("Gemini provider not registered: google-genai library not found.")

runtime/node/agent/providers/gemini_provider.py

@@ -0,0 +1,833 @@
+"""Gemini provider implementation."""
+
+import base64
+import binascii
+import json
+import os
+import uuid
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+from google import genai
+from google.genai import types as genai_types
+from google.genai.types import GenerateContentResponse
+
+from entity.messages import (
+    AttachmentRef,
+    FunctionCallOutputEvent,
+    Message,
+    MessageBlock,
+    MessageBlockType,
+    MessageRole,
+    ToolCallPayload,
+)
+from entity.tool_spec import ToolSpec
+from runtime.node.agent import ModelProvider
+from runtime.node.agent import ModelResponse
+from utils.token_tracker import TokenUsage
+
+
+class GeminiProvider(ModelProvider):
+    """Gemini provider implementation."""
+
+    CSV_INLINE_CHAR_LIMIT = 200_000
+    CSV_INLINE_SIZE_THRESHOLD_BYTES = 3 * 1024 * 1024  # 3 MB
+
+    def create_client(self):
+        """
+        Create and return the Gemini client.
+        """
+        client_kwargs: Dict[str, Any] = {}
+        if self.api_key:
+            client_kwargs["api_key"] = self.api_key
+
+        base_url = (self.base_url or "").strip()
+        http_options = self._build_http_options(base_url)
+        if http_options:
+            client_kwargs["http_options"] = http_options
+
+        return genai.Client(**client_kwargs)
+
+    def call_model(
+        self,
+        client,
+        conversation: List[Message],
+        timeline: List[Any],
+        tool_specs: Optional[List[ToolSpec]] = None,
+        **kwargs,
+    ) -> ModelResponse:
+        """
+        Call the Gemini model using the unified conversation timeline.
+        """
+        contents, system_instruction = self._build_contents(timeline)
+        config = self._build_generation_config(system_instruction, tool_specs, kwargs)
+        # print(contents)
+        # print(config)
+
+        response: GenerateContentResponse = client.models.generate_content(
+            model=self.model_name,
+            contents=contents,
+            config=config,
+        )
+
+        # print(response)
+
+        self._track_token_usage(response)
+        self._append_response_contents(timeline, response)
+        message = self._deserialize_response(response)
+        return ModelResponse(message=message, raw_response=response)
+
+    def extract_token_usage(self, response: Any) -> TokenUsage:
+        """Extract token usage from Gemini usage metadata."""
+        usage_metadata = getattr(response, "usage_metadata", None)
+        if not usage_metadata:
+            return TokenUsage()
+
+        prompt_tokens = getattr(usage_metadata, "prompt_token_count", None) or 0
+        candidate_tokens = getattr(usage_metadata, "candidates_token_count", None) or 0
+        total_tokens = getattr(usage_metadata, "total_token_count", None)
+        cached_tokens = getattr(usage_metadata, "cached_content_token_count", None)
+
+        metadata = {
+            "prompt_token_count": prompt_tokens,
+            "candidates_token_count": candidate_tokens,
+        }
+        if total_tokens is not None:
+            metadata["total_token_count"] = total_tokens
+        if cached_tokens is not None:
+            metadata["cached_content_token_count"] = cached_tokens
+
+        return TokenUsage(
+            input_tokens=prompt_tokens,
+            output_tokens=candidate_tokens,
+            total_tokens=total_tokens or (prompt_tokens + candidate_tokens),
+            metadata=metadata,
+        )
+
+    # ---------------------------------------------------------------------
+    # Serialization helpers
+    # ---------------------------------------------------------------------
+
+    def _build_contents(
+        self,
+        timeline: List[Any],
+    ) -> Tuple[List[genai_types.Content], Optional[str]]:
+        contents: List[genai_types.Content] = []
+        system_prompts: List[str] = []
+
+        for item in timeline:
+            if isinstance(item, Message):
+                if item.role is MessageRole.SYSTEM:
+                    text = item.text_content().strip()
+                    if text:
+                        system_prompts.append(text)
+                    continue
+                contents.append(self._message_to_content(item))
+                continue
+
+            if isinstance(item, FunctionCallOutputEvent):
+                contents.append(self._function_output_event_to_content(item))
+                continue
+
+            if isinstance(item, genai_types.Content):
+                contents.append(item)
+
+        if not contents:
+            contents.append(
+                genai_types.Content(
+                    role="user",
+                    parts=[genai_types.Part(text="")],
+                )
+            )
+
+        system_instruction = "\n\n".join(system_prompts) if system_prompts else None
+        return contents, system_instruction
+
+    def _append_response_contents(self, timeline: List[Any], response: Any) -> None:
+        candidates = getattr(response, "candidates", None)
+        if not candidates:
+            return
+        for candidate in candidates:
+            content = getattr(candidate, "content", None)
+            if content:
+                timeline.append(content)
+
+    def _message_to_content(self, message: Message) -> genai_types.Content:
+        role = self._map_role(message.role)
+        if message.role is MessageRole.TOOL:
+            part = self._build_tool_response_part(message)
+            return genai_types.Content(role="user", parts=[part])
+
+        parts: List[genai_types.Part] = []
+        for block in message.blocks():
+            parts.extend(self._block_to_parts(block))
+        if not parts:
+            text = message.text_content()
+            parts.append(genai_types.Part(text=text))
+        return genai_types.Content(role=role, parts=parts)
+
+    def _function_output_event_to_content(
+        self,
+        event: FunctionCallOutputEvent,
+    ) -> genai_types.Content:
+        function_name = event.function_name or event.call_id or "tool"
+        payload: Dict[str, Any] = {}
+        function_result_parts: List[genai_types.FunctionResponsePart] = []
+        result_texts: List[str] = []
+
+        if event.output_blocks:
+            for block in event.output_blocks:
+                # Describe the block for the text result
+                desc = self._describe_block(block)
+                if desc:
+                    result_texts.append(desc)
+
+                if self._block_has_attachment(block):
+                    # Check if we should inline this attachment as text
+                    if self._should_inline_attachment_as_text(block):
+                        text_content = self._read_attachment_text(block.attachment)
+                        if text_content:
+                            result_texts.append(f"\n[Attachment Content: {block.attachment.name}]\n{text_content}")
+                        continue
+
+                    # Otherwise treat as binary part
+                    general_parts = self._block_to_parts(block)
+                    function_result_parts.extend(self._general_parts_to_function_response_parts(general_parts))
+        else:
+            if event.output_text:
+                result_texts.append(event.output_text)
+
+        payload["result"] = "\n".join(result_texts)
+
+        function_part = genai_types.Part.from_function_response(
+            name=function_name,
+            response=payload or {"result": ""},
+            parts=function_result_parts or None
+        )
+
+        parts: List[genai_types.Part] = [function_part]
+        return genai_types.Content(role="user", parts=parts)
+
+    def _should_inline_attachment_as_text(self, block: MessageBlock) -> bool:
+        if not block.attachment:
+            return False
+        mime = (block.attachment.mime_type or "").lower()
+        return (
+            mime.startswith("text/") or
+            mime == "application/json" or
+            mime.endswith("+json") or
+            mime.endswith("+xml")
+        )
+
+    def _read_attachment_text(self, attachment: AttachmentRef) -> Optional[str]:
+        data_bytes = self._read_attachment_bytes(attachment)
+        return self._bytes_to_text(data_bytes)
+
+    def _general_parts_to_function_response_parts(self, parts: List[genai_types.Part]) -> List[genai_types.FunctionResponsePart]:
+        function_response_parts: List[genai_types.FunctionResponsePart] = []
+        for part in parts:
+            if part.inline_data:
+                # Convert inline_data (bytes) to base64 data URI and use from_uri
+                function_response_parts.append(
+                    genai_types.FunctionResponsePart.from_bytes(data=part.inline_data.data, mime_type=part.inline_data.mime_type or "application/octet-stream")
+                )
+            if part.file_data:
+                function_response_parts.append(
+                    genai_types.FunctionResponsePart.from_uri(file_uri=part.file_data.file_uri, mime_type=part.file_data.mime_type or "application/octet-stream")
+                )
+        return function_response_parts
+
+    def _build_tool_response_part(self, message: Message) -> genai_types.Part:
+        tool_name = message.metadata.get("tool_name") if isinstance(message.metadata, dict) else None
+        tool_name = tool_name or message.tool_call_id or "tool"
+        payload, block_parts = self._serialize_tool_message_payload(message)
+        return genai_types.Part(
+            function_response=genai_types.FunctionResponse(
+                name=tool_name,
+                response=payload,
+                parts=block_parts or None,
+            )
+        )
+
+    def _block_has_attachment(self, block: Any) -> bool:
+        return isinstance(block, MessageBlock) and block.attachment is not None
+
+    def _serialize_tool_message_payload(self, message: Message) -> Tuple[Dict[str, Any], List[genai_types.FunctionResponsePart]]:
+        content = message.content
+        blocks: List[MessageBlock] = []
+        if isinstance(content, str):
+            stripped = content.strip()
+            if stripped:
+                try:
+                    payload = json.loads(stripped)
+                except json.JSONDecodeError:
+                    payload = {"result": stripped}
+            else:
+                payload = {"result": ""}
+            return payload, []
+
+        if isinstance(content, list):
+            blocks_payload = []
+            for block in content:
+                if isinstance(block, MessageBlock):
+                    blocks_payload.append(block.to_dict())
+                    blocks.append(block)
+                elif isinstance(block, dict):
+                    blocks_payload.append(block)
+                    try:
+                        blocks.append(MessageBlock.from_dict(block))
+                    except Exception:
+                        continue
+            parts = self._blocks_to_function_parts(blocks)
+            return {"blocks": blocks_payload, "result": message.text_content()}, parts
+
+        parts = self._blocks_to_function_parts(blocks)
+        return {"result": message.text_content()}, parts
+
+    def _describe_block(self, block: Any) -> str:
+        if isinstance(block, MessageBlock):
+            return block.describe()
+        if isinstance(block, dict):
+            text = block.get("text")
+            if text:
+                return str(text)
+        return str(block)
+
+    def _block_to_parts(self, block: MessageBlock) -> List[genai_types.Part]:
+        if block.type is MessageBlockType.TEXT:
+            return [genai_types.Part(text=block.text or "")]
+
+        if block.type is MessageBlockType.FILE:
+            csv_text = self._maybe_inline_large_csv(block)
+            if csv_text is not None:
+                return [genai_types.Part(text=csv_text)]
+
+        if block.type in (
+            MessageBlockType.IMAGE,
+            MessageBlockType.AUDIO,
+            MessageBlockType.VIDEO,
+            MessageBlockType.FILE,
+        ):
+            media_part = self._attachment_block_to_part(block)
+            return [media_part] if media_part else []
+
+        if block.type is MessageBlockType.DATA:
+            data_payload = block.data or {}
+            text = block.text or json.dumps(data_payload, ensure_ascii=False)
+            return [genai_types.Part(text=text)]
+
+        return []
+
+    def _maybe_inline_large_csv(self, block: MessageBlock) -> Optional[str]:
+        """Convert large CSV attachments to inline text to avoid Gemini upload size limits."""
+
+        attachment = block.attachment
+        if not attachment:
+            return None
+
+        mime = (attachment.mime_type or "").lower()
+        name = (attachment.name or "").lower()
+        if "text/csv" not in mime and not name.endswith(".csv"):
+            return None
+        if attachment.remote_file_id:
+            return None
+
+        threshold = getattr(
+            self,
+            "csv_inline_size_threshold_bytes",
+            self.CSV_INLINE_SIZE_THRESHOLD_BYTES,
+        )
+
+        size_bytes = attachment.size
+        data_bytes: Optional[bytes] = None
+        if size_bytes is None:
+            data_bytes = self._read_attachment_bytes(attachment)
+            if data_bytes is None:
+                return None
+            size_bytes = len(data_bytes)
+
+        if size_bytes is None or size_bytes <= threshold:
+            return None
+
+        if data_bytes is None:
+            data_bytes = self._read_attachment_bytes(attachment)
+            if data_bytes is None:
+                return None
+
+        text = self._bytes_to_text(data_bytes)
+        if text is None:
+            return None
+
+        char_limit = getattr(self, "csv_inline_char_limit", self.CSV_INLINE_CHAR_LIMIT)
+        truncated = False
+        if len(text) > char_limit:
+            text = text[:char_limit]
+            truncated = True
+
+        display_name = attachment.name or attachment.attachment_id or "attachment.csv"
+        suffix = f"\n\n[truncated after {char_limit} characters]" if truncated else ""
+        return f"CSV file '{display_name}' (converted from >3MB upload):\n{text}{suffix}"
+
+    def _bytes_to_text(self, data_bytes: Optional[bytes]) -> Optional[str]:
+        if data_bytes is None:
+            return None
+        try:
+            return data_bytes.decode("utf-8")
+        except UnicodeDecodeError:
+            return data_bytes.decode("utf-8", errors="replace")
+
+    def _attachment_block_to_part(self, block: MessageBlock) -> Optional[genai_types.Part]:
+        attachment = block.attachment
+        if not attachment:
+            return None
+
+        metadata = attachment.metadata or {}
+        gemini_file_uri = metadata.get("gemini_file_uri") or attachment.remote_file_id
+        mime_type = attachment.mime_type or self._guess_mime_from_block(block)
+
+        if gemini_file_uri:
+            return genai_types.Part(
+                file_data=genai_types.FileData(
+                    file_uri=gemini_file_uri,
+                    mime_type=mime_type,
+                    # display_name=attachment.name
+                )
+            )
+
+        blob_data = self._read_attachment_bytes(attachment)
+        if blob_data is None:
+            return None
+
+        return genai_types.Part(
+            inline_data=genai_types.Blob(
+                mime_type=mime_type or "application/octet-stream",
+                data=blob_data,
+                # display_name=attachment.name,
+            )
+        )
+
+    def _blocks_to_function_parts(
+        self,
+        blocks: Optional[Sequence[Any]],
+    ) -> List[genai_types.FunctionResponsePart]:
+        if not blocks:
+            return []
+        parts: List[genai_types.FunctionResponsePart] = []
+        for block in blocks:
+            if not isinstance(block, MessageBlock):
+                if isinstance(block, dict):
+                    try:
+                        block = MessageBlock.from_dict(block)
+                    except Exception:
+                        continue
+                else:
+                    continue
+            attachment = block.attachment
+            if not attachment:
+                continue
+            mime_type = attachment.mime_type or self._guess_mime_from_block(block)
+            file_uri = (attachment.metadata or {}).get("gemini_file_uri") or attachment.remote_file_id
+            if file_uri:
+                parts.append(
+                    genai_types.FunctionResponsePart(
+                        file_data=genai_types.FunctionResponseFileData(
+                            file_uri=file_uri,
+                            mime_type=mime_type,
+                            display_name=attachment.name,
+                        )
+                    )
+                )
+                continue
+            data_bytes = self._read_attachment_bytes(attachment)
+            if not data_bytes:
+                continue
+            parts.append(
+                genai_types.FunctionResponsePart(
+                    inline_data=genai_types.FunctionResponseBlob(
+                        mime_type=mime_type or "application/octet-stream",
+                        data=data_bytes,
+                        display_name=attachment.name,
+                    )
+                )
+            )
+        return parts
+
+    def _coerce_message_blocks(self, payload: Any) -> List[MessageBlock]:
+        if not isinstance(payload, Sequence) or isinstance(payload, (str, bytes, bytearray)):
+            return []
+        blocks: List[MessageBlock] = []
+        for item in payload:
+            if isinstance(item, MessageBlock):
+                blocks.append(item)
+            elif isinstance(item, dict):
+                try:
+                    blocks.append(MessageBlock.from_dict(item))
+                except Exception:
+                    continue
+        return blocks
+
+    def _encode_thought_signature(self, value: Any) -> Optional[str]:
+        if value is None:
+            return None
+        if isinstance(value, bytes):
+            return base64.b64encode(value).decode("ascii")
+        try:
+            return str(value)
+        except Exception:
+            return None
+
+    def _read_attachment_bytes(self, attachment: AttachmentRef) -> Optional[bytes]:
+        if attachment.data_uri:
+            decoded = self._decode_data_uri(attachment.data_uri)
+            if decoded is not None:
+                return decoded
+        if attachment.local_path and os.path.exists(attachment.local_path):
+            try:
+                with open(attachment.local_path, "rb") as handle:
+                    return handle.read()
+            except OSError:
+                return None
+        return None
+
+    def _decode_data_uri(self, data_uri: str) -> Optional[bytes]:
+        if not data_uri.startswith("data:"):
+            return None
+        header, _, data = data_uri.partition(",")
+        if not _:
+            return None
+        if ";base64" in header:
+            try:
+                return base64.b64decode(data)
+            except (ValueError, binascii.Error):
+                return None
+        return data.encode("utf-8")
+
+    def _guess_mime_from_block(self, block: MessageBlock) -> str:
+        if block.attachment and block.attachment.mime_type:
+            return block.attachment.mime_type
+        if block.type is MessageBlockType.IMAGE:
+            return "image/png"
+        if block.type is MessageBlockType.AUDIO:
+            return "audio/mpeg"
+        if block.type is MessageBlockType.VIDEO:
+            return "video/mp4"
+        return "application/octet-stream"
+
+    def _map_role(self, role: MessageRole) -> str:
+        if role is MessageRole.USER:
+            return "user"
+        if role is MessageRole.ASSISTANT:
+            return "model"
+        if role is MessageRole.TOOL:
+            return "tool"
+        return "user"
+
+    # ---------------------------------------------------------------------
+    # Config builders
+    # ---------------------------------------------------------------------
+
+    def _build_generation_config(
+        self,
+        system_instruction: Optional[str],
+        tool_specs: Optional[List[ToolSpec]],
+        call_params: Dict[str, Any],
+    ) -> genai_types.GenerateContentConfig:
+        params = dict(self.params or {})
+        params.update(call_params)
+
+        config_kwargs: Dict[str, Any] = {}
+        if system_instruction:
+            config_kwargs["system_instruction"] = system_instruction
+
+        for key in (
+            "temperature",
+            "top_p",
+            "top_k",
+            "candidate_count",
+            "max_output_tokens",
+            "response_modalities",
+            "stop_sequences",
+            "seed",
+            "presence_penalty",
+            "frequency_penalty",
+        ):
+            if key in params:
+                config_kwargs[key] = params.pop(key)
+
+        safety_settings = params.pop("safety_settings", None)
+        if safety_settings:
+            config_kwargs["safety_settings"] = safety_settings
+
+        image_config = params.pop("image_config", None)
+        aspect_ratio = params.pop("aspect_ratio", None)
+        if aspect_ratio:
+            if image_config is None:
+                image_config = {"aspect_ratio": aspect_ratio}
+            elif isinstance(image_config, dict):
+                image_config = dict(image_config)
+                image_config.setdefault("aspect_ratio", aspect_ratio)
+            elif isinstance(image_config, genai_types.ImageConfig):
+                try:
+                    image_config.aspect_ratio = aspect_ratio
+                except Exception:
+                    image_config = {"aspect_ratio": aspect_ratio}
+            else:
+                image_config = {"aspect_ratio": aspect_ratio}
+        if image_config:
+            config_kwargs["image_config"] = self._coerce_image_config(image_config)
+
+        audio_config = params.pop("audio_config", None)
+        if audio_config:
+            config_kwargs["audio_config"] = audio_config
+
+        video_config = params.pop("video_config", None)
+        if video_config:
+            config_kwargs["video_config"] = video_config
+
+        tools = self._build_tools(tool_specs or [])
+        if tools:
+            config_kwargs["tools"] = tools
+
+        tool_config_payload = params.pop("tool_config", None)
+        function_calling_payload = params.pop("function_calling_config", None)
+        if function_calling_payload:
+            tool_config_payload = tool_config_payload or {}
+            tool_config_payload["function_calling_config"] = function_calling_payload
+
+        if tool_config_payload:
+            config_kwargs["tool_config"] = self._coerce_tool_config(tool_config_payload)
+
+        automatic_fn_calling = params.pop("automatic_function_calling", None)
+        if automatic_fn_calling:
+            config_kwargs["automatic_function_calling"] = self._coerce_automatic_function_calling(
+                automatic_fn_calling
+            )
+
+        return genai_types.GenerateContentConfig(**config_kwargs)
+
+    def _build_http_options(self, base_url: str) -> Optional[genai_types.HttpOptions]:
+        if not base_url:
+            return None
+        try:
+            return genai_types.HttpOptions(base_url=base_url, timeout=4 * 60 * 1000)  # 4 min
+        except Exception:
+            return None
+
+    def _coerce_image_config(self, image_config: Any) -> Any:
+        if isinstance(image_config, genai_types.ImageConfig):
+            return image_config
+        if isinstance(image_config, dict):
+            try:
+                return genai_types.ImageConfig(**image_config)
+            except Exception:
+                return image_config
+        return image_config
+
+    def _build_tools(self, tool_specs: List[ToolSpec]) -> List[genai_types.Tool]:
+        if not tool_specs:
+            return []
+
+        declarations = []
+        for spec in tool_specs:
+            fn_payload = spec.to_gemini_function()
+            parameters = fn_payload.get("parameters") or {"type": "object", "properties": {}}
+            if 'title' in parameters:
+                parameters.pop('title')
+            # Replace 'title' with 'description' in properties
+            for prop_name, prop_value in parameters.get('properties', {}).items():
+                if isinstance(prop_value, dict) and 'title' in prop_value:
+                    prop_value['description'] = prop_value.pop('title')
+            declarations.append(
+                genai_types.FunctionDeclaration(
+                    name=fn_payload.get("name", ""),
+                    description=fn_payload.get("description") or "",
+                    parameters=parameters,
+                )
+            )
+        return [genai_types.Tool(function_declarations=declarations)]
+
+    def _coerce_tool_config(self, payload: Any) -> genai_types.ToolConfig:
+        if isinstance(payload, genai_types.ToolConfig):
+            return payload
+        kwargs: Dict[str, Any] = {}
+        if isinstance(payload, dict):
+            fn_payload = payload.get("function_calling_config")
+            if fn_payload:
+                kwargs["function_calling_config"] = self._coerce_function_calling_config(fn_payload)
+        return genai_types.ToolConfig(**kwargs)
+
+    def _coerce_function_calling_config(self, payload: Any) -> genai_types.FunctionCallingConfig:
+        if isinstance(payload, genai_types.FunctionCallingConfig):
+            return payload
+        if isinstance(payload, str):
+            return genai_types.FunctionCallingConfig(mode=payload)
+        if isinstance(payload, dict):
+            return genai_types.FunctionCallingConfig(**payload)
+        raise ValueError("Invalid function calling configuration payload")
+
+    def _coerce_automatic_function_calling(self, payload: Any) -> Any:
+        config_cls = getattr(genai_types, "AutomaticFunctionCallingConfig", None)
+        if config_cls is None:
+            raise ValueError("Automatic function calling config not supported in current SDK version")
+        if isinstance(payload, config_cls):
+            return payload
+        if isinstance(payload, dict):
+            return config_cls(**payload)
+        raise ValueError("Invalid automatic function calling config payload")
+
+    # ---------------------------------------------------------------------
+    # Response parsing
+    # ---------------------------------------------------------------------
+
+    def _deserialize_response(self, response: Any) -> Message:
+        candidate = self._select_primary_candidate(response)
+        if not candidate:
+            return Message(role=MessageRole.ASSISTANT, content="")
+
+        content = getattr(candidate, "content", None)
+        if not content:
+            return Message(role=MessageRole.ASSISTANT, content=response.text if hasattr(response, "text") else "")
+
+        blocks, tool_calls = self._parse_candidate_parts(getattr(content, "parts", []) or [])
+        if not blocks:
+            fallback = getattr(response, "text", None) or ""
+            blocks = [MessageBlock(MessageBlockType.TEXT, text=fallback)] if fallback else []
+
+        return Message(
+            role=MessageRole.ASSISTANT,
+            content=blocks or "",
+            tool_calls=tool_calls,
+        )
+
+    def _select_primary_candidate(self, response: Any) -> Any:
+        candidates = getattr(response, "candidates", None) or []
+        if not candidates:
+            return None
+        return candidates[0]
+
+    def _parse_candidate_parts(
+        self,
+        parts: Sequence[Any],
+    ) -> Tuple[List[MessageBlock], List[ToolCallPayload]]:
+        blocks: List[MessageBlock] = []
+        tool_calls: List[ToolCallPayload] = []
+
+        for part in parts:
+            if hasattr(part, "text") and part.text is not None:
+                blocks.append(MessageBlock(MessageBlockType.TEXT, text=part.text))
+                continue
+
+            function_call = getattr(part, "function_call", None)
+            if function_call:
+                thought_signature = getattr(part, "thought_signature", None)
+                tool_calls.append(
+                    self._build_tool_call_payload(function_call, thought_signature=thought_signature)
+                )
+                continue
+
+            inline_data = getattr(part, "inline_data", None)
+            if inline_data:
+                blocks.append(self._build_inline_block(inline_data))
+                continue
+
+            file_data = getattr(part, "file_data", None)
+            if file_data:
+                blocks.append(self._build_file_block(file_data))
+                continue
+
+            function_response = getattr(part, "function_response", None)
+            if function_response:
+                blocks.append(
+                    MessageBlock(
+                        type=MessageBlockType.DATA,
+                        text=json.dumps(function_response.response or {}, ensure_ascii=False),
+                        data={
+                            "function_name": getattr(function_response, "name", ""),
+                            "response": function_response.response or {},
+                        },
+                    )
+                )
+                continue
+
+        return blocks, tool_calls
+
+    def _build_tool_call_payload(self, fn_call: Any, *, thought_signature: Any = None) -> ToolCallPayload:
+        call_id = getattr(fn_call, "name", "") or uuid.uuid4().hex
+        arguments = getattr(fn_call, "args", {}) or {}
+        try:
+            arg_str = json.dumps(arguments, ensure_ascii=False)
+        except (TypeError, ValueError):
+            arg_str = str(arguments)
+        metadata: Dict[str, Any] = {}
+        encoded_signature = self._encode_thought_signature(thought_signature)
+        if encoded_signature:
+            metadata["gemini_thought_signature_b64"] = encoded_signature
+        return ToolCallPayload(
+            id=call_id,
+            function_name=getattr(fn_call, "name", "") or call_id,
+            arguments=arg_str,
+            type="function",
+            metadata=metadata,
+        )
+
+    def _build_inline_block(self, blob: Any) -> MessageBlock:
+        mime_type = getattr(blob, "mime_type", "") or "application/octet-stream"
+        data_bytes = getattr(blob, "data", None) or b""
+        data_uri = self._encode_data_uri(mime_type, data_bytes)
+        block_type = self._block_type_from_mime(mime_type)
+        return MessageBlock(
+            type=block_type,
+            attachment=AttachmentRef(
+                attachment_id=uuid.uuid4().hex,
+                mime_type=mime_type,
+                data_uri=data_uri,
+                metadata={"source": "gemini_inline"},
+            ),
+        )
+
+    def _build_file_block(self, file_data: Any) -> MessageBlock:
+        mime_type = getattr(file_data, "mime_type", None)
+        file_uri = getattr(file_data, "file_uri", None) or getattr(file_data, "file", None)
+        block_type = self._block_type_from_mime(mime_type or "")
+        return MessageBlock(
+            type=block_type,
+            attachment=AttachmentRef(
+                attachment_id=uuid.uuid4().hex,
+                mime_type=mime_type,
+                remote_file_id=file_uri,
+                metadata={"gemini_file_uri": file_uri, "source": "gemini_file"},
+            ),
+        )
+
+    def _block_type_from_mime(self, mime_type: str) -> MessageBlockType:
+        if mime_type.startswith("image/"):
+            return MessageBlockType.IMAGE
+        if mime_type.startswith("audio/"):
+            return MessageBlockType.AUDIO
+        if mime_type.startswith("video/"):
+            return MessageBlockType.VIDEO
+        return MessageBlockType.FILE
+
+    def _encode_data_uri(self, mime_type: str, data: bytes) -> str:
+        encoded = base64.b64encode(data).decode("utf-8")
+        return f"data:{mime_type};base64,{encoded}"
+
+    # ---------------------------------------------------------------------
+    # Token tracking
+    # ---------------------------------------------------------------------
+
+    def _track_token_usage(self, response: Any) -> None:
+        token_tracker = getattr(self.config, "token_tracker", None)
+        if not token_tracker:
+            return
+
+        usage = self.extract_token_usage(response)
+        if usage.input_tokens == 0 and usage.output_tokens == 0 and not usage.metadata:
+            return
+
+        node_id = getattr(self.config, "node_id", "ALL")
+        usage.node_id = node_id
+        usage.model_name = self.model_name
+        usage.workflow_id = token_tracker.workflow_id
+        usage.provider = "gemini"
+
+        token_tracker.record_usage(node_id, self.model_name, usage, provider="gemini")

runtime/node/agent/providers/openai_provider.py

@@ -0,0 +1,809 @@
+"""OpenAI provider implementation."""
+
+import base64
+import hashlib
+import re
+
+import binascii
+import os
+from typing import Any, Dict, List, Optional, Union
+from urllib.parse import unquote_to_bytes
+
+import openai
+from openai import OpenAI
+
+from entity.messages import (
+    AttachmentRef,
+    FunctionCallOutputEvent,
+    Message,
+    MessageBlock,
+    MessageBlockType,
+    MessageRole,
+    ToolCallPayload,
+)
+from entity.tool_spec import ToolSpec
+from runtime.node.agent import ModelProvider
+from runtime.node.agent import ModelResponse
+from utils.token_tracker import TokenUsage
+
+
+class OpenAIProvider(ModelProvider):
+    """OpenAI provider implementation."""
+
+    CSV_INLINE_CHAR_LIMIT = 200_000  # safeguard large attachments
+    TEXT_INLINE_CHAR_LIMIT = 200_000  # safeguard large text/* attachments
+    MAX_INLINE_FILE_BYTES = 50 * 1024 * 1024  # OpenAI function output limit (~50 MB)
+
+    def create_client(self):
+        """
+        Create and return the OpenAI client.
+        
+        Returns:
+            OpenAI client instance with token tracking if available
+        """
+        if self.base_url:
+            return OpenAI(
+                api_key=self.api_key,
+                base_url=self.base_url,
+            )
+        else:
+            return OpenAI(
+                api_key=self.api_key,
+            )
+
+    def call_model(
+        self,
+        client: openai.Client,
+        conversation: List[Message],
+        timeline: List[Any],
+        tool_specs: Optional[List[ToolSpec]] = None,
+        **kwargs,
+    ) -> ModelResponse:
+        """
+        Call the OpenAI model with the given messages and parameters.
+        """
+        # 1. Determine if we should use Chat Completions directly
+        is_chat = self._is_chat_completions_mode(client)
+        
+        if is_chat:
+            request_payload = self._build_chat_payload(conversation, tool_specs, kwargs)
+            response = client.chat.completions.create(**request_payload)
+            self._track_token_usage(response)
+            self._append_chat_response_output(timeline, response)
+            message = self._deserialize_chat_response(response)
+            return ModelResponse(message=message, raw_response=response)
+
+        # 2. Try Responses API with fallback
+        request_payload = self._build_request_payload(timeline, tool_specs, kwargs)
+        try:
+            response = client.responses.create(**request_payload)
+            self._track_token_usage(response)
+            self._append_response_output(timeline, response)
+            message = self._deserialize_response(response)
+            return ModelResponse(message=message, raw_response=response)
+        except Exception as e:
+            new_request_payload = self._build_chat_payload(conversation, tool_specs, kwargs)
+            response = client.chat.completions.create(**new_request_payload)
+            self._track_token_usage(response)
+            self._append_chat_response_output(timeline, response)
+            message = self._deserialize_chat_response(response)
+            return ModelResponse(message=message, raw_response=response)
+
+    def _is_chat_completions_mode(self, client: Any) -> bool:
+        """Determine if we should use standard chat completions instead of responses API."""
+        protocol = self.params.get("protocol")
+        if protocol == "chat":
+            return True
+        if protocol == "responses":
+            return False
+        # Default to Responses API only if it exists on the client
+        return not hasattr(client, "responses")
+
+    def extract_token_usage(self, response: Any) -> TokenUsage:
+        """
+        Extract token usage from the OpenAI API response.
+        
+        Args:
+            response: OpenAI API response from the model call
+            
+        Returns:
+            TokenUsage instance with token counts
+        """
+        usage = getattr(response, "usage", None)
+        if not usage:
+            return TokenUsage()
+
+        def _get(name: str) -> Any:
+            if hasattr(usage, name):
+                return getattr(usage, name)
+            if isinstance(usage, dict):
+                return usage.get(name)
+            return None
+
+        prompt_tokens = _get("prompt_tokens")
+        completion_tokens = _get("completion_tokens")
+        input_tokens = _get("input_tokens")
+        output_tokens = _get("output_tokens")
+
+        resolved_input = input_tokens if input_tokens is not None else prompt_tokens or 0
+        resolved_output = output_tokens if output_tokens is not None else completion_tokens or 0
+
+        total_tokens = _get("total_tokens")
+        if total_tokens is None:
+            total_tokens = (resolved_input or 0) + (resolved_output or 0)
+
+        metadata = {
+            "prompt_tokens": prompt_tokens or 0,
+            "completion_tokens": completion_tokens or 0,
+            "input_tokens": resolved_input or 0,
+            "output_tokens": resolved_output or 0,
+            "total_tokens": total_tokens or 0,
+        }
+
+        return TokenUsage(
+            input_tokens=resolved_input or 0,
+            output_tokens=resolved_output or 0,
+            total_tokens=total_tokens or 0,
+            metadata=metadata,
+        )
+
+    def _track_token_usage(self, response: Any) -> None:
+        """Record token usage if a tracker is attached to the config."""
+        token_tracker = getattr(self.config, "token_tracker", None)
+        if not token_tracker:
+            return
+
+        usage = self.extract_token_usage(response)
+        if usage.input_tokens == 0 and usage.output_tokens == 0 and not usage.metadata:
+            return
+
+        node_id = getattr(self.config, "node_id", "ALL")
+        usage.node_id = node_id
+        usage.model_name = self.model_name
+        usage.workflow_id = token_tracker.workflow_id
+        usage.provider = "openai"
+
+        token_tracker.record_usage(node_id, self.model_name, usage, provider="openai")
+
+    def _build_request_payload(
+        self,
+        timeline: List[Any],
+        tool_specs: Optional[List[ToolSpec]],
+        raw_params: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """Construct the Responses API payload from event timeline."""
+        params = dict(raw_params)
+        max_tokens = params.pop("max_tokens", None)
+        max_output_tokens = params.pop("max_output_tokens", None)
+        if max_output_tokens is None and max_tokens is not None:
+            max_output_tokens = max_tokens
+
+        input_messages: List[Any] = []
+        for item in timeline:
+            serialized = self._serialize_timeline_item(item)
+            if serialized is not None:
+                input_messages.append(serialized)
+
+        if not input_messages:
+            input_messages = [
+                {
+                    "role": "user",
+                    "content": [{"type": "input_text", "text": ""}],
+                }
+            ]
+
+        payload: Dict[str, Any] = {
+            "model": self.model_name,
+            "input": input_messages,
+            "temperature": params.pop("temperature", 0.7),
+            "timeout": params.pop("timeout", 300),  # 5 min
+        }
+        if max_output_tokens is not None:
+            payload["max_output_tokens"] = max_output_tokens
+        elif self.params.get("max_output_tokens"):
+            payload["max_output_tokens"] = self.params["max_output_tokens"]
+
+        user_tools = params.pop("tools", None)
+        merged_tools: List[Any] = []
+        if isinstance(user_tools, list):
+            merged_tools.extend(user_tools)
+        elif user_tools is not None:
+            raise ValueError("params.tools must be a list when provided")
+
+        if tool_specs:
+            merged_tools.extend(spec.to_openai_dict() for spec in tool_specs)
+
+        if merged_tools:
+            payload["tools"] = merged_tools
+
+        tool_choice = params.pop("tool_choice", None)
+        if tool_choice is not None:
+            payload["tool_choice"] = tool_choice
+        elif tool_specs:
+            payload.setdefault("tool_choice", "auto")
+
+        # Pass any remaining kwargs directly
+        payload.update(params)
+        return payload
+
+    def _build_chat_payload(
+        self,
+        conversation: List[Message],
+        tool_specs: Optional[List[ToolSpec]],
+        raw_params: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """Construct standard Chat Completions API payload."""
+        params = dict(raw_params)
+        max_output_tokens = params.pop("max_output_tokens", None)
+        max_tokens = params.pop("max_tokens", None)
+        if max_tokens is None and max_output_tokens is not None:
+            max_tokens = max_output_tokens
+
+        messages: List[Any] = []
+        for item in conversation:
+            serialized = self._serialize_message_for_chat(item)
+            if serialized is not None:
+                messages.append(serialized)
+
+        if not messages:
+            messages = [{"role": "user", "content": ""}]
+
+        payload: Dict[str, Any] = {
+            "model": self.model_name,
+            "messages": messages,
+            "temperature": params.pop("temperature", 0.7),
+        }
+        if max_tokens is not None:
+            payload["max_tokens"] = max_tokens
+        elif self.params.get("max_tokens"):
+             payload["max_tokens"] = self.params["max_tokens"]
+
+        user_tools = params.pop("tools", None)
+        merged_tools: List[Any] = []
+        if isinstance(user_tools, list):
+            merged_tools.extend(user_tools)
+
+        if tool_specs:
+            for spec in tool_specs:
+                merged_tools.append({
+                    "type": "function",
+                    "function": {
+                        "name": spec.name,
+                        "description": spec.description,
+                        "parameters": spec.parameters or {"type": "object", "properties": {}},
+                    }
+                })
+
+        if merged_tools:
+            payload["tools"] = merged_tools
+
+        tool_choice = params.pop("tool_choice", None)
+        if tool_choice is not None:
+            payload["tool_choice"] = tool_choice
+        elif tool_specs:
+            payload.setdefault("tool_choice", "auto")
+
+        payload.update(params)
+        return payload
+
+    def _serialize_timeline_item_for_chat(self, item: Any) -> Optional[Any]:
+        if isinstance(item, Message):
+            return self._serialize_message_for_chat(item)
+        if isinstance(item, FunctionCallOutputEvent):
+            return self._serialize_function_call_output_event_for_chat(item)
+        if isinstance(item, dict):
+             # basic conversion if it looks like a Responses output
+             role = item.get("role")
+             content = item.get("content")
+             tool_calls = item.get("tool_calls")
+             if role and (content or tool_calls):
+                  return {
+                       "role": role,
+                       "content": self._transform_blocks_for_chat(content) if isinstance(content, list) else content,
+                       "tool_calls": tool_calls
+                  }
+        return None
+
+    def _serialize_message_for_chat(self, message: Message) -> Dict[str, Any]:
+        """Convert internal Message to standard Chat Completions schema."""
+        role_value = message.role.value
+        blocks = message.blocks()
+        if not blocks or message.role == MessageRole.TOOL:
+            content = message.text_content()
+        else:
+            content = self._transform_blocks_for_chat(self._serialize_blocks(blocks, message.role))
+
+        payload: Dict[str, Any] = {
+            "role": role_value,
+            "content": content,
+        }
+        if message.name:
+            payload["name"] = message.name
+        if message.tool_call_id:
+            payload["tool_call_id"] = message.tool_call_id
+        if message.tool_calls:
+            payload["tool_calls"] = [tc.to_openai_dict() for tc in message.tool_calls]
+        return payload
+
+    def _serialize_function_call_output_event_for_chat(self, event: FunctionCallOutputEvent) -> Dict[str, Any]:
+        """Convert tool result to standard Chat Completions schema."""
+        text = event.output_text or ""
+        if event.output_blocks:
+             # simple concatenation for tool output in chat mode
+             text = "\n".join(b.describe() for b in event.output_blocks)
+        
+        return {
+            "role": "tool",
+            "tool_call_id": event.call_id or "tool_call",
+            "content": text,
+        }
+
+    def _transform_blocks_for_chat(self, blocks: List[Dict[str, Any]]) -> Union[str, List[Dict[str, Any]]]:
+        """Convert Responses block types to Chat block types (e.g., input_text -> text)."""
+        transformed: List[Dict[str, Any]] = []
+        for block in blocks:
+            b_type = block.get("type", "")
+            if b_type in ("input_text", "output_text"):
+                transformed.append({"type": "text", "text": block.get("text", "")})
+            elif b_type in ("input_image", "output_image"):
+                transformed.append({"type": "image_url", "image_url": {"url": block.get("image_url", "")}})
+            else:
+                # Keep as is or drop if complex
+                transformed.append(block)
+        
+        # If only one text block, return as string for better compatibility
+        if len(transformed) == 1 and transformed[0]["type"] == "text":
+            return transformed[0]["text"]
+        return transformed
+
+    def _deserialize_chat_response(self, response: Any) -> Message:
+        """Convert Chat Completions output to internal Message."""
+        choices = self._get_attr(response, "choices") or []
+        if not choices:
+            return Message(role=MessageRole.ASSISTANT, content="")
+            
+        choice = choices[0]
+        msg = self._get_attr(choice, "message")
+        
+        tool_calls: List[ToolCallPayload] = []
+        tc_data = self._get_attr(msg, "tool_calls")
+        if tc_data:
+            for idx, tc in enumerate(tc_data):
+                f_data = self._get_attr(tc, "function") or {}
+                function_name = self._get_attr(f_data, "name") or ""
+                arguments = self._get_attr(f_data, "arguments") or ""
+                if not isinstance(arguments, str):
+                    arguments = str(arguments)
+                call_id = self._get_attr(tc, "id")
+                if not call_id:
+                    call_id = self._build_tool_call_id(function_name, arguments, fallback_prefix=f"tool_call_{idx}")
+                tool_calls.append(ToolCallPayload(
+                    id=call_id,
+                    function_name=function_name,
+                    arguments=arguments,
+                    type="function"
+                ))
+        
+        content = self._get_attr(msg, "content") or ""
+        content = self._strip_thinking_tokens(content)
+
+        return Message(
+            role=MessageRole.ASSISTANT,
+            content=content,
+            tool_calls=tool_calls
+        )
+
+    _THINK_PATTERN = re.compile(r"<think>.*?</think>\s*", re.DOTALL)
+
+    @classmethod
+    def _strip_thinking_tokens(cls, text: str) -> str:
+        """Strip <think>...</think> blocks from model output (e.g. DeepSeek-R1, MiniMax-M2.7)."""
+        if "<think>" not in text:
+            return text
+        return cls._THINK_PATTERN.sub("", text).strip()
+
+    def _append_chat_response_output(self, timeline: List[Any], response: Any) -> None:
+        """Add chat response to timeline, preserving tool_calls (Chat API compatible)."""
+        msg = response.choices[0].message
+        content = self._strip_thinking_tokens(msg.content or "")
+        assistant_msg = {
+            "role": "assistant",
+            "content": content
+        }
+
+        if getattr(msg, "tool_calls", None):
+            assistant_msg["tool_calls"] = []
+            for idx, tc in enumerate(msg.tool_calls):
+                function_name = tc.function.name
+                arguments = tc.function.arguments or ""
+                if not isinstance(arguments, str):
+                    arguments = str(arguments)
+                call_id = tc.id or self._build_tool_call_id(function_name, arguments, fallback_prefix=f"tool_call_{idx}")
+                assistant_msg["tool_calls"].append({
+                    "id": call_id,
+                    "type": "function",
+                    "function": {
+                        "name": function_name,
+                        "arguments": arguments,
+                    },
+                })
+
+        timeline.append(assistant_msg)
+
+    def _serialize_timeline_item(self, item: Any) -> Optional[Any]:
+        if isinstance(item, Message):
+            return self._serialize_message_for_responses(item)
+        if isinstance(item, FunctionCallOutputEvent):
+            return self._serialize_function_call_output_event(item)
+        return item
+
+    def _serialize_message_for_responses(self, message: Message) -> Dict[str, Any]:
+        """Convert internal Message to Responses input schema."""
+        role_value = message.role.value
+        content_blocks = self._serialize_content_blocks(message)
+        payload: Dict[str, Any] = {
+            "role": role_value,
+            "content": content_blocks,
+        }
+        if message.name:
+            payload["name"] = message.name
+        if message.tool_call_id:
+            payload["tool_call_id"] = message.tool_call_id
+        return payload
+
+    def _serialize_content_blocks(self, message: Message) -> List[Dict[str, Any]]:
+        blocks = message.blocks()
+        if not blocks:
+            text = message.text_content()
+            block_type = "output_text" if message.role is MessageRole.ASSISTANT else "input_text"
+            return [{"type": block_type, "text": text}]
+
+        return self._serialize_blocks(blocks, message.role)
+
+    def _serialize_blocks(self, blocks: List[MessageBlock], role: MessageRole) -> List[Dict[str, Any]]:
+        serialized: List[Dict[str, Any]] = []
+        for block in blocks:
+            serialized.append(self._serialize_block(block, role))
+        return serialized
+
+    def _serialize_block(self, block: MessageBlock, role: MessageRole) -> Dict[str, Any]:
+        if block.type is MessageBlockType.TEXT:
+            content_type = "output_text" if role is MessageRole.ASSISTANT else "input_text"
+            return {
+                "type": content_type,
+                "text": block.text or "",
+            }
+
+        attachment = block.attachment
+        if block.type is MessageBlockType.IMAGE:
+            media_type = "output_image" if role is MessageRole.ASSISTANT else "input_image"
+            return self._serialize_media_block(media_type, attachment)
+        if block.type is MessageBlockType.AUDIO:
+            media_type = "output_audio" if role is MessageRole.ASSISTANT else "input_audio"
+            return self._serialize_media_block(media_type, attachment)
+        if block.type is MessageBlockType.VIDEO:
+            media_type = "output_video" if role is MessageRole.ASSISTANT else "input_video"
+            return self._serialize_media_block(media_type, attachment)
+        if block.type is MessageBlockType.FILE:
+            inline_text = self._maybe_inline_text_file(block)
+            if inline_text is not None:
+                content_type = "output_text" if role is MessageRole.ASSISTANT else "input_text"
+                return {
+                    "type": content_type,
+                    "text": inline_text,
+                }
+            return self._serialize_file_block(attachment, block)
+
+        # Fallback: treat as text/data
+        return {
+            "type": "input_text",
+            "text": block.describe(),
+        }
+
+    def _serialize_media_block(
+        self,
+        media_type: str,
+        attachment: Optional[AttachmentRef],
+    ) -> Dict[str, Any]:
+        payload: Dict[str, Any] = {"type": media_type}
+        if not attachment:
+            return payload
+
+        url_key = {
+            "input_image": "image_url",
+            "output_image": "image_url",
+            "input_audio": "audio_url",
+            "output_audio": "audio_url",
+            "input_video": "video_url",
+            "output_video": "video_url",
+        }.get(media_type)
+
+        if attachment.remote_file_id:
+            payload["file_id"] = attachment.remote_file_id
+        elif attachment.data_uri and url_key:
+            payload[url_key] = attachment.data_uri
+        elif attachment.local_path and url_key:
+            payload[url_key] = self._make_data_uri_from_path(attachment.local_path, attachment.mime_type)
+        return payload
+
+    def _serialize_file_block(
+        self,
+        attachment: Optional[AttachmentRef],
+        block: MessageBlock,
+    ) -> Dict[str, Any]:
+        payload: Dict[str, Any] = {"type": "input_file"}
+        if attachment:
+            if attachment.remote_file_id:
+                payload["file_id"] = attachment.remote_file_id
+            else:
+                data_uri = attachment.data_uri
+                if not data_uri and attachment.local_path:
+                    data_uri = self._make_data_uri_from_path(attachment.local_path, attachment.mime_type)
+                if data_uri:
+                    payload["file_data"] = data_uri
+                else:
+                    raise ValueError("Attachment missing file_id or data for input_file block")
+            if attachment.name:
+                payload["filename"] = attachment.name
+        else:
+            raise ValueError("File block requires an attachment reference")
+        return payload
+
+    def _maybe_inline_text_file(self, block: MessageBlock) -> Optional[str]:
+        """Inline local text/* attachments to avoid unsupported file-type uploads."""
+
+        attachment = block.attachment
+        if not attachment:
+            return None
+
+        mime = (attachment.mime_type or "").lower()
+        name = (attachment.name or "").lower()
+        is_json = mime in {
+            "application/json",
+            "application/jsonl",
+            "application/x-ndjson",
+            "application/ndjson",
+        } or name.endswith((".json", ".jsonl", ".ndjson"))
+        if not (mime.startswith("text/") or is_json):
+            return None
+        if attachment.remote_file_id:
+            return None  # nothing to inline if already remote-only
+
+        text = self._read_attachment_text(attachment)
+        if text is None:
+            return None
+
+        is_csv = "text/csv" in mime or name.endswith(".csv")
+        limit_attr = "csv_inline_char_limit" if is_csv else "text_inline_char_limit"
+        default_limit = self.CSV_INLINE_CHAR_LIMIT if is_csv else self.TEXT_INLINE_CHAR_LIMIT
+        limit = getattr(self, limit_attr, default_limit)
+        truncated = False
+        if len(text) > limit:
+            text = text[:limit]
+            truncated = True
+
+        display_name = attachment.name or attachment.attachment_id or ("attachment.csv" if is_csv else "attachment.txt")
+        suffix = "\n\n[truncated after %d characters]" % limit if truncated else ""
+        if is_csv:
+            return f"CSV file '{display_name}':\n{text}{suffix}"
+        mime_display = attachment.mime_type or "text/*"
+        return f"Text file '{display_name}' ({mime_display}):\n```text\n{text}\n```{suffix}"
+
+    def _maybe_inline_csv(self, block: MessageBlock) -> Optional[str]:
+        """Backward compatible alias for older call sites/tests."""
+        return self._maybe_inline_text_file(block)
+
+    def _read_attachment_text(self, attachment: AttachmentRef) -> Optional[str]:
+        data_bytes: Optional[bytes] = None
+        if attachment.data_uri:
+            data_bytes = self._decode_data_uri(attachment.data_uri)
+        elif attachment.local_path and os.path.exists(attachment.local_path):
+            try:
+                with open(attachment.local_path, "rb") as handle:
+                    data_bytes = handle.read()
+            except OSError:
+                return None
+        if data_bytes is None:
+            return None
+        try:
+            return data_bytes.decode("utf-8")
+        except UnicodeDecodeError:
+            return data_bytes.decode("utf-8", errors="replace")
+
+    def _decode_data_uri(self, data_uri: str) -> Optional[bytes]:
+        if not data_uri.startswith("data:"):
+            return None
+        header, _, data = data_uri.partition(",")
+        if not _:
+            return None
+        if ";base64" in header:
+            try:
+                return base64.b64decode(data)
+            except (ValueError, binascii.Error):
+                return None
+        return unquote_to_bytes(data)
+
+    def _deserialize_response(self, response: Any) -> Message:
+        """Convert Responses API output to internal Message."""
+        output_blocks = getattr(response, "output", []) or []
+        assistant_blocks: List[MessageBlock] = []
+        tool_calls: List[ToolCallPayload] = []
+
+        for item in output_blocks:
+            item_type = self._get_attr(item, "type")
+            if item_type == "message":
+                role_value = self._get_attr(item, "role") or "assistant"
+                if role_value != "assistant":
+                    continue
+                content_items = self._get_attr(item, "content") or []
+                parsed_blocks, parsed_calls = self._parse_output_content(content_items)
+                assistant_blocks.extend(parsed_blocks)
+                tool_calls.extend(parsed_calls)
+            elif item_type == "image_generation_call":
+                assistant_blocks.append(self._parse_image_generation_call(item))
+            elif item_type in {"tool_call", "function_call"}:
+                parsed_call = self._parse_tool_call(item)
+                if parsed_call:
+                    tool_calls.append(parsed_call)
+
+        if not assistant_blocks:
+            fallback_text = self._extract_fallback_text(response)
+            if fallback_text:
+                assistant_blocks.append(MessageBlock(MessageBlockType.TEXT, text=fallback_text))
+
+        return Message(
+            role=MessageRole.ASSISTANT,
+            content=assistant_blocks or "",
+            tool_calls=tool_calls,
+        )
+
+    def _extract_fallback_text(self, response: Any) -> Optional[str]:
+        """Return the concatenated output text without triggering Responses errors."""
+        output = getattr(response, "output", None)
+        if not output:
+            return None
+        try:
+            return getattr(response, "output_text", None)
+        except TypeError:
+            # OpenAI SDK raises TypeError when output is None; treat as missing text
+            return None
+        except AttributeError:
+            return None
+
+    def _parse_output_content(
+        self,
+        content_items: List[Any],
+    ) -> tuple[List[MessageBlock], List[ToolCallPayload]]:
+        blocks: List[MessageBlock] = []
+        tool_calls: List[ToolCallPayload] = []
+        for part in content_items:
+            part_type = self._get_attr(part, "type")
+            if part_type in {"output_text", "text"}:
+                blocks.append(MessageBlock(MessageBlockType.TEXT, text=self._get_attr(part, "text") or ""))
+            elif part_type in {"output_image", "image"}:
+                blocks.append(
+                    MessageBlock(
+                        type=MessageBlockType.IMAGE,
+                        attachment=AttachmentRef(
+                            attachment_id=self._get_attr(part, "id") or "",
+                            data_uri=self._get_attr(part, "image_base64"),
+                            metadata=self._get_attr(part, "metadata") or {},
+                        ),
+                    )
+                )
+            elif part_type in {"tool_call", "function_call"}:
+                parsed = self._parse_tool_call(part)
+                if parsed:
+                    tool_calls.append(parsed)
+            else:
+                blocks.append(
+                    MessageBlock(
+                        type=MessageBlockType.DATA,
+                        text=str(self._get_attr(part, "text") or ""),
+                        data=self._maybe_to_dict(part),
+                    )
+                )
+        return blocks, tool_calls
+
+    def _parse_image_generation_call(self, payload: Any) -> MessageBlock:
+        status = self._get_attr(payload, "status") or ""
+        if status != "completed":
+            raise RuntimeError(f"Image generation call not completed (status={status})")
+        image_b64 = self._get_attr(payload, "result")
+        if not image_b64:
+            raise RuntimeError("Image generation call returned empty result")
+        attachment_id = self._get_attr(payload, "id") or ""
+        data_uri = f"data:image/png;base64,{image_b64}"
+        return MessageBlock(
+            type=MessageBlockType.IMAGE,
+            attachment=AttachmentRef(
+                attachment_id=attachment_id,
+                data_uri=data_uri,
+                metadata={"source": "image_generation_call"},
+            ),
+        )
+
+    def _parse_tool_call(self, payload: Any) -> Optional[ToolCallPayload]:
+        function_payload = self._get_attr(payload, "function") or {}
+        function_name = self._get_attr(function_payload, "name") or self._get_attr(payload, "name") or ""
+        arguments = self._get_attr(function_payload, "arguments") or self._get_attr(payload, "arguments") or ""
+        if not function_name:
+            return None
+        if isinstance(arguments, (dict, list)):
+            try:
+                import json
+
+                arguments_str = json.dumps(arguments, ensure_ascii=False)
+            except Exception:
+                arguments_str = str(arguments)
+        else:
+            arguments_str = str(arguments)
+        call_id = self._get_attr(payload, "call_id") or self._get_attr(payload, "id") or ""
+        if not call_id:
+            call_id = self._build_tool_call_id(function_name, arguments_str)
+        return ToolCallPayload(
+            id=call_id,
+            function_name=function_name,
+            arguments=arguments_str,
+            type="function",
+        )
+
+    def _build_tool_call_id(self, function_name: str, arguments: str, *, fallback_prefix: str = "tool_call") -> str:
+        base = function_name or fallback_prefix
+        payload = f"{base}:{arguments or ''}".encode("utf-8")
+        digest = hashlib.md5(payload).hexdigest()[:8]
+        return f"{base}_{digest}"
+
+    def _get_attr(self, payload: Any, key: str) -> Any:
+        if hasattr(payload, key):
+            return getattr(payload, key)
+        if isinstance(payload, dict):
+            return payload.get(key)
+        return None
+
+    def _maybe_to_dict(self, payload: Any) -> Dict[str, Any]:
+        if hasattr(payload, "model_dump"):
+            try:
+                return payload.model_dump()
+            except Exception:
+                return {}
+        if isinstance(payload, dict):
+            return payload
+        return {}
+
+    def _make_data_uri_from_path(self, path: str, mime_type: Optional[str]) -> str:
+        mime = mime_type or "application/octet-stream"
+        file_size = os.path.getsize(path)
+        if file_size > self.MAX_INLINE_FILE_BYTES:
+            raise ValueError(
+                f"Attachment '{path}' is {file_size} bytes; exceeds inline limit of {self.MAX_INLINE_FILE_BYTES} bytes"
+            )
+        with open(path, "rb") as handle:
+            encoded = base64.b64encode(handle.read()).decode("utf-8")
+        return f"data:{mime};base64,{encoded}"
+
+    def _serialize_function_call_output_event(
+        self,
+        event: FunctionCallOutputEvent,
+    ) -> Dict[str, Any]:
+        payload: Dict[str, Any] = {
+            "type": event.type,
+            "call_id": event.call_id or event.function_name or "tool_call",
+        }
+        if event.output_blocks:
+            payload["output"] = self._serialize_blocks(event.output_blocks, MessageRole.TOOL)
+        else:
+            text = event.output_text or ""
+            payload["output"] = [
+                {
+                    "type": "input_text",
+                    "text": text,
+                }
+            ]
+        return payload
+
+    def _append_response_output(self, timeline: List[Any], response: Any) -> None:
+        output = getattr(response, "output", None)
+        if not output:
+            return
+        timeline.extend(output)

runtime/node/agent/providers/response.py

@@ -0,0 +1,39 @@
+"""Normalized provider response dataclasses."""
+
+from dataclasses import dataclass
+from typing import Any
+
+from entity.messages import Message
+
+
+@dataclass
+class ModelResponse:
+    """Represents a provider response with normalized message payload."""
+
+    message: Message
+    raw_response: Any | None = None
+
+    def has_tool_calls(self) -> bool:
+        return bool(self.message.tool_calls)
+
+    def to_dict(self) -> dict:
+        """Return a simple dict representation for compatibility."""
+        payload = {
+            "role": self.message.role.value,
+        }
+        if isinstance(self.message.content, list):
+            payload["content"] = [
+                block.to_dict() if hasattr(block, "to_dict") else block for block in self.message.content  # type: ignore[arg-type]
+            ]
+        else:
+            payload["content"] = self.message.content
+        if self.message.tool_calls:
+            payload["tool_calls"] = [call.to_openai_dict() for call in self.message.tool_calls]
+        if self.message.tool_call_id:
+            payload["tool_call_id"] = self.message.tool_call_id
+        if self.message.name:
+            payload["name"] = self.message.name
+        return payload
+
+    def str_raw_response(self):
+        return self.raw_response.__str__()