chore: import upstream snapshot with attribution

utils/attachments.py

@@ -0,0 +1,349 @@
+"""Attachment storage and serialization helpers."""
+
+import base64
+import hashlib
+import json
+import mimetypes
+import shutil
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from entity.messages import AttachmentRef, MessageBlock, MessageBlockType
+
+DEFAULT_INLINE_LIMIT = 512 * 1024  # 512 KB
+
+
+@dataclass
+class AttachmentRecord:
+    """Stores metadata about an attachment tracked inside a workflow run."""
+
+    ref: AttachmentRef
+    kind: MessageBlockType = MessageBlockType.FILE
+    description: Optional[str] = None
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "ref": self.ref.to_dict(),
+            "kind": self.kind.value,
+            "description": self.description,
+            "extra": self.extra,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "AttachmentRecord":
+        ref_data = data.get("ref") or {}
+        raw_kind = data.get("kind", MessageBlockType.FILE.value)
+        try:
+            kind = MessageBlockType(raw_kind)
+        except ValueError:
+            kind = MessageBlockType.FILE
+        return cls(
+            ref=AttachmentRef.from_dict(ref_data),
+            kind=kind,
+            description=data.get("description"),
+            extra=data.get("extra") or {},
+        )
+
+    def as_message_block(self) -> MessageBlock:
+        """Convert to a MessageBlock referencing this attachment."""
+        return MessageBlock(
+            type=self.kind,
+            attachment=self.ref.copy(),
+            data=dict(self.extra),
+        )
+
+
+class AttachmentStore:
+    """Filesystem-backed attachment manifest for a workflow execution."""
+
+    def __init__(self, root_dir: Path | str, inline_size_limit: int = DEFAULT_INLINE_LIMIT) -> None:
+        self.root = Path(root_dir)
+        self.inline_size_limit = inline_size_limit
+        self.root.mkdir(parents=True, exist_ok=True)
+        self.manifest_path = self.root / "attachments_manifest.json"
+        self._records: Dict[str, AttachmentRecord] = {}
+        self._persistent_ids: set[str] = set()
+        self._hash_index: Dict[str, str] = {}
+        self._load_manifest()
+
+    def register_file(
+        self,
+        file_path: Path | str,
+        *,
+        kind: MessageBlockType = MessageBlockType.FILE,
+        display_name: Optional[str] = None,
+        mime_type: Optional[str] = None,
+        attachment_id: Optional[str] = None,
+        copy_file: bool = True,
+        description: Optional[str] = None,
+        extra: Optional[Dict[str, Any]] = None,
+        persist: bool = True,
+        deduplicate: bool = False,
+    ) -> AttachmentRecord:
+        """Register a local file and return its attachment record."""
+        source = Path(file_path)
+        if not source.exists():
+            raise FileNotFoundError(f"Attachment source not found: {source}")
+
+        guessed_mime = mime_type or (mimetypes.guess_type(source.name)[0] or "application/octet-stream")
+        attachment_id = attachment_id or uuid.uuid4().hex
+        sha256_source = _sha256_file(source)
+
+        if deduplicate:
+            existing = self._find_duplicate_by_hash(
+                sha256_source,
+                copy_file=copy_file,
+                source_path=source,
+            )
+            if existing:
+                return existing
+        if copy_file:
+            target_dir = self.root / attachment_id
+            target_dir.mkdir(parents=True, exist_ok=True)
+            target_path = target_dir / source.name
+            shutil.copy2(source, target_path)
+        else:
+            target_path = source.resolve()
+
+        size = target_path.stat().st_size
+        sha256 = sha256_source or _sha256_file(target_path)
+        data_uri = None
+        # if size <= self.inline_size_limit:
+        #     data_uri = encode_file_to_data_uri(target_path, guessed_mime)
+
+        ref = AttachmentRef(
+            attachment_id=attachment_id,
+            mime_type=guessed_mime,
+            name=display_name or source.name,
+            size=size,
+            sha256=sha256,
+            local_path=str(target_path),
+            data_uri=data_uri,
+        )
+        record = AttachmentRecord(
+            ref=ref,
+            kind=kind,
+            description=description,
+            extra=dict(extra) if extra else {},
+        )
+        self._records[attachment_id] = record
+        if sha256:
+            self._hash_index[sha256] = attachment_id
+        if persist:
+            self._persistent_ids.add(attachment_id)
+            self._save_manifest()
+        else:
+            self._persistent_ids.discard(attachment_id)
+        return record
+
+    def register_bytes(
+        self,
+        data: bytes | bytearray,
+        *,
+        kind: MessageBlockType = MessageBlockType.FILE,
+        mime_type: Optional[str] = None,
+        display_name: Optional[str] = None,
+        attachment_id: Optional[str] = None,
+        description: Optional[str] = None,
+        extra: Optional[Dict[str, Any]] = None,
+        persist: bool = True,
+    ) -> AttachmentRecord:
+        """Register an in-memory payload as an attachment."""
+
+        if not isinstance(data, (bytes, bytearray)):
+            raise TypeError("register_bytes expects bytes or bytearray data")
+
+        attachment_id = attachment_id or uuid.uuid4().hex
+        filename = display_name or _default_filename_for_mime(mime_type)
+
+        target_dir = self.root / attachment_id
+        target_dir.mkdir(parents=True, exist_ok=True)
+        target_path = target_dir / filename
+        with target_path.open("wb") as handle:
+            handle.write(bytes(data))
+
+        return self.register_file(
+            target_path,
+            kind=kind,
+            display_name=display_name or filename,
+            mime_type=mime_type,
+            attachment_id=attachment_id,
+            copy_file=False,
+            description=description,
+            extra=extra,
+            persist=persist,
+        )
+
+    def register_remote_file(
+        self,
+        *,
+        remote_file_id: str,
+        name: str,
+        mime_type: Optional[str] = None,
+        size: Optional[int] = None,
+        kind: MessageBlockType = MessageBlockType.FILE,
+        attachment_id: Optional[str] = None,
+        description: Optional[str] = None,
+        extra: Optional[Dict[str, Any]] = None,
+        persist: bool = True,
+    ) -> AttachmentRecord:
+        """Register an already-uploaded file (e.g., OpenAI file ID)."""
+        attachment_id = attachment_id or uuid.uuid4().hex
+        ref = AttachmentRef(
+            attachment_id=attachment_id,
+            mime_type=mime_type,
+            name=name,
+            size=size,
+            remote_file_id=remote_file_id,
+        )
+        record = AttachmentRecord(ref=ref, kind=kind, description=description, extra=extra or {})
+        self._records[attachment_id] = record
+        if persist:
+            self._persistent_ids.add(attachment_id)
+            self._save_manifest()
+        else:
+            self._persistent_ids.discard(attachment_id)
+        if ref.sha256:
+            self._hash_index[ref.sha256] = attachment_id
+        return record
+
+    def update_remote_file_id(self, attachment_id: str, remote_file_id: str) -> None:
+        """Attach a provider file_id to an existing record (after upload)."""
+        record = self._records.get(attachment_id)
+        if not record:
+            raise KeyError(f"Attachment '{attachment_id}' not found")
+        record.ref.remote_file_id = remote_file_id
+        if attachment_id in self._persistent_ids:
+            self._save_manifest()
+
+    def get(self, attachment_id: str) -> AttachmentRecord | None:
+        return self._records.get(attachment_id)
+
+    def to_message_block(self, attachment_id: str) -> MessageBlock:
+        record = self._records.get(attachment_id)
+        if not record:
+            raise KeyError(f"Attachment '{attachment_id}' not found")
+        return record.as_message_block()
+
+    def list_records(self) -> Dict[str, AttachmentRecord]:
+        return dict(self._records)
+
+    def export_manifest(self) -> Dict[str, Any]:
+        return {
+            attachment_id: record.to_dict()
+            for attachment_id, record in self._records.items()
+            if attachment_id in self._persistent_ids
+        }
+
+    def _find_duplicate_by_hash(
+        self,
+        sha256: Optional[str],
+        *,
+        copy_file: bool,
+        source_path: Optional[Path],
+    ) -> Optional[AttachmentRecord]:
+        if not sha256:
+            return None
+        existing_id = self._hash_index.get(sha256)
+        if not existing_id:
+            return None
+        record = self._records.get(existing_id)
+        if not record:
+            self._hash_index.pop(sha256, None)
+            return None
+        if not copy_file and source_path is not None:
+            existing_path = record.ref.local_path
+            if not existing_path:
+                return None
+            try:
+                if Path(existing_path).resolve() != source_path.resolve():
+                    return None
+            except FileNotFoundError:
+                return None
+        return record
+
+    def ingest_record(
+        self,
+        record: AttachmentRecord,
+        *,
+        copy_file: bool = True,
+        persist: bool = True,
+    ) -> AttachmentRecord:
+        """
+        Import an existing attachment record (e.g., from a session upload) into this store.
+        Optionally copies the underlying file into the store directory.
+        """
+        source_ref = record.ref
+        attachment_id = source_ref.attachment_id or uuid.uuid4().hex
+        new_ref = source_ref.copy()
+        new_ref.attachment_id = attachment_id
+
+        local_path = source_ref.local_path
+        if local_path and copy_file:
+            source_path = Path(local_path)
+            if source_path.exists():
+                target_dir = self.root / attachment_id
+                target_dir.mkdir(parents=True, exist_ok=True)
+                target_path = target_dir / source_path.name
+                shutil.copy2(source_path, target_path)
+                new_ref.local_path = str(target_path)
+        self._records[attachment_id] = AttachmentRecord(
+            ref=new_ref,
+            kind=record.kind,
+            description=record.description,
+            extra=dict(record.extra),
+        )
+        if persist:
+            self._persistent_ids.add(attachment_id)
+            self._save_manifest()
+        else:
+            self._persistent_ids.discard(attachment_id)
+        if new_ref.sha256:
+            self._hash_index[new_ref.sha256] = attachment_id
+        return self._records[attachment_id]
+
+    def _load_manifest(self) -> None:
+        if not self.manifest_path.exists():
+            return
+        try:
+            data = json.loads(self.manifest_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            return
+        for attachment_id, record_data in data.items():
+            try:
+                record = AttachmentRecord.from_dict(record_data)
+            except Exception:
+                continue
+            self._records[attachment_id] = record
+            self._persistent_ids.add(attachment_id)
+            if record.ref.sha256:
+                self._hash_index[record.ref.sha256] = attachment_id
+
+    def _save_manifest(self) -> None:
+        serialized = self.export_manifest()
+        self.manifest_path.write_text(json.dumps(serialized, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def _sha256_file(path: Path) -> str:
+    hasher = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def encode_file_to_data_uri(path: Path, mime_type: str) -> str:
+    data = path.read_bytes()
+    encoded = base64.b64encode(data).decode("utf-8")
+    return f"data:{mime_type};base64,{encoded}"
+
+
+def _default_filename_for_mime(mime_type: Optional[str]) -> str:
+    if mime_type:
+        ext = mimetypes.guess_extension(mime_type)
+        if ext:
+            return f"attachment{ext}"
+    return "attachment.bin"

utils/env_loader.py

@@ -0,0 +1,36 @@
+"""Environment loading utilities for root-level vars interpolation."""
+
+
+import os
+from pathlib import Path
+from typing import Dict
+
+_DOTENV_LOADED = False
+
+
+def load_dotenv_file(dotenv_path: Path | None = None) -> None:
+    """Populate ``os.environ`` with key/value pairs from a .env file once per process."""
+    global _DOTENV_LOADED
+    if _DOTENV_LOADED:
+        return
+
+    path = dotenv_path or Path(".env")
+    if path.exists():
+        for line in path.read_text(encoding="utf-8").splitlines():
+            stripped = line.strip()
+            if not stripped or stripped.startswith("#"):
+                continue
+            if "=" not in stripped:
+                continue
+            key, value = stripped.split("=", 1)
+            key = key.strip()
+            value = value.strip().strip('"').strip("'")
+            os.environ.setdefault(key, value)
+
+    _DOTENV_LOADED = True
+
+
+def build_env_var_map(extra_vars: Dict[str, str] | None = None) -> Dict[str, str]:
+    merged: Dict[str, str] = dict(os.environ)
+    merged.update(extra_vars or {})
+    return merged

utils/error_handler.py

@@ -0,0 +1,147 @@
+"""Error handling utilities for the DevAll workflow system."""
+
+from fastapi import Request
+from fastapi.responses import JSONResponse
+from fastapi.encoders import jsonable_encoder
+import traceback
+
+from utils.structured_logger import get_server_logger
+from utils.exceptions import MACException, ValidationError, SecurityError, ConfigurationError, \
+    WorkflowExecutionError, ResourceNotFoundError, ResourceConflictError, TimeoutError, ExternalServiceError
+
+
+# Error code mapping to HTTP status codes
+ERROR_CODE_TO_STATUS = {
+    "VALIDATION_ERROR": 400,
+    "SECURITY_ERROR": 403,
+    "CONFIGURATION_ERROR": 500,
+    "WORKFLOW_EXECUTION_ERROR": 500,
+    "RESOURCE_NOT_FOUND": 404,
+    "RESOURCE_CONFLICT": 409,
+    "TIMEOUT_ERROR": 408,
+    "EXTERNAL_SERVICE_ERROR": 502,
+    "GENERIC_ERROR": 500
+}
+
+
+async def handle_validation_error(request: Request, exc: ValidationError) -> JSONResponse:
+    """Handle validation errors."""
+    return await handle_mac_exception(request, exc)
+
+
+async def handle_security_error(request: Request, exc: SecurityError) -> JSONResponse:
+    """Handle security errors."""
+    return await handle_mac_exception(request, exc)
+
+
+async def handle_configuration_error(request: Request, exc: ConfigurationError) -> JSONResponse:
+    """Handle configuration errors."""
+    return await handle_mac_exception(request, exc)
+
+
+async def handle_workflow_execution_error(request: Request, exc: WorkflowExecutionError) -> JSONResponse:
+    """Handle workflow execution errors."""
+    return await handle_mac_exception(request, exc)
+
+
+async def handle_resource_not_found_error(request: Request, exc: ResourceNotFoundError) -> JSONResponse:
+    """Handle resource not found errors."""
+    return await handle_mac_exception(request, exc)
+
+
+async def handle_resource_conflict_error(request: Request, exc: ResourceConflictError) -> JSONResponse:
+    """Handle resource conflict errors."""
+    return await handle_mac_exception(request, exc)
+
+
+async def handle_timeout_error(request: Request, exc: TimeoutError) -> JSONResponse:
+    """Handle timeout errors."""
+    return await handle_mac_exception(request, exc)
+
+
+async def handle_external_service_error(request: Request, exc: ExternalServiceError) -> JSONResponse:
+    """Handle external service errors."""
+    return await handle_mac_exception(request, exc)
+
+
+async def handle_mac_exception(request: Request, exc: MACException) -> JSONResponse:
+    """Handle DevAll exceptions and return standardized error response."""
+    logger = get_server_logger()
+    
+    # Log the error
+    logger.log_exception(
+        exc,
+        f"DevAll exception occurred: {exc.error_code} - {exc.message}",
+        correlation_id=getattr(request.state, 'correlation_id', None),
+        url=str(request.url),
+        method=request.method
+    )
+    
+    # Determine the HTTP status code
+    status_code = ERROR_CODE_TO_STATUS.get(exc.error_code, 500)
+    
+    # Prepare response data
+    response_data = {
+        "error": {
+            "code": exc.error_code,
+            "message": exc.message,
+            "details": exc.details
+        },
+        "timestamp": exc.__dict__.get('_timestamp', __import__('datetime').datetime.utcnow().isoformat())
+    }
+    
+    return JSONResponse(
+        status_code=status_code,
+        content=jsonable_encoder(response_data)
+    )
+
+
+async def handle_general_exception(request: Request, exc: Exception) -> JSONResponse:
+    """Handle general exceptions and return standardized error response."""
+    logger = get_server_logger()
+    
+    # Log the error with traceback
+    logger.log_exception(
+        exc,
+        f"General exception occurred: {type(exc).__name__} - {str(exc)}",
+        correlation_id=getattr(request.state, 'correlation_id', None),
+        url=str(request.url),
+        method=request.method
+    )
+    
+    # For security, don't expose internal error details to the client
+    error_details = {
+        "code": "INTERNAL_ERROR",
+        "message": "An internal server error occurred",
+        "details": {}  # Don't send internal details to client
+    }
+    
+    # In development, we might want to include more details
+    import os
+    if os.getenv("ENVIRONMENT") == "development":
+        error_details["details"]["debug_info"] = {
+            "exception_type": type(exc).__name__,
+            "exception_message": str(exc),
+            "traceback": traceback.format_exc()
+        }
+    
+    return JSONResponse(
+        status_code=500,
+        content=jsonable_encoder({"error": error_details})
+    )
+
+
+def add_exception_handlers(app):
+    """Add exception handlers to FastAPI app."""
+    app.add_exception_handler(ValidationError, handle_validation_error)
+    app.add_exception_handler(SecurityError, handle_security_error)
+    app.add_exception_handler(ConfigurationError, handle_configuration_error)
+    app.add_exception_handler(WorkflowExecutionError, handle_workflow_execution_error)
+    app.add_exception_handler(ResourceNotFoundError, handle_resource_not_found_error)
+    app.add_exception_handler(ResourceConflictError, handle_resource_conflict_error)
+    app.add_exception_handler(TimeoutError, handle_timeout_error)
+    app.add_exception_handler(ExternalServiceError, handle_external_service_error)
+    app.add_exception_handler(MACException, handle_mac_exception)
+    app.add_exception_handler(Exception, handle_general_exception)
+    
+    return app

utils/exceptions.py

@@ -0,0 +1,115 @@
+"""Custom exceptions for the DevAll workflow system."""
+
+from typing import Optional, Dict, Any
+import json
+
+
+class MACException(Exception):
+    """Base exception for DevAll workflow system."""
+    
+    def __init__(self, message: str, error_code: str = None, details: Dict[str, Any] = None):
+        super().__init__(message)
+        self.message = message
+        self.error_code = error_code or "GENERIC_ERROR"
+        self.details = details or {}
+        
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert exception to dictionary format for JSON response."""
+        return {
+            "error_code": self.error_code,
+            "message": self.message,
+            "details": self.details
+        }
+    
+    def to_json(self) -> str:
+        """Convert exception to JSON string."""
+        return json.dumps(self.to_dict())
+
+
+class ValidationError(MACException):
+    """Raised when validation fails."""
+    
+    def __init__(self, message: str, field: str = None, details: Dict[str, Any] = None):
+        super().__init__(message, "VALIDATION_ERROR", details or {})
+        if field:
+            self.details["field"] = field
+
+
+class SecurityError(MACException):
+    """Raised when a security violation occurs."""
+    
+    def __init__(self, message: str, details: Dict[str, Any] = None):
+        super().__init__(message, "SECURITY_ERROR", details or {})
+
+
+class ConfigurationError(MACException):
+    """Raised when configuration is invalid or missing."""
+    
+    def __init__(self, message: str, config_key: str = None, details: Dict[str, Any] = None):
+        super().__init__(message, "CONFIGURATION_ERROR", details or {})
+        if config_key:
+            self.details["config_key"] = config_key
+
+
+class WorkflowExecutionError(MACException):
+    """Raised when workflow execution fails."""
+    
+    def __init__(self, message: str, workflow_id: str = None, node_id: str = None, details: Dict[str, Any] = None):
+        super().__init__(message, "WORKFLOW_EXECUTION_ERROR", details or {})
+        if workflow_id:
+            self.details["workflow_id"] = workflow_id
+        if node_id:
+            self.details["node_id"] = node_id
+
+
+class WorkflowCancelledError(MACException):
+    """Raised when a workflow execution is cancelled mid-flight."""
+
+    def __init__(self, message: str, workflow_id: str = None, details: Dict[str, Any] = None):
+        super().__init__(message, "WORKFLOW_CANCELLED", details or {})
+        if workflow_id:
+            self.details["workflow_id"] = workflow_id
+
+
+class ResourceNotFoundError(MACException):
+    """Raised when a requested resource is not found."""
+    
+    def __init__(self, message: str, resource_type: str = None, resource_id: str = None, details: Dict[str, Any] = None):
+        super().__init__(message, "RESOURCE_NOT_FOUND", details or {})
+        if resource_type:
+            self.details["resource_type"] = resource_type
+        if resource_id:
+            self.details["resource_id"] = resource_id
+
+
+class ResourceConflictError(MACException):
+    """Raised when there's a conflict with an existing resource."""
+    
+    def __init__(self, message: str, resource_type: str = None, resource_id: str = None, details: Dict[str, Any] = None):
+        super().__init__(message, "RESOURCE_CONFLICT", details or {})
+        if resource_type:
+            self.details["resource_type"] = resource_type
+        if resource_id:
+            self.details["resource_id"] = resource_id
+
+
+class TimeoutError(MACException):
+    """Raised when an operation times out."""
+    
+    def __init__(self, message: str, operation: str = None, timeout_duration: float = None, details: Dict[str, Any] = None):
+        super().__init__(message, "TIMEOUT_ERROR", details or {})
+        if operation:
+            self.details["operation"] = operation
+        if timeout_duration is not None:
+            self.details["timeout_duration"] = timeout_duration
+
+
+class ExternalServiceError(MACException):
+    """Raised when an external service call fails."""
+    
+    def __init__(self, message: str, service_name: str = None, status_code: int = None, details: Dict[str, Any] = None):
+        super().__init__(message, "EXTERNAL_SERVICE_ERROR", details or {})
+        if service_name:
+            self.details["service_name"] = service_name
+        if status_code is not None:
+            self.details["status_code"] = status_code

utils/function_catalog.py

@@ -0,0 +1,353 @@
+"""Utility helpers for introspecting function-calling tools."""
+
+import inspect
+from collections import abc
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Any, Dict, List, Literal, Mapping, Sequence, Tuple, Union, get_args, get_origin
+
+from utils.function_manager import FUNCTION_CALLING_DIR, get_function_manager
+
+
+@dataclass(frozen=True)
+class ParamMeta:
+    """Declarative metadata for Annotated parameters."""
+
+    description: str | None = None
+    enum: Sequence[Any] | None = None
+
+
+@dataclass(frozen=True)
+class FunctionMetadata:
+    """Normalized metadata for a Python callable."""
+
+    name: str
+    description: str | None
+    parameters_schema: Dict[str, Any]
+    module: str
+    file_path: str
+    module_name: str
+
+
+class FunctionCatalog:
+    """Inspect and cache callable metadata for tool schemas."""
+
+    def __init__(self, functions_dir: str | Path = FUNCTION_CALLING_DIR) -> None:
+        self._functions_dir = Path(functions_dir).resolve()
+        self._metadata: Dict[str, FunctionMetadata] = {}
+        self._loaded = False
+        self._load_error: Exception | None = None
+        self._module_index: Dict[str, List[str]] = {}
+
+    def refresh(self) -> None:
+        """Reload metadata from the function directory."""
+        self._metadata.clear()
+        self._module_index = {}
+        self._load_error = None
+        manager = get_function_manager(self._functions_dir)
+        try:
+            manager.load_functions()
+        except Exception as exc:  # pragma: no cover - propagated via catalog usage
+            self._loaded = True
+            self._load_error = exc
+            return
+
+        module_index: Dict[str, List[str]] = {}
+        for name, fn in manager.list_functions().items():
+            try:
+                metadata = _build_function_metadata(name, fn, self._functions_dir)
+                self._metadata[name] = metadata
+                module_bucket = module_index.setdefault(metadata.module_name, [])
+                module_bucket.append(name)
+            except Exception as exc:  # pragma: no cover - guarded to avoid cascading failures
+                print(f"[FunctionCatalog] Failed to load metadata for {name}: {exc}")
+        for module_name, names in module_index.items():
+            names.sort()
+        self._module_index = module_index
+        self._loaded = True
+
+    def _ensure_loaded(self) -> None:
+        if not self._loaded:
+            self.refresh()
+
+    def get(self, name: str) -> FunctionMetadata | None:
+        self._ensure_loaded()
+        return self._metadata.get(name)
+
+    def list_function_names(self) -> List[str]:
+        self._ensure_loaded()
+        return sorted(self._metadata.keys())
+
+    def list_metadata(self) -> Dict[str, FunctionMetadata]:
+        self._ensure_loaded()
+        return self._metadata.copy()
+
+    def iter_modules(self) -> List[Tuple[str, List[FunctionMetadata]]]:
+        """Return functions grouped by Python file (module_name)."""
+
+        self._ensure_loaded()
+        modules: List[Tuple[str, List[FunctionMetadata]]] = []
+        for module_name in sorted(self._module_index.keys()):
+            names = self._module_index.get(module_name, [])
+            entries: List[FunctionMetadata] = []
+            for fn_name in names:
+                meta = self._metadata.get(fn_name)
+                if meta is not None:
+                    entries.append(meta)
+            modules.append((module_name, entries))
+        return modules
+
+    def functions_for_module(self, module_name: str) -> List[str]:
+        """Return sorted function names for the given module."""
+
+        self._ensure_loaded()
+        return list(self._module_index.get(module_name, []))
+
+    @property
+    def load_error(self) -> Exception | None:
+        self._ensure_loaded()
+        return self._load_error
+
+
+_catalog_registry: Dict[Path, FunctionCatalog] = {}
+
+
+def get_function_catalog(functions_dir: str | Path = FUNCTION_CALLING_DIR) -> FunctionCatalog:
+    directory = Path(functions_dir).resolve()
+    catalog = _catalog_registry.get(directory)
+    if catalog is None:
+        catalog = FunctionCatalog(directory)
+        _catalog_registry[directory] = catalog
+    return catalog
+
+
+def _build_function_metadata(name: str, fn: Any, functions_dir: Path) -> FunctionMetadata:
+    signature = inspect.signature(fn)
+    annotations = _resolve_annotations(fn)
+
+    description = _extract_description(fn)
+    schema = _build_parameters_schema(signature, annotations)
+    module = getattr(fn, "__module__", "")
+    file_path = inspect.getsourcefile(fn) or ""
+    module_name = _derive_module_name(file_path, functions_dir)
+    return FunctionMetadata(
+        name=name,
+        description=description,
+        parameters_schema=schema,
+        module=module,
+        file_path=file_path,
+        module_name=module_name,
+    )
+
+
+def _derive_module_name(file_path: str, functions_dir: Path) -> str:
+    if not file_path:
+        return "unknown"
+    try:
+        relative = Path(file_path).resolve().relative_to(functions_dir.resolve())
+        if relative.suffix:
+            relative = relative.with_suffix("")
+        parts = list(relative.parts)
+        if not parts:
+            return "unknown"
+        return "/".join(parts)
+    except Exception:
+        stem = Path(file_path).stem
+        return stem or "unknown"
+
+
+def _extract_description(fn: Any) -> str | None:
+    doc = inspect.getdoc(fn)
+    if not doc:
+        return None
+    trimmed = doc.strip()
+    if not trimmed:
+        return None
+    first_paragraph = trimmed.split("\n\n", 1)[0]
+    normalized_lines = [line.strip() for line in first_paragraph.splitlines() if line.strip()]
+    normalized = " ".join(normalized_lines)
+    max_len = 600
+    if len(normalized) > max_len:
+        normalized = normalized[: max_len - 1].rstrip() + "…"
+    return normalized or None
+
+
+def _resolve_annotations(fn: Any) -> Mapping[str, Any]:
+    fallback = getattr(fn, "__annotations__", {}) or {}
+    get_annotations = getattr(inspect, "get_annotations", None)
+    if get_annotations is None:
+        return fallback
+    try:
+        return inspect.get_annotations(fn, eval_str=True, include_extras=True)
+    except TypeError:
+        try:
+            return inspect.get_annotations(fn, eval_str=True)
+        except TypeError:
+            try:
+                return inspect.get_annotations(fn)
+            except Exception:
+                return fallback
+    except Exception:
+        return fallback
+
+
+def _build_parameters_schema(signature: inspect.Signature, annotations: Mapping[str, Any]) -> Dict[str, Any]:
+    properties: Dict[str, Any] = {}
+    required: List[str] = []
+
+    for param in signature.parameters.values():
+        if param.name.startswith("_"):
+            continue
+        if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
+            continue
+
+        annotation = annotations.get(param.name, inspect._empty)
+        annotation, meta = _unwrap_annotation(annotation)
+        annotation, optional_from_type = _strip_optional(annotation)
+        schema = _annotation_to_schema(annotation)
+        schema = _apply_param_meta(schema, meta)
+
+        if param.default is not inspect._empty:
+            schema.setdefault("default", param.default)
+
+        properties[param.name] = schema
+        is_required = param.default is inspect._empty and not optional_from_type
+        if is_required:
+            required.append(param.name)
+
+    payload: Dict[str, Any] = {
+        "type": "object",
+        "properties": properties,
+    }
+    if required:
+        payload["required"] = required
+    return payload
+
+
+def _unwrap_annotation(annotation: Any) -> Tuple[Any, ParamMeta | None]:
+    origin = get_origin(annotation)
+    if origin is Annotated:
+        args = get_args(annotation)
+        if not args:
+            return annotation, None
+        base = args[0]
+        meta = next((arg for arg in args[1:] if isinstance(arg, ParamMeta)), None)
+        return base, meta
+    return annotation, None
+
+
+def _strip_optional(annotation: Any) -> Tuple[Any, bool]:
+    origin = get_origin(annotation)
+    if origin is Union:
+        args = [arg for arg in get_args(annotation) if arg is not type(None)]  # noqa: E721
+        if len(args) == 1 and len(args) != len(get_args(annotation)):
+            return args[0], True
+    return annotation, False
+
+
+def _annotation_to_schema(annotation: Any) -> Dict[str, Any]:
+    if annotation is inspect._empty or annotation is Any:
+        return {"type": "string"}
+
+    origin = get_origin(annotation)
+    if origin is None:
+        return _primitive_schema(annotation)
+
+    if origin is list or origin is List or origin is abc.Sequence or origin is abc.MutableSequence:
+        item_annotation = get_args(annotation)[0] if get_args(annotation) else Any
+        return {
+            "type": "array",
+            "items": _annotation_to_schema(item_annotation),
+        }
+
+    if origin in {dict, Dict, abc.Mapping, abc.MutableMapping}:
+        return {"type": "object"}
+
+    if origin is Union:
+        literals = [arg for arg in get_args(annotation) if arg is not type(None)]  # noqa: E721
+        literal_schema = _try_literal_schema(literals)
+        if literal_schema:
+            return literal_schema
+        return {"type": "string"}
+
+    if origin is Literal:
+        values = list(get_args(annotation))
+        return _literal_schema(values)
+
+    return {"type": "string"}
+
+
+def _primitive_schema(annotation: Any) -> Dict[str, Any]:
+    if isinstance(annotation, type) and issubclass(annotation, Enum):
+        values = [member.value for member in annotation]
+        schema = _literal_schema(values)
+        return schema if schema else {"type": "string"}
+
+    if annotation in {str}:
+        return {"type": "string"}
+    if annotation in {int}:
+        return {"type": "integer"}
+    if annotation in {float}:
+        return {"type": "number"}
+    if annotation in {bool}:
+        return {"type": "boolean"}
+    if annotation in {dict, abc.Mapping}:
+        return {"type": "object"}
+    if annotation in {list, abc.Sequence}:
+        return {"type": "array", "items": {"type": "string"}}
+
+    return {"type": "string"}
+
+
+def _apply_param_meta(schema: Dict[str, Any], meta: ParamMeta | None) -> Dict[str, Any]:
+    if meta is None:
+        return schema
+    updated = dict(schema)
+    if meta.description:
+        updated["description"] = meta.description
+    if meta.enum:
+        updated["enum"] = list(meta.enum)
+        inferred = _infer_literal_type(meta.enum)
+        if inferred:
+            updated["type"] = inferred
+    return updated
+
+
+def _literal_schema(values: Sequence[Any]) -> Dict[str, Any]:
+    if not values:
+        return {"type": "string"}
+    schema: Dict[str, Any] = {"enum": list(values)}
+    literal_type = _infer_literal_type(values)
+    if literal_type:
+        schema["type"] = literal_type
+    return schema
+
+
+def _try_literal_schema(values: Sequence[Any]) -> Dict[str, Any] | None:
+    if not values:
+        return None
+    literal_type = _infer_literal_type(values)
+    if literal_type is None:
+        return None
+    return {"type": literal_type, "enum": list(values)}
+
+
+def _infer_literal_type(values: Sequence[Any]) -> str | None:
+    if all(isinstance(value, bool) for value in values):
+        return "boolean"
+    if all(isinstance(value, int) and not isinstance(value, bool) for value in values):
+        return "integer"
+    if all(isinstance(value, float) for value in values):
+        return "number"
+    if all(isinstance(value, str) for value in values):
+        return "string"
+    return None
+
+
+__all__ = [
+    "FunctionCatalog",
+    "FunctionMetadata",
+    "ParamMeta",
+    "get_function_catalog",
+]

utils/function_manager.py

@@ -0,0 +1,134 @@
+"""Unified function management."""
+import importlib.util
+import inspect
+import os
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional
+
+_MODULE_PREFIX = "_dynamic_functions"
+_FUNCTION_CALLING_ENV = "MAC_FUNCTIONS_DIR"
+_EDGE_FUNCTION_ENV = "MAC_EDGE_FUNCTIONS_DIR"
+_EDGE_PROCESSOR_FUNCTION_ENV = "MAC_EDGE_PROCESSOR_FUNCTIONS_DIR"
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+_DEFAULT_FUNCTIONS_ROOT = Path("functions")
+_DEFAULT_FUNCTION_CALLING_DIR = _DEFAULT_FUNCTIONS_ROOT / "function_calling"
+_DEFAULT_EDGE_FUNCTION_DIR = _DEFAULT_FUNCTIONS_ROOT / "edge"
+_DEFAULT_EDGE_PROCESSOR_DIR = _DEFAULT_FUNCTIONS_ROOT / "edge_processor"
+
+
+def _resolve_dir(default: Path, env_var: str | None = None) -> Path:
+    """Resolve a directory path with optional environment override."""
+    override = os.environ.get(env_var) if env_var else None
+    if override:
+        return Path(override).expanduser()
+    if default.is_absolute():
+        return default
+    return _REPO_ROOT / default
+
+
+FUNCTION_CALLING_DIR = _resolve_dir(_DEFAULT_FUNCTION_CALLING_DIR, _FUNCTION_CALLING_ENV).resolve()
+EDGE_FUNCTION_DIR = _resolve_dir(_DEFAULT_EDGE_FUNCTION_DIR, _EDGE_FUNCTION_ENV).resolve()
+EDGE_PROCESSOR_FUNCTION_DIR = _resolve_dir(_DEFAULT_EDGE_PROCESSOR_DIR, _EDGE_PROCESSOR_FUNCTION_ENV).resolve()
+
+
+class FunctionManager:
+    """Unified function manager for loading and managing functions across the project."""
+
+    def __init__(self, functions_dir: str | Path = "functions") -> None:
+        self.functions_dir = Path(functions_dir)
+        self.functions: Dict[str, Callable] = {}
+        self._loaded = False
+
+    def load_functions(self) -> None:
+        """Load all Python functions from functions directory."""
+        if self._loaded:
+            return
+            
+        if not self.functions_dir.exists():
+            raise ValueError(f"Functions directory does not exist: {self.functions_dir}")
+
+        for file in self.functions_dir.rglob("*.py"):
+            if file.name.startswith("_") or file.name == "__init__.py":
+                continue
+            if "__pycache__" in file.parts:
+                continue
+                
+            module_name = self._build_module_name(file)
+            try:
+                # Import module dynamically
+                spec = importlib.util.spec_from_file_location(module_name, file)
+                if spec is None or spec.loader is None:
+                    continue
+                    
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)
+
+                current_file = file.resolve()
+                # Get all functions defined in the module
+                for name, obj in inspect.getmembers(module, inspect.isfunction):
+                    if name.startswith("_"):
+                        continue
+                    # Only register functions defined in the current module/file
+                    if getattr(obj, "__module__", None) != module.__name__:
+                        code = getattr(obj, "__code__", None)
+                        source_path = Path(code.co_filename).resolve() if code else None
+                        if source_path != current_file:
+                            continue
+                    self.functions[name] = obj
+            except Exception as e:
+                print(f"Error loading module {module_name}: {e}")
+        
+        self._loaded = True
+
+    def _build_module_name(self, filepath: Path) -> str:
+        """Create a unique module name for a function file."""
+        relative = filepath.relative_to(self.functions_dir)
+        parts = "_".join(relative.with_suffix("").parts) or "module"
+        unique_suffix = f"{abs(hash(filepath.as_posix())) & 0xFFFFFFFF:X}"
+        return f"{_MODULE_PREFIX}.{parts}_{unique_suffix}"
+
+    def get_function(self, name: str) -> Optional[Callable]:
+        """Get a function by name."""
+        if not self._loaded:
+            self.load_functions()
+        return self.functions.get(name)
+
+    def has_function(self, name: str) -> bool:
+        """Check if a function exists."""
+        if not self._loaded:
+            self.load_functions()
+        return name in self.functions
+
+    def call_function(self, name: str, *args, **kwargs) -> Any:
+        """Call a function by name with given arguments."""
+        func = self.get_function(name)
+        if func is None:
+            raise ValueError(f"Function {name} not found")
+        return func(*args, **kwargs)
+
+    def list_functions(self) -> Dict[str, Callable]:
+        """List all available functions."""
+        if not self._loaded:
+            self.load_functions()
+        return self.functions.copy()
+
+    def reload_functions(self) -> None:
+        """Reload all functions from the functions directory."""
+        self.functions.clear()
+        self._loaded = False
+        self.load_functions()
+
+
+# Global function manager registry keyed by directory
+_function_managers: Dict[Path, FunctionManager] = {}
+
+
+def get_function_manager(functions_dir: str | Path) -> FunctionManager:
+    """Get or create the global function manager instance for a directory."""
+    directory = Path(functions_dir).resolve()
+
+    manager = _function_managers.get(directory)
+    if manager is None:
+        manager = FunctionManager(directory)
+        _function_managers[directory] = manager
+    return manager

utils/human_prompt.py

@@ -0,0 +1,164 @@
+"""Human-in-the-loop prompt service with pluggable channels."""
+
+import threading
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Protocol
+
+from entity.messages import MessageBlock, MessageBlockType, MessageContent
+from utils.log_manager import LogManager
+
+
+@dataclass
+class PromptResult:
+    """Typed result returned from prompt channels."""
+
+    text: str
+    blocks: Optional[List[MessageBlock]] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def as_message_content(self) -> MessageContent:
+        return self.blocks if self.blocks is not None else self.text
+
+
+class PromptChannel(Protocol):
+    """Channel interface that performs the actual user interaction."""
+
+    def request(
+        self,
+        *,
+        node_id: str,
+        task: str,
+        inputs: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> PromptResult:
+        """Collect user feedback and return the structured response."""
+
+
+@dataclass
+class CliPromptChannel:
+    """Default channel that prompts the operator via CLI input()."""
+
+    input_func: Callable[[str], str] = input
+
+    def request(
+        self,
+        *,
+        node_id: str,
+        task: str,
+        inputs: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> PromptResult:
+        header = ["===== HUMAN INPUT REQUIRED ====="]
+        if inputs:
+            header.append("=== Node inputs ===")
+            header.append(inputs)
+        header.append(f"=== Task for human ({node_id}) ===")
+        header.append(task)
+        header.append("=== Your response: ===")
+        prompt = "\n".join(header) + "\n"
+        response = self.input_func(prompt)
+        return PromptResult(
+            text=response,
+            blocks=[MessageBlock.text_block(response or "")],
+        )
+
+
+class HumanPromptService:
+    """Coordinates human feedback collection across nodes and tools."""
+
+    def __init__(
+        self,
+        *,
+        log_manager: LogManager,
+        channel: PromptChannel,
+        session_id: Optional[str] = None,
+    ) -> None:
+        self._log_manager = log_manager
+        self._channel = channel
+        self._session_id = session_id
+        self._lock = threading.Lock()
+
+    def request(
+        self,
+        node_id: str,
+        task_description: str,
+        *,
+        inputs: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> PromptResult:
+        """Request human input through the configured channel."""
+
+        meta = dict(metadata or {})
+        if self._session_id and "session_id" not in meta:
+            meta["session_id"] = self._session_id
+
+        with self._lock:
+            with self._log_manager.human_timer(node_id):
+                raw_result = self._channel.request(
+                    node_id=node_id,
+                    task=task_description,
+                    inputs=inputs,
+                    metadata=meta,
+                )
+
+        prompt_result = self._normalize_result(raw_result)
+        sanitized_text = self._sanitize_response(prompt_result.text)
+        normalized_blocks = self._normalize_blocks(prompt_result.blocks, sanitized_text)
+        combined_metadata = {**prompt_result.metadata, **meta}
+
+        self._log_manager.record_human_interaction(
+            node_id,
+            inputs,
+            sanitized_text,
+            details={"task_description": task_description, **combined_metadata},
+        )
+        return PromptResult(
+            text=sanitized_text,
+            blocks=normalized_blocks,
+            metadata=combined_metadata,
+        )
+
+    @staticmethod
+    def _sanitize_response(response: Any) -> str:
+        text = response if isinstance(response, str) else str(response)
+        return text.encode("utf-8", errors="ignore").decode("utf-8", errors="ignore")
+
+    def _normalize_result(self, raw_result: PromptResult | str | Any) -> PromptResult:
+        if isinstance(raw_result, PromptResult):
+            return raw_result
+        text = self._sanitize_response(raw_result)
+        return PromptResult(text=text, blocks=[MessageBlock.text_block(text)])
+
+    def _normalize_blocks(
+        self,
+        blocks: Optional[List[MessageBlock]],
+        fallback_text: str,
+    ) -> List[MessageBlock]:
+        if not blocks:
+            return [MessageBlock.text_block(fallback_text)]
+        normalized: List[MessageBlock] = []
+        for block in blocks:
+            dup = block.copy()
+            if dup.type is MessageBlockType.TEXT and dup.text is not None:
+                dup.text = self._sanitize_response(dup.text)
+            normalized.append(dup)
+        return normalized
+
+
+def resolve_prompt_channel(workspace_hook: Any) -> PromptChannel | None:
+    """Helper to fetch a PromptChannel from a workspace hook if available."""
+
+    if workspace_hook is None:
+        return None
+
+    getter = getattr(workspace_hook, "get_prompt_channel", None)
+    if callable(getter):
+        channel = getter()
+        if channel is not None:
+            return channel
+
+    channel = getattr(workspace_hook, "prompt_channel", None)
+    if channel is not None:
+        return channel
+
+    return None

utils/io_utils.py

@@ -0,0 +1,6 @@
+from typing import Any, Dict
+import yaml
+
+def read_yaml(path) -> Dict[str, Any]:
+    with open(path, mode="r", encoding="utf-8") as f:
+        return yaml.load(f, Loader=yaml.FullLoader)

utils/log_manager.py

@@ -0,0 +1,215 @@
+"""Log manager compatibility shim.
+
+LogManager now wraps WorkflowLogger for backward compatibility.
+All timing helpers live inside WorkflowLogger; prefer using it directly.
+"""
+
+import time
+from contextlib import contextmanager
+from typing import Any, Dict, List
+
+from entity.enums import CallStage, LogLevel
+from utils.logger import WorkflowLogger
+
+
+class LogManager:
+    """Backward-compatible wrapper that delegates to ``WorkflowLogger``."""
+
+    def __init__(self, logger: WorkflowLogger = None):
+        self.logger = logger
+
+    def get_logger(self) -> WorkflowLogger:
+        """Return the underlying ``WorkflowLogger`` instance."""
+        return self.logger
+
+    # ================================================================
+    # Timer context managers delegated to WorkflowLogger
+    # ================================================================
+
+    @contextmanager
+    def node_timer(self, node_id: str):
+        """Context manager that times node execution."""
+        with self.logger.node_timer(node_id):
+            yield
+
+    @contextmanager
+    def model_timer(self, node_id: str):
+        """Context manager that times model invocations."""
+        with self.logger.model_timer(node_id):
+            yield
+
+    @contextmanager
+    def agent_timer(self, node_id: str):
+        """Context manager that times agent invocations."""
+        with self.logger.agent_timer(node_id):
+            yield
+
+    @contextmanager
+    def human_timer(self, node_id: str):
+        """Context manager that times human interactions."""
+        with self.logger.human_timer(node_id):
+            yield
+
+    @contextmanager
+    def tool_timer(self, node_id: str, tool_name: str):
+        """Context manager that times tool invocations."""
+        with self.logger.tool_timer(node_id, tool_name):
+            yield
+
+    @contextmanager
+    def thinking_timer(self, node_id: str, stage: str):
+        """Context manager that times thinking workflows."""
+        with self.logger.thinking_timer(node_id, stage):
+            yield
+
+    @contextmanager
+    def memory_timer(self, node_id: str, operation_type: str, stage: str):
+        """Context manager that times memory operations."""
+        with self.logger.memory_timer(node_id, operation_type, stage):
+            yield
+
+    @contextmanager
+    def operation_timer(self, operation_name: str):
+        """Context manager that times custom operations."""
+        start_time = time.time()
+        try:
+            yield
+        finally:
+            end_time = time.time()
+            duration = (end_time - start_time)
+            self.logger._timers[operation_name] = duration
+
+    # ================================================================
+    # Logging methods delegated to WorkflowLogger
+    # ================================================================
+
+    def record_node_start(self, node_id: str, inputs: List[Dict[str, str]], node_type: str = None,
+                          details: Dict[str, Any] = None) -> None:
+        """Record the start of a node."""
+        self.logger.enter_node(node_id, inputs, node_type, details)
+
+    def record_node_end(self, node_id: str, output: str = None,
+                        details: Dict[str, Any] = None) -> None:
+        """Record the end of a node."""
+        output_size = len(str(output)) if output is not None else 0
+        duration = self.logger.get_timer(node_id)
+        self.logger.exit_node(node_id, output, duration, output_size, details)
+
+    def record_edge_process(self, from_node: str, to_node: str,
+                            details: Dict[str, Any] = None) -> None:
+        """Record an edge processing event."""
+        self.logger.record_edge_process(from_node, to_node, details)
+
+    def record_human_interaction(self, node_id: str, input_data: Any = None, output: Any = None,
+                                 details: Dict[str, Any] = None) -> None:
+        """Record a human interaction."""
+        input_size = len(str(input_data)) if input_data is not None else 0
+        output_size = len(str(output)) if output is not None else 0
+        duration = self.logger.get_timer(f"human_{node_id}")
+        call_details = {
+            "input_size": input_size,
+            "output_size": output_size,
+            **(details or {})
+        }
+        self.logger.record_human_interaction(
+            node_id, input_data, output, duration, call_details
+        )
+
+    def record_model_call(self, node_id: str, model_name: str,
+                          input_data: Any = None, output: Any = None,
+                          details: Dict[str, Any] = None,
+                          stage: CallStage = CallStage.AFTER) -> None:
+        """Record a model invocation."""
+        input_size = len(str(input_data)) if input_data is not None else 0
+        output_size = len(str(output)) if output is not None else 0
+        duration = self.logger.get_timer(f"model_{node_id}")
+
+        call_details = {
+            "input_size": input_size,
+            "output_size": output_size,
+            **(details or {})
+        }
+
+        self.logger.record_model_call(
+            node_id, model_name, input_data, output, duration, call_details, stage
+        )
+
+    def record_tool_call(self, node_id: str, tool_name: str,
+                         success: bool | None = True, tool_result: Any = None,
+                         details: Dict[str, Any] = None,
+                         stage: CallStage = CallStage.AFTER) -> None:
+        """Record a tool invocation."""
+        duration = self.logger.get_timer(f"tool_{node_id}_{tool_name}")
+        tool_details = {
+            "result_size": len(str(tool_result)) if tool_result is not None else 0,
+            **(details or {})
+        }
+        self.logger.record_tool_call(node_id, tool_name, tool_result, duration, success, tool_details, stage)
+
+    def record_thinking_process(self, node_id: str, thinking_mode: str, thinking_result: str,
+                                stage: str, details: Dict[str, Any] = None) -> None:
+        """Record a thinking stage."""
+        duration = self.logger.get_timer(f"thinking_{node_id}_{stage}")
+        self.logger.record_thinking_process(node_id, thinking_mode, thinking_result, stage, duration, details)
+
+    def record_memory_operation(self, node_id: str, operation_type: str,
+                                stage: str, retrieved_memory: Any = None,
+                                details: Dict[str, Any] = None) -> None:
+        """Record a memory operation."""
+        duration = self.logger.get_timer(f"memory_{node_id}_{operation_type}_{stage}")
+        memory_details = {
+            "result_size": len(str(retrieved_memory)) if retrieved_memory is not None else 0,
+            **(details or {})
+        }
+        self.logger.record_memory_operation(node_id, retrieved_memory, operation_type, stage, duration, memory_details)
+
+    def record_workflow_start(self, workflow_config: Dict[str, Any] = None) -> None:
+        """Record the workflow start event."""
+        self.logger.record_workflow_start(workflow_config)
+
+    def record_workflow_end(self, success: bool = True,
+                            details: Dict[str, Any] = None) -> None:
+        """Record the workflow end event."""
+        workflow_duration = (time.time() - self.logger.start_time.timestamp())
+        self.logger.record_workflow_end(success, workflow_duration, details)
+
+    def debug(self, message: str, node_id: str = None,
+              details: Dict[str, Any] = None) -> None:
+        """Record debug information."""
+        self.logger.debug(message, node_id, details=details)
+
+    def info(self, message: str, node_id: str = None,
+             details: Dict[str, Any] = None) -> None:
+        """Record general information."""
+        self.logger.info(message, node_id, details=details)
+
+    def warning(self, message: str, node_id: str = None,
+                details: Dict[str, Any] = None) -> None:
+        """Record warning information."""
+        self.logger.warning(message, node_id, details=details)
+
+    def error(self, message: str, node_id: str = None,
+              details: Dict[str, Any] = None) -> None:
+        """Record error information."""
+        self.logger.error(message, node_id, details=details)
+
+    def critical(self, message: str, node_id: str = None,
+                 details: Dict[str, Any] = None) -> None:
+        """Record critical error information."""
+        self.logger.critical(message, node_id, details=details)
+
+    def get_execution_summary(self) -> Dict[str, Any]:
+        """Return the execution summary."""
+        return self.logger.get_execution_summary()
+
+    def get_all_logs(self) -> list:
+        """Return all logs."""
+        return self.logger.get_logs()
+
+    def logs_to_dict(self) -> Dict[str, Any]:
+        """Convert the logs to dictionary form."""
+        return self.logger.to_dict()
+
+    def save_logs(self, filepath: str) -> None:
+        """Persist logs to a file."""
+        self.logger.save_to_file(filepath)

utils/logger.py

@@ -0,0 +1,493 @@
+import os
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+import json
+import copy
+import traceback
+
+from entity.enums import CallStage, EventType, LogLevel
+from utils.structured_logger import StructuredLogger, LogType, get_workflow_logger
+from utils.exceptions import MACException
+
+
+def _json_safe(value: Any) -> Any:
+    """Recursively convert objects into JSON-encodable primitives."""
+    if value is None or isinstance(value, (str, int, float, bool)):
+        return value
+    if isinstance(value, dict):
+        return {str(key): _json_safe(val) for key, val in value.items()}
+    if isinstance(value, (list, tuple, set)):
+        return [_json_safe(item) for item in value]
+    to_dict = getattr(value, "to_dict", None)
+    if callable(to_dict):
+        try:
+            return _json_safe(to_dict())
+        except Exception:
+            pass
+    if hasattr(value, "__dict__"):
+        try:
+            return _json_safe(vars(value))
+        except Exception:
+            pass
+    return str(value)
+
+
+@dataclass
+class LogEntry:
+    """Single log entry that captures execution details."""
+    timestamp: str
+    level: LogLevel
+    node_id: Optional[str] = None
+    event_type: Optional[EventType] = None
+    message: Optional[str] = None
+    details: Dict[str, Any] = field(default_factory=dict)
+    execution_path: List[str] = field(default_factory=list)  # Execution path for tracing
+    duration: Optional[float] = None  # Duration in seconds
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "timestamp": self.timestamp,
+            "level": self.level,
+            "node_id": self.node_id,
+            "event_type": self.event_type,
+            "message": self.message,
+            "details": self.details,
+            "execution_path": self.execution_path,
+            "duration": self.duration
+        }
+
+
+class WorkflowLogger:
+    """Workflow logger that tracks the entire execution lifecycle."""
+
+    def __init__(self, workflow_id: str = None, log_level: LogLevel = LogLevel.DEBUG, use_structured_logging: bool = True, log_to_console: bool = True):
+        self.workflow_id = workflow_id or f"workflow_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        self.logs: List[LogEntry] = []
+        self.start_time = datetime.now()
+        self.current_path: List[str] = []
+        self.log_level: LogLevel = log_level
+
+        self.log_to_console: bool = log_to_console
+        self.use_structured_logging = use_structured_logging
+        self.structured_logger: Optional[StructuredLogger] = None
+        if use_structured_logging:
+            self.structured_logger = get_workflow_logger(self.workflow_id)
+
+    def add_log(self, level: LogLevel, message: str = None, node_id: str = None,
+                event_type: EventType = None, details: Dict[str, Any] = None,
+                duration: float = None) -> LogEntry | None:
+        """Add a log entry."""
+        if level < self.log_level:
+            return None
+
+        timestamp = datetime.now().isoformat()
+        execution_path = copy.deepcopy(self.current_path)
+
+        safe_details = _json_safe(details or {})
+
+        log_entry = LogEntry(
+            timestamp=timestamp,
+            level=level,
+            node_id=node_id,
+            event_type=event_type,
+            message=message,
+            details=safe_details,
+            execution_path=execution_path,
+            duration=duration
+        )
+        self.logs.append(log_entry)
+
+        # Log to console if enabled
+        if self.log_to_console:
+            print(f"[{timestamp}] [{level.value}] "
+                  f"{f'Node {node_id} - ' if node_id else ''}"
+                  f"{f'Event {event_type} - ' if event_type else ''}"
+                  f"{message} "
+                  f"{f'Details: {details} ' if details else ''}"
+                  f"{f'Duration: {duration}' if duration else ''}")
+        
+        # Log using structured logger if enabled
+        if self.use_structured_logging and self.structured_logger:
+            structured_details = {
+                "workflow_id": self.workflow_id,
+                "node_id": node_id,
+                "event_type": event_type.value if event_type else None,
+                "execution_path": execution_path,
+                "duration": duration,
+                **safe_details
+            }
+            
+            if level == LogLevel.DEBUG:
+                self.structured_logger.debug(message, **structured_details)
+            elif level == LogLevel.INFO:
+                self.structured_logger.info(message, **structured_details)
+            elif level == LogLevel.WARNING:
+                self.structured_logger.warning(message, **structured_details)
+            elif level == LogLevel.ERROR:
+                self.structured_logger.error(message, **structured_details)
+            elif level == LogLevel.CRITICAL:
+                self.structured_logger.critical(message, **structured_details)
+
+        return log_entry
+
+    def debug(self, message: str, node_id: str = None, event_type: EventType = None,
+              details: Dict[str, Any] = None, duration: float | None = None) -> None:
+        self.add_log(LogLevel.DEBUG, message, node_id, event_type, details, duration)
+
+    def info(self, message: str, node_id: str = None, event_type: EventType = None,
+             details: Dict[str, Any] = None, duration: float | None = None) -> None:
+        self.add_log(LogLevel.INFO, message, node_id, event_type, details, duration)
+
+    def warning(self, message: str, node_id: str = None, event_type: EventType = None,
+                details: Dict[str, Any] = None, duration: float | None = None) -> None:
+        self.add_log(LogLevel.WARNING, message, node_id, event_type, details, duration)
+
+    def error(self, message: str, node_id: str = None, event_type: EventType = None,
+              details: Dict[str, Any] = None, duration: float | None = None) -> None:
+        self.add_log(LogLevel.ERROR, message, node_id, event_type, details, duration)
+
+    def critical(self, message: str, node_id: str = None, event_type: EventType = None,
+                 details: Dict[str, Any] = None) -> None:
+        self.add_log(LogLevel.CRITICAL, message, node_id, event_type, details)
+
+    def enter_node(self, node_id: str, inputs: List[Dict[str, str]], node_type: str = None,
+                   details: Dict[str, Any] = None) -> None:
+        """Record data when entering a node."""
+        self.current_path.append(node_id)
+        self.info(
+            f"Entering node {node_id}",
+            node_id=node_id,
+            event_type=EventType.NODE_START,
+            details={
+                "inputs": inputs,
+                # "combined_input": combined_input,
+                "node_type": node_type,
+                **(details or {})
+            }
+        )
+
+    def exit_node(self, node_id: str, output: str, duration: float = None,
+                  output_size: int = None, details: Dict[str, Any] = None) -> None:
+        """Record data when exiting a node."""
+        # Keep enter and exit logs separate so we can easily identify progress
+        if self.current_path and self.current_path[-1] == node_id:
+            self.current_path.pop()
+
+        exit_details = {
+            "output": output,
+            "output_size": output_size,
+            **(details or {})
+        }
+
+        self.info(
+            f"Exiting node {node_id}",
+            node_id=node_id,
+            event_type=EventType.NODE_END,
+            details=exit_details,
+            duration=duration
+        )
+
+    def record_edge_process(self, from_node: str, to_node: str,
+                            details: Dict[str, Any] = None) -> None:
+        """Record an edge-processing event."""
+        self.debug(
+            f"Processing edge from {from_node} to {to_node}",
+            node_id=from_node,
+            event_type=EventType.EDGE_PROCESS,
+            details={
+                "to_node": to_node,
+                **(details or {})
+            }
+        )
+
+    def record_human_interaction(self, node_id: str, input_data: str = None, output: str = None,
+                                 duration: float = None, details: Dict[str, Any] = None) -> None:
+        """Record a human interaction."""
+        call_details = {
+            "input_data": input_data,
+            "output": output,
+            **(details or {})
+        }
+
+        self.info(
+            f"Human interaction for node {node_id}",
+            node_id=node_id,
+            event_type=EventType.HUMAN_INTERACTION,
+            details=call_details,
+            duration=duration
+        )
+
+    def record_model_call(self, node_id: str, model_name: str,
+                          input_data: str = None, output: str = None,
+                          duration: float = None, details: Dict[str, Any] = None,
+                          stage: CallStage | str | None = None) -> None:
+        """Record a model invocation."""
+        stage_value = stage.value if isinstance(stage, CallStage) else stage
+        call_details = {
+            "model_name": model_name,
+            "input_data": input_data,
+            "output": output,
+            **(details or {})
+        }
+        if stage_value:
+            call_details["stage"] = stage_value
+
+        self.info(
+            f"Model call for node {node_id}",
+            node_id=node_id,
+            event_type=EventType.MODEL_CALL,
+            details=call_details,
+            duration=duration
+        )
+
+    def record_tool_call(self, node_id: str, tool_name: str, tool_result: str,
+                         duration: float = None, success: bool | None = True,
+                         details: Dict[str, Any] = None,
+                         stage: CallStage | str | None = None) -> None:
+        """Record a tool invocation."""
+        stage_value = stage.value if isinstance(stage, CallStage) else stage
+        tool_details = {
+            "tool_result": tool_result,
+            "tool_name": tool_name,
+            "success": success,
+            **(details or {})
+        }
+        if stage_value:
+            tool_details["stage"] = stage_value
+
+        level = LogLevel.INFO if success is not False else LogLevel.ERROR
+        self.add_log(
+            level,
+            f"Tool call {tool_name} for node {node_id}",
+            node_id=node_id,
+            event_type=EventType.TOOL_CALL,
+            details=tool_details,
+            duration=duration
+        )
+
+    def record_thinking_process(self, node_id: str, thinking_mode: str, thinking_result: str, stage: str,
+                                duration: float = None, details: Dict[str, Any] = None) -> None:
+        """Record a thinking-stage entry."""
+        thinking_details = {
+            "thinking_result": thinking_result,
+            "thinking_mode": thinking_mode,
+            "stage": stage,
+            **(details or {})
+        }
+
+        self.info(
+            f"Thinking process for node {node_id} ({thinking_mode} at {stage})",
+            node_id=node_id,
+            event_type=EventType.THINKING_PROCESS,
+            details=thinking_details,
+            duration=duration
+        )
+
+    def record_memory_operation(self, node_id: str, retrieved_memory: str, operation_type: str, stage: str,
+                                duration: float = None, details: Dict[str, Any] = None) -> None:
+        """Record a memory operation (retrieve/update)."""
+        memory_details = {
+            "retrieved_memory": retrieved_memory,
+            "operation_type": operation_type,  # RETRIEVE or UPDATE
+            "stage": stage,
+            **(details or {})
+        }
+
+        self.info(
+            f"Memory {operation_type} operation for node {node_id} at {stage}",
+            node_id=node_id,
+            event_type=EventType.MEMORY_OPERATION,
+            details=memory_details,
+            duration=duration
+        )
+
+    def record_workflow_start(self, workflow_config: Dict[str, Any] = None) -> None:
+        """Record the workflow start event."""
+        self.info(
+            "Workflow execution started",
+            event_type=EventType.WORKFLOW_START,
+            details={
+                "workflow_id": self.workflow_id,
+                "node_count": workflow_config.get("node_count") if workflow_config else None,
+                "edge_count": workflow_config.get("edge_count") if workflow_config else None,
+            }
+        )
+
+    def record_workflow_end(self, success: bool = True,
+                            duration: float = None, details: Dict[str, Any] = None) -> None:
+        """Record the workflow end event."""
+        end_details = {
+            "success": success,
+            "total_logs": len(self.logs),
+            **(details or {})
+        }
+
+        level = LogLevel.INFO if success else LogLevel.ERROR
+        self.add_log(
+            level,
+            "Workflow execution completed",
+            event_type=EventType.WORKFLOW_END,
+            details=end_details,
+            duration=duration
+        )
+
+    def get_logs(self) -> List[Dict[str, Any]]:
+        """Return all log entries as dictionaries."""
+        return [log.to_dict() for log in self.logs]
+
+    def get_logs_by_level(self, level: str) -> List[Dict[str, Any]]:
+        """Return logs filtered by level."""
+        return [log.to_dict() for log in self.logs if log.level == level]
+
+    def get_logs_by_node(self, node_id: str) -> List[Dict[str, Any]]:
+        """Return logs filtered by node id."""
+        return [log.to_dict() for log in self.logs if log.node_id == node_id]
+
+    def get_execution_summary(self) -> Dict[str, Any]:
+        """Return an execution summary."""
+        total_duration = (datetime.now() - self.start_time).total_seconds() * 1000
+
+        node_durations = {}
+        for log in self.logs:
+            if log.node_id and log.duration:
+                if log.node_id not in node_durations:
+                    node_durations[log.node_id] = 0
+                node_durations[log.node_id] += log.duration
+
+        error_count = len([log for log in self.logs if log.level in ["ERROR", "CRITICAL"]])
+        warning_count = len([log for log in self.logs if log.level == "WARNING"])
+
+        return {
+            "workflow_id": self.workflow_id,
+            "start_time": self.start_time.isoformat(),
+            "total_duration": total_duration,
+            "total_logs": len(self.logs),
+            "error_count": error_count,
+            "warning_count": warning_count,
+            "node_durations": node_durations,
+            "execution_path": self.current_path
+        }
+
+    def to_dict(self) -> Dict[str, Any]:
+        log_data = {
+            "workflow_id": self.workflow_id,
+            "start_time": self.start_time.isoformat(),
+            "logs": self.get_logs(),
+            "summary": self.get_execution_summary()
+        }
+        return log_data
+
+    def to_json(self) -> str:
+        """Serialize all logs to a JSON string."""
+        return json.dumps(self.to_dict(), ensure_ascii=False, indent=2)
+
+    def save_to_file(self, filepath: str) -> None:
+        """Persist logs to a file on disk."""
+        # with open(filepath, 'w', encoding='utf-8') as f:
+        #     f.write(self.to_json())
+        path = Path(filepath)
+        path.parent.mkdir(parents=True, exist_ok=True)  # Create any missing parent directories
+        path.write_text(self.to_json(), encoding='utf-8')
+    
+    # ================================================================
+    # Timer Context Managers (integrated from LogManager)
+    # ================================================================
+    
+    def __init_timers__(self):
+        """Initialize timer storage if not exists."""
+        if not hasattr(self, '_timers'):
+            self._timers: Dict[str, float] = {}
+    
+    @contextmanager
+    def node_timer(self, node_id: str):
+        """Context manager that times node execution."""
+        self.__init_timers__()
+        start_time = time.time()
+        try:
+            yield
+        finally:
+            end_time = time.time()
+            duration = (end_time - start_time)
+            self._timers[node_id] = duration
+    
+    @contextmanager
+    def model_timer(self, node_id: str):
+        """Context manager that times model invocations."""
+        self.__init_timers__()
+        start_time = time.time()
+        try:
+            yield
+        finally:
+            end_time = time.time()
+            duration = (end_time - start_time)
+            self._timers[f"model_{node_id}"] = duration
+    
+    @contextmanager
+    def agent_timer(self, node_id: str):
+        """Context manager that times agent invocations."""
+        self.__init_timers__()
+        start_time = time.time()
+        try:
+            yield
+        finally:
+            end_time = time.time()
+            duration = (end_time - start_time)
+            self._timers[f"agent_{node_id}"] = duration
+    
+    @contextmanager
+    def human_timer(self, node_id: str):
+        """Context manager that times human interactions."""
+        self.__init_timers__()
+        start_time = time.time()
+        try:
+            yield
+        finally:
+            end_time = time.time()
+            duration = (end_time - start_time)
+            self._timers[f"human_{node_id}"] = duration
+    
+    @contextmanager
+    def tool_timer(self, node_id: str, tool_name: str):
+        """Context manager that times tool invocations."""
+        self.__init_timers__()
+        start_time = time.time()
+        try:
+            yield
+        finally:
+            end_time = time.time()
+            duration = (end_time - start_time)
+            self._timers[f"tool_{node_id}_{tool_name}"] = duration
+    
+    @contextmanager
+    def thinking_timer(self, node_id: str, stage: str):
+        """Context manager that times thinking stages."""
+        self.__init_timers__()
+        start_time = time.time()
+        try:
+            yield
+        finally:
+            end_time = time.time()
+            duration = (end_time - start_time)
+            self._timers[f"thinking_{node_id}_{stage}"] = duration
+    
+    @contextmanager
+    def memory_timer(self, node_id: str, operation_type: str, stage: str):
+        """Context manager that times memory operations."""
+        self.__init_timers__()
+        start_time = time.time()
+        try:
+            yield
+        finally:
+            end_time = time.time()
+            duration = (end_time - start_time)
+            self._timers[f"memory_{node_id}_{operation_type}_{stage}"] = duration
+    
+    def get_timer(self, timer_key: str) -> Optional[float]:
+        """Return the elapsed time recorded by the timer key."""
+        self.__init_timers__()
+        return self._timers.get(timer_key)

utils/middleware.py

@@ -0,0 +1,128 @@
+"""Custom middleware for the DevAll workflow system."""
+
+import uuid
+from typing import Callable, Awaitable
+from fastapi import Request, HTTPException, FastAPI
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+import time
+import re
+import os
+
+from utils.structured_logger import get_server_logger, LogType
+from utils.exceptions import SecurityError
+
+
+async def correlation_id_middleware(request: Request, call_next: Callable):
+    """Add correlation ID to requests for tracing."""
+    correlation_id = request.headers.get("X-Correlation-ID") or str(uuid.uuid4())
+    request.state.correlation_id = correlation_id
+    
+    start_time = time.time()
+    response = await call_next(request)
+    duration = time.time() - start_time
+    
+    # Log the request and response
+    logger = get_server_logger()
+    logger.log_request(
+        request.method,
+        str(request.url),
+        correlation_id=correlation_id,
+        path=request.url.path,
+        query_params=dict(request.query_params),
+        client_host=request.client.host if request.client else None,
+        user_agent=request.headers.get("user-agent")
+    )
+    
+    logger.log_response(
+        response.status_code,
+        duration,
+        correlation_id=correlation_id,
+        content_length=response.headers.get("content-length")
+    )
+    
+    # Add correlation ID to response headers
+    response.headers["X-Correlation-ID"] = correlation_id
+    
+    return response
+
+
+async def security_middleware(request: Request, call_next: Callable):
+    """Security middleware to validate requests."""
+    # Validate content type for JSON endpoints
+    if request.url.path.startswith("/api/") and request.method in ["POST", "PUT", "PATCH"]:
+        content_type = request.headers.get("content-type", "").lower()
+        if not content_type.startswith("application/json") and request.method != "GET":
+            # Skip validation for file uploads
+            if not content_type.startswith("multipart/form-data"):
+                raise HTTPException(
+                    status_code=400,
+                    detail="Content-Type must be application/json for API endpoints"
+                )
+    
+    # Validate file paths to prevent path traversal
+    # Check URL path for suspicious patterns
+    path = request.url.path
+    if ".." in path or "./" in path:
+        # Use a more thorough check
+        if re.search(r"(\.{2}[/\\])|([/\\]\.{2})", path):
+            logger = get_server_logger()
+            logger.log_security_event(
+                "PATH_TRAVERSAL_ATTEMPT",
+                f"Suspicious path detected: {path}",
+                correlation_id=getattr(request.state, 'correlation_id', str(uuid.uuid4()))
+            )
+            raise HTTPException(status_code=400, detail="Invalid path")
+    
+    response = await call_next(request)
+    return response
+
+
+async def rate_limit_middleware(request: Request, call_next: Callable):
+    """Rate limiting middleware (basic implementation)."""
+    # This is a simple rate limiting implementation
+    # In production, you would use Redis or other storage for tracking
+    # This is just a placeholder for now
+    response = await call_next(request)
+    return response
+
+
+def add_cors_middleware(app: FastAPI) -> None:
+    """Configure and attach CORS middleware."""
+    # Dev defaults; override via CORS_ALLOW_ORIGINS (comma-separated)
+    default_origins = [
+        "http://localhost:5173",
+        "http://127.0.0.1:5173",
+    ]
+    env_origins = os.getenv("CORS_ALLOW_ORIGINS")
+    if env_origins:
+        origins = [o.strip() for o in env_origins.split(",") if o.strip()]
+        origin_regex = None
+    else:
+        origins = default_origins
+        # Helpful in dev: allow localhost/127.0.0.1 on any port
+        origin_regex = r"^https?://(localhost|127\.0\.0\.1)(:\d+)?$"
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=origins,
+        allow_origin_regex=origin_regex,
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+        expose_headers=["X-Correlation-ID"],
+        max_age=600,
+    )
+
+
+def add_middleware(app: FastAPI):
+    """Add all middleware to the FastAPI application."""
+    # Attach CORS first to handle preflight requests and allow origins.
+    add_cors_middleware(app)
+
+    # Add other middleware
+    app.middleware("http")(correlation_id_middleware)
+    app.middleware("http")(security_middleware)
+    # app.middleware("http")(rate_limit_middleware)  # Enable if needed
+    
+    return app

utils/registry.py

@@ -0,0 +1,74 @@
+"""Generic registry utilities for pluggable backend components."""
+
+from dataclasses import dataclass, field
+from importlib import import_module
+from typing import Any, Callable, Dict, Iterable, Optional
+
+
+class RegistryError(RuntimeError):
+    """Raised when registering duplicated or invalid entries."""
+
+
+@dataclass(slots=True)
+class RegistryEntry:
+    name: str
+    loader: Callable[[], Any]
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def load(self) -> Any:
+        return self.loader()
+
+
+class Registry:
+    """Lightweight registry with lazy module loading support."""
+
+    def __init__(self, namespace: str) -> None:
+        self.namespace = namespace
+        self._entries: Dict[str, RegistryEntry] = {}
+
+    def register(
+        self,
+        name: str,
+        *,
+        loader: Callable[[], Any] | None = None,
+        target: Any | None = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        module_path: str | None = None,
+        attr_name: str | None = None,
+    ) -> None:
+        if name in self._entries:
+            raise RegistryError(f"Duplicate registration for '{name}' in {self.namespace}")
+
+        if loader is None:
+            if target is None and module_path is None:
+                raise RegistryError("Must provide loader, target, or module_path/attr_name")
+            if target is not None:
+                loader = lambda target=target: target
+            else:
+                if not attr_name:
+                    raise RegistryError("module_path requires attr_name")
+
+                def _lazy_loader(mod_path: str = module_path, attr: str = attr_name) -> Any:
+                    module = import_module(mod_path)
+                    return getattr(module, attr)
+
+                loader = _lazy_loader
+
+        entry = RegistryEntry(name=name, loader=loader, metadata=dict(metadata or {}))
+        self._entries[name] = entry
+
+    def get(self, name: str) -> RegistryEntry:
+        try:
+            return self._entries[name]
+        except KeyError as exc:
+            raise RegistryError(f"Unknown entry '{name}' in {self.namespace}") from exc
+
+    def names(self) -> Iterable[str]:
+        return self._entries.keys()
+
+    def items(self) -> Iterable[tuple[str, RegistryEntry]]:
+        return self._entries.items()
+
+    def metadata_for(self, name: str) -> Dict[str, Any]:
+        return dict(self.get(name).metadata)
+

utils/schema_exporter.py

@@ -0,0 +1,139 @@
+"""Schema exporter for dynamic configuration metadata."""
+
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List, Mapping, Sequence, Type
+
+from entity.configs import BaseConfig
+from entity.configs.graph import DesignConfig
+
+SCHEMA_VERSION = "0.1.0"
+
+
+class SchemaResolutionError(ValueError):
+    """Raised when breadcrumbs fail to resolve to a config node."""
+
+
+@dataclass(frozen=True)
+class Breadcrumb:
+    """Describes one hop in the config tree."""
+
+    node: str
+    field: str | None = None
+    value: Any | None = None
+
+    @classmethod
+    def from_mapping(cls, data: Mapping[str, Any]) -> "Breadcrumb":
+        node = str(data.get("node")) if data.get("node") else ""
+        if not node:
+            raise SchemaResolutionError("breadcrumb entry missing 'node'")
+        field = data.get("field")
+        if field is not None:
+            field = str(field)
+        index = data.get("index")
+        if index is not None and not isinstance(index, int):
+            raise SchemaResolutionError("breadcrumb 'index' must be integer when provided")
+        value = data.get("value")
+        return cls(node=node, field=field, value=value)
+
+    def to_json(self) -> Dict[str, Any]:
+        payload: Dict[str, Any] = {"node": self.node}
+        if self.field is not None:
+            payload["field"] = self.field
+        if self.value is not None:
+            payload["value"] = self.value
+        return payload
+
+
+def _normalize_breadcrumbs(raw: Sequence[Mapping[str, Any]] | None) -> List[Breadcrumb]:
+    if not raw:
+        return []
+    return [Breadcrumb.from_mapping(item) for item in raw]
+
+
+def _resolve_config_class(
+    breadcrumbs: Sequence[Breadcrumb],
+    *,
+    root_cls: Type[BaseConfig] = DesignConfig,
+) -> Type[BaseConfig]:
+    current_cls: Type[BaseConfig] = root_cls
+    for crumb in breadcrumbs:
+        if crumb.node != current_cls.__name__:
+            raise SchemaResolutionError(
+                f"breadcrumb node '{crumb.node}' does not match current config '{current_cls.__name__}'"
+            )
+        if crumb.field is None:
+            continue
+        child_cls = current_cls.resolve_child(crumb.field, crumb.value)
+        if child_cls is None:
+            spec = current_cls.field_specs().get(crumb.field)
+            if not spec or spec.child is None:
+                raise SchemaResolutionError(
+                    f"field '{crumb.field}' on {current_cls.__name__} is not navigable"
+                )
+            child_cls = spec.child
+        current_cls = child_cls
+    return current_cls
+
+
+def _serialize_field(config_cls: Type[BaseConfig], name: str, spec_dict: Dict[str, Any]) -> Dict[str, Any]:
+    field_spec = spec_dict[name]
+    data = field_spec.to_json()
+    routes = [
+        {
+            "childKey": key.to_json(),
+            "childNode": target.__name__,
+        }
+        for key, target in config_cls.child_routes().items()
+        if key.field == name
+    ]
+    if routes:
+        data["childRoutes"] = routes
+    return data
+
+
+def _ordered_field_names(specs: Mapping[str, Any]) -> List[str]:
+    """Return field names with required ones first while keeping relative order."""
+
+    items = list(specs.items())
+    required_names = [name for name, spec in items if getattr(spec, "required", False)]
+    optional_names = [name for name, spec in items if not getattr(spec, "required", False)]
+    return required_names + optional_names
+
+
+def _hash_payload(payload: Dict[str, Any]) -> str:
+    serialized = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str)
+    return hashlib.sha1(serialized.encode("utf-8")).hexdigest()
+
+
+def build_schema_response(
+    breadcrumbs_raw: Sequence[Mapping[str, Any]] | None = None,
+    *,
+    root_cls: Type[BaseConfig] = DesignConfig,
+) -> Dict[str, Any]:
+    """Return a JSON-serializable schema response for the provided breadcrumbs."""
+
+    breadcrumbs = _normalize_breadcrumbs(breadcrumbs_raw)
+    target_cls = _resolve_config_class(breadcrumbs, root_cls=root_cls)
+    schema_node = target_cls.collect_schema()
+    field_specs = target_cls.field_specs()
+    ordered_fields = _ordered_field_names(field_specs)
+    fields_payload = [_serialize_field(target_cls, name, field_specs) for name in ordered_fields]
+
+    response = {
+        "schemaVersion": SCHEMA_VERSION,
+        "node": schema_node.node,
+        "fields": fields_payload,
+        "constraints": [constraint.to_json() for constraint in schema_node.constraints],
+        "breadcrumbs": [crumb.to_json() for crumb in breadcrumbs],
+    }
+    response["cacheKey"] = _hash_payload({"node": schema_node.node, "breadcrumbs": response["breadcrumbs"]})
+    return response
+
+
+__all__ = [
+    "Breadcrumb",
+    "SchemaResolutionError",
+    "build_schema_response",
+]

utils/strs.py

@@ -0,0 +1,5 @@
+def titleize(value: str) -> str:
+    sanitized = value.replace("_", " ").replace("-", " ").strip()
+    if not sanitized:
+        return value
+    return " ".join(part.capitalize() for part in sanitized.split())

utils/structured_logger.py

@@ -0,0 +1,187 @@
+"""Structured logging utilities for the DevAll workflow system."""
+
+import json
+import logging
+import sys
+import traceback
+import datetime
+from enum import Enum
+from pathlib import Path
+import os
+
+from entity.enums import LogLevel
+from utils.exceptions import MACException
+
+
+class LogType(str, Enum):
+    """Types of structured logs."""
+    REQUEST = "request"
+    RESPONSE = "response"
+    ERROR = "error"
+    WORKFLOW = "workflow"
+    SECURITY = "security"
+    PERFORMANCE = "performance"
+
+
+class StructuredLogger:
+    """A structured logger that outputs JSON format logs with consistent fields."""
+    
+    def __init__(self, name: str, log_level: LogLevel = LogLevel.INFO, log_file: str = None):
+        self.name = name
+        self.log_level = log_level
+        self.logger = logging.getLogger(name)
+        self.logger.setLevel(self._get_logging_level(log_level))
+        
+        # Create formatter
+        formatter = logging.Formatter('%(message)s')
+        
+        # Create handler
+        if log_file:
+            # Ensure log directory exists
+            log_path = Path(log_file)
+            log_path.parent.mkdir(parents=True, exist_ok=True)
+            handler = logging.FileHandler(log_file)
+        else:
+            handler = logging.StreamHandler(sys.stdout)
+        
+        handler.setFormatter(formatter)
+        self.logger.addHandler(handler)
+        
+        # For correlation IDs
+        self.correlation_id = None
+    
+    def _get_logging_level(self, log_level: LogLevel) -> int:
+        """Convert LogLevel enum to logging module level."""
+        level_map = {
+            LogLevel.DEBUG: logging.DEBUG,
+            LogLevel.INFO: logging.INFO,
+            LogLevel.WARNING: logging.WARNING,
+            LogLevel.ERROR: logging.ERROR,
+            LogLevel.CRITICAL: logging.CRITICAL
+        }
+        return level_map.get(log_level, logging.INFO)
+    
+    def _should_log(self, level: LogLevel) -> bool:
+        """Check if a log level should be logged based on configured level."""
+        return level >= self.log_level
+    
+    def _format_log(self, log_type: LogType, level: LogLevel, message: str, 
+                    correlation_id: str = None, **kwargs) -> str:
+        """Format log entry as JSON string."""
+        log_entry = {
+            "timestamp": datetime.datetime.now(datetime.UTC),
+            "log_type": log_type.value,
+            "level": level.value,
+            "logger": self.name,
+            "message": message,
+            "correlation_id": correlation_id or self.correlation_id,
+            **kwargs
+        }
+        return json.dumps(log_entry, default=str)
+    
+    def _log(self, log_type: LogType, level: LogLevel, message: str, 
+             correlation_id: str = None, **kwargs):
+        """Internal logging method."""
+        if self._should_log(level):
+            formatted_log = self._format_log(log_type, level, message, correlation_id, **kwargs)
+            log_level = self._get_logging_level(level)
+            self.logger.log(log_level, formatted_log)
+    
+    def info(self, message: str, correlation_id: str = None, log_type: LogType = LogType.WORKFLOW, **kwargs):
+        """Log information."""
+        self._log(log_type, LogLevel.INFO, message, correlation_id, **kwargs)
+    
+    def debug(self, message: str, correlation_id: str = None, log_type: LogType = LogType.WORKFLOW, **kwargs):
+        """Log debug information."""
+        self._log(log_type, LogLevel.DEBUG, message, correlation_id, **kwargs)
+    
+    def warning(self, message: str, correlation_id: str = None, log_type: LogType = LogType.WORKFLOW, **kwargs):
+        """Log warning."""
+        self._log(log_type, LogLevel.WARNING, message, correlation_id, **kwargs)
+    
+    def error(self, message: str, correlation_id: str = None, log_type: LogType = LogType.ERROR, **kwargs):
+        """Log error with details."""
+        self._log(log_type, LogLevel.ERROR, message, correlation_id, **kwargs)
+    
+    def critical(self, message: str, correlation_id: str = None, log_type: LogType = LogType.ERROR, **kwargs):
+        """Log critical error."""
+        self._log(log_type, LogLevel.CRITICAL, message, correlation_id, **kwargs)
+    
+    def log_exception(self, exception: Exception, message: str = None, 
+                      correlation_id: str = None, **kwargs) -> None:
+        """Log an exception with its traceback."""
+        if message is None:
+            message = str(exception)
+        
+        # Include exception info
+        exception_info = {
+            "exception_type": type(exception).__name__,
+            "exception_message": str(exception),
+            "traceback": traceback.format_exc()
+        }
+        
+        if isinstance(exception, MACException):
+            exception_info["error_code"] = exception.error_code
+            exception_info["exception_details"] = exception.details
+        
+        self._log(LogType.ERROR, LogLevel.ERROR, message, correlation_id, 
+                 exception=exception_info, **kwargs)
+    
+    def log_request(self, method: str, url: str, correlation_id: str = None, **kwargs):
+        """Log incoming request."""
+        self._log(LogType.REQUEST, LogLevel.INFO, f"Incoming {method} request to {url}", 
+                 correlation_id, method=method, url=url, **kwargs)
+    
+    def log_response(self, status_code: int, response_time: float, correlation_id: str = None, **kwargs):
+        """Log outgoing response."""
+        self._log(LogType.RESPONSE, LogLevel.INFO, 
+                 f"Response with status {status_code} in {response_time:.3f}s", 
+                 correlation_id, status_code=status_code, response_time=response_time, **kwargs)
+    
+    def log_security_event(self, event_type: str, message: str, correlation_id: str = None, **kwargs):
+        """Log security-related events."""
+        self._log(LogType.SECURITY, LogLevel.WARNING, message, correlation_id, 
+                 event_type=event_type, **kwargs)
+    
+    def log_performance(self, operation: str, duration: float, correlation_id: str = None, **kwargs):
+        """Log performance metrics."""
+        self._log(LogType.PERFORMANCE, LogLevel.INFO, 
+                 f"Operation {operation} completed in {duration:.3f}s", 
+                 correlation_id, operation=operation, duration=duration, **kwargs)
+    
+    def log_workflow_event(self, workflow_id: str, event_type: str, message: str, 
+                          correlation_id: str = None, **kwargs):
+        """Log workflow-specific events."""
+        self._log(LogType.WORKFLOW, LogLevel.INFO, message, correlation_id,
+                 workflow_id=workflow_id, event_type=event_type, **kwargs)
+    
+    def set_correlation_id(self, correlation_id: str):
+        """Set the correlation ID for this logger instance."""
+        self.correlation_id = correlation_id
+
+
+# Global logger instances
+_server_logger = None
+_workflow_logger = None
+
+
+def get_server_logger() -> StructuredLogger:
+    """Get the global server logger instance."""
+    global _server_logger
+    if _server_logger is None:
+        log_file = os.getenv('SERVER_LOG_FILE', 'logs/server.log')
+        log_level_str = os.getenv('LOG_LEVEL', 'INFO').upper()
+        log_level = LogLevel[log_level_str]
+        _server_logger = StructuredLogger('server', log_level, log_file)
+    return _server_logger
+
+
+def get_workflow_logger(name: str = 'workflow') -> StructuredLogger:
+    """Get a workflow logger instance."""
+    global _workflow_logger
+    if _workflow_logger is None:
+        log_file = os.getenv('WORKFLOW_LOG_FILE', f'logs/{name}.log')
+        log_level_str = os.getenv('LOG_LEVEL', 'INFO').upper()
+        log_level = LogLevel[log_level_str]
+        _workflow_logger = StructuredLogger(name, log_level, log_file)
+    return _workflow_logger

utils/task_input.py

@@ -0,0 +1,61 @@
+"""Helpers for building initial task inputs with optional attachments."""
+
+import mimetypes
+from pathlib import Path
+from typing import List, Sequence, Union
+
+from entity.messages import Message, MessageBlock, MessageBlockType, MessageRole
+from utils.attachments import AttachmentStore
+
+
+class TaskInputBuilder:
+    """Builds task input payloads that optionally include attachments."""
+
+    def __init__(self, attachment_store: AttachmentStore):
+        self.attachment_store = attachment_store
+
+    def build_from_file_paths(
+        self,
+        prompt: str,
+        attachment_paths: Sequence[str],
+    ) -> Union[str, List[Message]]:
+        if not attachment_paths:
+            return prompt
+
+        blocks: List[MessageBlock] = []
+
+        for raw_path in attachment_paths:
+            file_path = Path(raw_path).expanduser()
+            if not file_path.exists():
+                raise FileNotFoundError(f"Attachment not found: {file_path}")
+            mime_type, _ = mimetypes.guess_type(str(file_path))
+            record = self.attachment_store.register_file(
+                file_path,
+                kind=MessageBlockType.from_mime_type(mime_type),
+                display_name=file_path.name,
+                mime_type=mime_type,
+                extra={
+                    "source": "user_upload",
+                    "origin": "cli_attachment",
+                    "original_path": str(file_path),
+                },
+            )
+            blocks.append(record.as_message_block())
+
+        return self.build_from_blocks(prompt, blocks)
+
+    @staticmethod
+    def build_from_blocks(prompt: str, blocks: Sequence[MessageBlock]) -> List[Message]:
+        final_blocks: List[MessageBlock] = []
+        if prompt:
+            final_blocks.append(MessageBlock.text_block(prompt))
+        final_blocks.extend(blocks)
+        if not final_blocks:
+            final_blocks.append(MessageBlock.text_block(""))
+        return [
+            Message(
+                role=MessageRole.USER,
+                content=final_blocks,
+                metadata={"source": "TASK"},
+            )
+        ]

utils/token_tracker.py

@@ -0,0 +1,152 @@
+"""Token usage tracking module for DevAll project."""
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Dict, List, Optional, Any
+from collections import defaultdict
+
+
+@dataclass
+class TokenUsage:
+    """Stores token usage metrics for individual API calls."""
+    input_tokens: int = 0
+    output_tokens: int = 0
+    total_tokens: int = 0
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    timestamp: datetime = field(default_factory=datetime.now)
+    node_id: Optional[str] = None
+    model_name: Optional[str] = None
+    workflow_id: Optional[str] = None
+    provider: Optional[str] = None  # Add provider field
+
+    def to_dict(self):
+        """Convert to dictionary format."""
+        return {
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "total_tokens": self.total_tokens,
+            "metadata": dict(self.metadata),
+            "timestamp": self.timestamp.isoformat(),
+            "node_id": self.node_id,
+            "model_name": self.model_name,
+            "workflow_id": self.workflow_id,
+            "provider": self.provider  # Include provider in output
+        }
+
+
+class TokenTracker:
+    """Singleton class to track token usage across a workflow."""
+    
+    def __init__(self, workflow_id: str):
+        self.workflow_id = workflow_id
+        self.total_usage = TokenUsage()
+        self.node_usages = defaultdict(TokenUsage)
+        self.model_usages = defaultdict(TokenUsage)
+        self.call_history = []
+        self.node_call_counts = defaultdict(int)  # Track how many times each node is called
+
+    def record_usage(self, node_id: str, model_name: str, usage: TokenUsage, provider: str = None):
+        """Records token usage for a specific call, handling multiple node executions."""
+        # Update the usage with provider if it wasn't set already
+        if provider and not usage.provider:
+            usage.provider = provider
+            
+        # Add to total usage
+        self.total_usage.input_tokens += usage.input_tokens
+        self.total_usage.output_tokens += usage.output_tokens
+        self.total_usage.total_tokens += usage.total_tokens
+        
+        # Add to node-specific usage
+        node_usage = self.node_usages[node_id]
+        node_usage.input_tokens += usage.input_tokens
+        node_usage.output_tokens += usage.output_tokens
+        node_usage.total_tokens += usage.total_tokens
+        if provider:
+            node_usage.provider = provider  # Store provider info
+        
+        # Add to model-specific usage
+        model_usage = self.model_usages[model_name]
+        model_usage.input_tokens += usage.input_tokens
+        model_usage.output_tokens += usage.output_tokens
+        model_usage.total_tokens += usage.total_tokens
+        if provider:
+            model_usage.provider = provider  # Store provider info
+        
+        # Increment call count for this node
+        self.node_call_counts[node_id] += 1
+        
+        # Add to call history
+        history_entry = {
+            "node_id": node_id,
+            "model_name": model_name,
+            "input_tokens": usage.input_tokens,
+            "output_tokens": usage.output_tokens,
+            "total_tokens": usage.total_tokens,
+            "metadata": dict(usage.metadata),
+            "timestamp": usage.timestamp.isoformat(),
+            "execution_number": self.node_call_counts[node_id]  # Track which execution this is
+        }
+        
+        # Add provider to history entry if available
+        if provider:
+            history_entry["provider"] = provider
+            
+        self.call_history.append(history_entry)
+
+    def get_total_usage(self) -> TokenUsage:
+        """Get total token usage for the workflow."""
+        return self.total_usage
+
+    def get_node_usage(self, node_id: str) -> TokenUsage:
+        """Get token usage for a specific node (across all its executions)."""
+        return self.node_usages[node_id]
+
+    def get_model_usage(self, model_name: str) -> TokenUsage:
+        """Get token usage for a specific model."""
+        return self.model_usages[model_name]
+
+    def get_node_execution_count(self, node_id: str) -> int:
+        """Get how many times a node was executed."""
+        return self.node_call_counts[node_id]
+
+    def get_token_usage(self) -> Dict[str, Any]:
+        data = {
+            "workflow_id": self.workflow_id,
+            "total_usage": {
+                "input_tokens": self.total_usage.input_tokens,
+                "output_tokens": self.total_usage.output_tokens,
+                "total_tokens": self.total_usage.total_tokens,
+            },
+            "node_usages": {
+                node_id: {
+                    "input_tokens": usage.input_tokens,
+                    "output_tokens": usage.output_tokens,
+                    "total_tokens": usage.total_tokens,
+                }
+                for node_id, usage in self.node_usages.items()
+            },
+            "model_usages": {
+                model_name: {
+                    "input_tokens": usage.input_tokens,
+                    "output_tokens": usage.output_tokens,
+                    "total_tokens": usage.total_tokens,
+                }
+                for model_name, usage in self.model_usages.items()
+            },
+            "node_execution_counts": dict(self.node_call_counts),
+            "call_history": self.call_history,
+        }
+        return data
+
+    def export_to_file(self, filepath: str):
+        """Export token usage data to a JSON file."""
+        import json
+        from pathlib import Path
+        
+        # Create directory if it doesn't exist
+        path = Path(filepath)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        
+        data = self.get_token_usage()
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)

utils/vars_resolver.py

@@ -0,0 +1,93 @@
+"""Placeholder resolution for design configs."""
+
+
+import re
+from typing import Any, Dict, Mapping, MutableMapping, Sequence
+
+from entity.configs.base import ConfigError, extend_path
+
+
+_PLACEHOLDER_PATTERN = re.compile(r"\$\{([A-Za-z0-9_]+)\}")
+_PLACEHOLDER_ONLY_PATTERN = re.compile(r"^\s*\$\{([A-Za-z0-9_]+)\}\s*$")
+
+
+class PlaceholderResolver:
+    """Resolve ``${VAR}`` placeholders within nested structures."""
+
+    def __init__(self, env_lookup: Mapping[str, Any], root_vars: Mapping[str, Any]):
+        self._env_lookup = dict(env_lookup)
+        self._raw_root = dict(root_vars or {})
+        self._resolved_root: Dict[str, Any] = {}
+
+    @property
+    def resolved_root(self) -> Dict[str, Any]:
+        # include untouched root vars so undeclared-but-needed entries remain available
+        merged = dict(self._raw_root)
+        merged.update(self._resolved_root)
+        return merged
+
+    def resolve(self, data: MutableMapping[str, Any], *, path: str = "root") -> MutableMapping[str, Any]:
+        if not isinstance(data, MutableMapping):
+            raise ConfigError("YAML root must be a mapping", path=path)
+        self._resolve_value(data, path, stack=())
+        return data
+
+    def _resolve_value(self, value: Any, path: str, *, stack: Sequence[str]) -> Any:
+        if isinstance(value, str):
+            return self._resolve_string(value, path, stack)
+        if isinstance(value, list):
+            for idx, item in enumerate(value):
+                value[idx] = self._resolve_value(item, extend_path(path, f"[{idx}]"), stack=stack)
+            return value
+        if isinstance(value, MutableMapping):
+            for key in list(value.keys()):
+                child_path = extend_path(path, str(key))
+                value[key] = self._resolve_value(value[key], child_path, stack=stack)
+            return value
+        return value
+
+    def _resolve_string(self, raw: str, path: str, stack: Sequence[str]) -> Any:
+        only_match = _PLACEHOLDER_ONLY_PATTERN.fullmatch(raw)
+        if only_match:
+            var_name = only_match.group(1)
+            return self._lookup(var_name, path, stack)
+
+        def replacer(match: re.Match[str]) -> str:
+            var_name = match.group(1)
+            resolved = self._lookup(var_name, path, stack)
+            return str(resolved)
+
+        return _PLACEHOLDER_PATTERN.sub(replacer, raw)
+
+    def _lookup(self, name: str, path: str, stack: Sequence[str]) -> Any:
+        if name in self._resolved_root:
+            return self._resolved_root[name]
+        if name in stack:
+            raise ConfigError(f"Detected placeholder cycle referencing '{name}'", path)
+        if name in self._raw_root:
+            resolved = self._resolve_value(self._raw_root[name], extend_path("vars", name), stack=stack + (name,))
+            self._resolved_root[name] = resolved
+            return resolved
+        if name in self._env_lookup:
+            return self._env_lookup[name]
+        raise ConfigError(f"Unresolved placeholder '${{{name}}}'", path)
+
+
+def resolve_design_placeholders(data: MutableMapping[str, Any], *, env_lookup: Mapping[str, Any], path: str = "root") -> Dict[str, Any]:
+    """Resolve placeholders in-place and return the resolved root vars."""
+    resolver = PlaceholderResolver(env_lookup, data.get("vars") or {})
+    resolver.resolve(data, path=path)
+    data["vars"] = resolver.resolved_root
+    return resolver.resolved_root
+
+
+def resolve_mapping_with_vars(
+    data: MutableMapping[str, Any],
+    *,
+    env_lookup: Mapping[str, Any],
+    vars_map: Mapping[str, Any],
+    path: str = "root",
+) -> MutableMapping[str, Any]:
+    """Resolve placeholders using an explicit vars map without mutating it."""
+    resolver = PlaceholderResolver(env_lookup, vars_map)
+    return resolver.resolve(data, path=path)

utils/workspace_scanner.py

@@ -0,0 +1,71 @@
+"""Utilities for scanning nested code_workspace directories."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator, List, Optional
+
+
+@dataclass
+class WorkspaceEntry:
+    """Metadata about a workspace file or directory."""
+
+    path: str  # relative path from workspace root
+    type: str  # "file" | "directory"
+    size: Optional[int]
+    modified_ts: Optional[float]
+    depth: int
+
+
+def iter_workspace_entries(
+    root: Path | str,
+    *,
+    recursive: bool = True,
+    max_depth: int = 5,
+    include_hidden: bool = False,
+) -> Iterator[WorkspaceEntry]:
+    """Yield entries under the workspace root respecting depth/hidden filters."""
+
+    base = Path(root).resolve()
+    if not base.exists():
+        return
+
+    stack: List[tuple[Path, int]] = [(base, 0)]
+    while stack:
+        current, depth = stack.pop()
+        try:
+            children = sorted(current.iterdir(), key=lambda p: p.name.lower())
+        except FileNotFoundError:
+            continue
+        except PermissionError:
+            continue
+        for child in children:
+            try:
+                rel = child.relative_to(base)
+            except ValueError:
+                continue
+            if not include_hidden and _is_hidden(rel):
+                continue
+            entry_type = "directory" if child.is_dir() else "file"
+            size = None
+            modified = None
+            try:
+                stat = child.stat()
+                modified = stat.st_mtime
+                if child.is_file():
+                    size = stat.st_size
+            except (FileNotFoundError, PermissionError, OSError):
+                pass
+            child_depth = depth + 1
+            yield WorkspaceEntry(
+                path=str(rel),
+                type=entry_type,
+                size=size,
+                modified_ts=modified,
+                depth=child_depth,
+            )
+            if recursive and child.is_dir() and child_depth < max_depth:
+                stack.append((child, child_depth))
+
+
+def _is_hidden(relative_path: Path) -> bool:
+    return any(part.startswith(".") for part in relative_path.parts)