chore: import upstream snapshot with attribution

server/services/artifact_dispatcher.py

@@ -0,0 +1,66 @@
+"""Utilities to distribute artifact events to internal consumers."""
+
+import logging
+from typing import Sequence
+
+from server.services.artifact_events import ArtifactEvent
+from server.services.session_store import WorkflowSessionStore
+from workflow.hooks.workspace_artifact import WorkspaceArtifact
+
+
+class ArtifactDispatcher:
+    """Persists artifact events and optionally mirrors them to WebSocket clients."""
+
+    def __init__(
+        self,
+        session_id: str,
+        session_store: WorkflowSessionStore,
+        websocket_manager=None,
+    ) -> None:
+        self.session_id = session_id
+        self.session_store = session_store
+        self.websocket_manager = websocket_manager
+        self.logger = logging.getLogger(__name__)
+
+    def emit_workspace_artifacts(self, artifacts: Sequence[WorkspaceArtifact]) -> None:
+        if not artifacts:
+            return
+        events = [self._workspace_to_event(artifact) for artifact in artifacts]
+        self.emit(events)
+
+    def emit(self, events: Sequence[ArtifactEvent]) -> None:
+        if not events:
+            return
+        queue = self.session_store.get_artifact_queue(self.session_id)
+        if not queue:
+            self.logger.debug("Artifact queue missing for session %s", self.session_id)
+            return
+        queue.append_many(events)
+        if self.websocket_manager:
+            payload = {
+                "type": "artifact_created",
+                "data": {
+                    "session_id": self.session_id,
+                    "events": [event.to_dict() for event in events],
+                },
+            }
+            try:
+                self.websocket_manager.send_message_sync(self.session_id, payload)
+            except Exception as exc:
+                self.logger.warning("Failed to broadcast artifact events: %s", exc)
+
+    def _workspace_to_event(self, artifact: WorkspaceArtifact) -> ArtifactEvent:
+        return ArtifactEvent(
+            node_id=artifact.node_id,
+            attachment_id=artifact.attachment_id,
+            file_name=artifact.file_name,
+            relative_path=artifact.relative_path,
+            workspace_path=artifact.absolute_path,
+            mime_type=artifact.mime_type,
+            size=artifact.size,
+            sha256=artifact.sha256,
+            data_uri=artifact.data_uri,
+            created_at=artifact.created_at,
+            change_type=artifact.change_type,
+            extra=artifact.extra,
+        )

server/services/artifact_events.py

@@ -0,0 +1,174 @@
+"""Artifact event queue utilities used to expose workflow-produced files."""
+
+import threading
+import time
+import uuid
+from collections import deque
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Deque, Dict, Iterable, List, Optional, Sequence
+
+
+@dataclass
+class ArtifactEvent:
+    """Represents a single file artifact surfaced to the frontend."""
+
+    node_id: str
+    attachment_id: str
+    file_name: str
+    relative_path: str
+    workspace_path: str
+    mime_type: Optional[str]
+    size: Optional[int]
+    sha256: Optional[str]
+    data_uri: Optional[str]
+    created_at: float = field(default_factory=lambda: time.time())
+    event_id: str = field(default_factory=lambda: uuid.uuid4().hex)
+    sequence: int = 0
+    change_type: str = "created"
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "event_id": self.event_id,
+            "sequence": self.sequence,
+            "node_id": self.node_id,
+            "attachment_id": self.attachment_id,
+            "file_name": self.file_name,
+            "relative_path": self.relative_path,
+            "workspace_path": self.workspace_path,
+            "mime_type": self.mime_type,
+            "size": self.size,
+            "sha256": self.sha256,
+            "data_uri": self.data_uri,
+            "created_at": self.created_at,
+            "change_type": self.change_type,
+            "extra": self.extra,
+        }
+
+    def matches_filter(
+        self,
+        *,
+        include_mime: Optional[Sequence[str]] = None,
+        include_ext: Optional[Sequence[str]] = None,
+        max_size: Optional[int] = None,
+    ) -> bool:
+        if max_size is not None and self.size is not None and self.size > max_size:
+            return False
+
+        if include_mime:
+            mime = (self.mime_type or "").lower()
+            if mime and any(mime.startswith(prefix.lower()) for prefix in include_mime):
+                pass
+            elif mime in (m.lower() for m in include_mime):
+                pass
+            else:
+                return False
+
+        if include_ext:
+            suffix = Path(self.file_name).suffix.lower()
+            if suffix.startswith("."):
+                suffix = suffix[1:]
+            include_ext_normalized = {ext.lower().lstrip(".") for ext in include_ext}
+            if suffix not in include_ext_normalized:
+                return False
+
+        return True
+
+
+class ArtifactEventQueue:
+    """Thread-safe bounded queue that supports blocking waits."""
+
+    def __init__(self, *, max_events: int = 2000) -> None:
+        self._events: Deque[ArtifactEvent] = deque()
+        self._condition = threading.Condition()
+        self._max_events = max_events
+        self._last_sequence = 0
+        self._min_sequence = 1
+
+    def append_many(self, events: Iterable[ArtifactEvent]) -> None:
+        materialized = [event for event in events if event is not None]
+        if not materialized:
+            return
+        with self._condition:
+            for event in materialized:
+                self._last_sequence += 1
+                event.sequence = self._last_sequence
+                self._events.append(event)
+            while len(self._events) > self._max_events:
+                self._events.popleft()
+                self._min_sequence = max(self._min_sequence, self._last_sequence - len(self._events) + 1)
+            self._condition.notify_all()
+
+    def snapshot(
+        self,
+        *,
+        after: Optional[int] = None,
+        include_mime: Optional[Sequence[str]] = None,
+        include_ext: Optional[Sequence[str]] = None,
+        max_size: Optional[int] = None,
+        limit: int = 50,
+    ) -> tuple[List[ArtifactEvent], int]:
+        limit = max(1, min(limit, 200))
+        start_seq = after if after is not None else 0
+        start_seq = max(start_seq, self._min_sequence - 1)
+
+        events: List[ArtifactEvent] = []
+        next_cursor = start_seq
+        for event in self._events:
+            if event.sequence <= start_seq:
+                continue
+            next_cursor = event.sequence
+            if event.matches_filter(
+                include_mime=include_mime,
+                include_ext=include_ext,
+                max_size=max_size,
+            ):
+                events.append(event)
+                if len(events) >= limit:
+                    break
+        if next_cursor < start_seq:
+            next_cursor = start_seq
+        return events, next_cursor
+
+    def wait_for_events(
+        self,
+        *,
+        after: Optional[int],
+        include_mime: Optional[Sequence[str]],
+        include_ext: Optional[Sequence[str]],
+        max_size: Optional[int],
+        limit: int,
+        timeout: float,
+    ) -> tuple[List[ArtifactEvent], int, bool]:
+        """Block until matching events appear or timeout expires.
+
+        Returns (events, next_cursor, timeout_reached)
+        """
+        deadline = time.time() + max(0.0, timeout)
+        with self._condition:
+            events, next_cursor = self.snapshot(
+                after=after,
+                include_mime=include_mime,
+                include_ext=include_ext,
+                max_size=max_size,
+                limit=limit,
+            )
+            while not events and time.time() < deadline:
+                remaining = deadline - time.time()
+                if remaining <= 0:
+                    break
+                self._condition.wait(timeout=remaining)
+                events, next_cursor = self.snapshot(
+                    after=after,
+                    include_mime=include_mime,
+                    include_ext=include_ext,
+                    max_size=max_size,
+                    limit=limit,
+                )
+            timed_out = not events
+            return events, next_cursor or (after or 0), timed_out
+
+    @property
+    def last_sequence(self) -> int:
+        return self._last_sequence

server/services/attachment_service.py

@@ -0,0 +1,131 @@
+"""Attachment helpers shared by HTTP routes and executors."""
+
+import logging
+import mimetypes
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from fastapi import UploadFile
+
+from entity.messages import MessageBlock, MessageBlockType
+from utils.attachments import AttachmentStore, AttachmentRecord
+
+
+class AttachmentService:
+    """Handles attachment lifecycle per session."""
+
+    def __init__(self, *, root: Path | str = Path("WareHouse")) -> None:
+        self.logger = logging.getLogger(__name__)
+        self.attachments_root = Path(root)
+        self.attachments_root.mkdir(parents=True, exist_ok=True)
+        env_flag = os.environ.get("MAC_AUTO_CLEAN_ATTACHMENTS", "0").strip().lower()
+        self.clean_on_cleanup = env_flag in {"1", "true", "yes"}
+
+    def prepare_session_workspace(self, session_id: str) -> Path:
+        return self._session_attachments_path(session_id, create=True)
+
+    def cleanup_session(self, session_id: str) -> None:
+        attachment_dir = self._session_attachments_path(session_id, create=False)
+        if not attachment_dir:
+            return
+        if self.clean_on_cleanup:
+            shutil.rmtree(attachment_dir, ignore_errors=True)
+            self.logger.info("Cleaned attachment directory for session %s", session_id)
+        else:
+            self.logger.info(
+                "Attachment cleanup disabled; preserved files for session %s", session_id
+            )
+
+    def get_attachment_store(self, session_id: str) -> AttachmentStore:
+        path = self.prepare_session_workspace(session_id)
+        return AttachmentStore(path)
+
+    @staticmethod
+    def _safe_upload_filename(raw: Optional[str]) -> str:
+        """Reduce a client-supplied upload filename to a safe basename.
+
+        The multipart ``filename`` is attacker-controlled. Joining it onto the
+        temporary upload directory verbatim allows path traversal (e.g.
+        ``../../../etc/cron.d/x``): the write target escapes the temp dir and the
+        cleanup step then unlinks the same traversed path, yielding arbitrary
+        file write and delete. Normalise both POSIX and Windows separators and
+        keep only the final path component so the write stays confined.
+        """
+        candidate = os.path.basename((raw or "").replace("\\", "/")).strip()
+        if not candidate or candidate in {".", ".."}:
+            return "upload.bin"
+        return candidate
+
+    async def save_upload_file(self, session_id: str, upload: UploadFile) -> AttachmentRecord:
+        filename = self._safe_upload_filename(upload.filename)
+        temp_dir = Path(tempfile.mkdtemp(prefix="mac_upload_"))
+        temp_path = temp_dir / filename
+        try:
+            with temp_path.open("wb") as buffer:
+                while True:
+                    chunk = await upload.read(1024 * 1024)
+                    if not chunk:
+                        break
+                    buffer.write(chunk)
+            store = self.get_attachment_store(session_id)
+            mime_type = upload.content_type or mimetypes.guess_type(filename)[0]
+            record = store.register_file(
+                temp_path,
+                kind=MessageBlockType.from_mime_type(mime_type),
+                display_name=filename,
+                mime_type=mime_type,
+                extra={
+                    "source": "user_upload",
+                    "origin": "web_upload",
+                    "session_id": session_id,
+                },
+            )
+            return record
+        finally:
+            if temp_path.exists():
+                try:
+                    temp_path.unlink()
+                except OSError:
+                    pass
+            try:
+                temp_dir.rmdir()
+            except OSError:
+                pass
+
+    def build_attachment_blocks(
+        self,
+        session_id: str,
+        attachment_ids: List[str],
+        *,
+        target_store: Optional[AttachmentStore] = None,
+    ) -> List[MessageBlock]:
+        if not attachment_ids:
+            return []
+        source_store = self.get_attachment_store(session_id)
+        source_root = source_store.root.resolve()
+        target_root = target_store.root.resolve() if target_store else None
+        blocks: List[MessageBlock] = []
+        for attachment_id in attachment_ids:
+            record = source_store.get(attachment_id)
+            if not record:
+                continue
+            if target_store:
+                copy_required = target_root != source_root
+                record = target_store.ingest_record(record, copy_file=copy_required)
+            blocks.append(record.as_message_block())
+        return blocks
+
+    def list_attachment_manifests(self, session_id: str) -> Dict[str, Any]:
+        store = self.get_attachment_store(session_id)
+        return store.export_manifest()
+
+    def _session_attachments_path(self, session_id: str, *, create: bool = True) -> Optional[Path]:
+        session_dir_name = session_id if session_id.startswith("session_") else f"session_{session_id}"
+        path = self.attachments_root / session_dir_name / "code_workspace" / "attachments"
+        if create:
+            path.mkdir(parents=True, exist_ok=True)
+            return path
+        return path if path.exists() else None

server/services/batch_parser.py

@@ -0,0 +1,192 @@
+"""Parse batch task files (CSV/Excel) into runnable tasks."""
+
+import json
+from dataclasses import dataclass
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import pandas as pd
+
+from utils.exceptions import ValidationError
+
+
+@dataclass(frozen=True)
+class BatchTask:
+    row_index: int
+    task_id: Optional[str]
+    task_prompt: str
+    attachment_paths: List[str]
+    vars_override: Dict[str, Any]
+
+
+def parse_batch_file(content: bytes, filename: str) -> Tuple[List[BatchTask], str]:
+    """Parse a CSV/Excel batch file and return tasks plus file base name."""
+    suffix = Path(filename or "").suffix.lower()
+    if suffix not in {".csv", ".xlsx", ".xls"}:
+        raise ValidationError("Unsupported file type; must be .csv or .xlsx/.xls", field="file")
+
+    if suffix == ".csv":
+        df = _read_csv(content)
+    else:
+        df = _read_excel(content)
+
+    file_base = Path(filename).stem or "batch"
+    tasks = _parse_dataframe(df)
+    if not tasks:
+        raise ValidationError("Batch file contains no tasks", field="file")
+    return tasks, file_base
+
+
+def _read_csv(content: bytes) -> pd.DataFrame:
+    try:
+        import chardet
+    except Exception:
+        chardet = None
+    encoding = "utf-8"
+    if chardet:
+        detected = chardet.detect(content)
+        encoding = detected.get("encoding") or encoding
+    try:
+        return pd.read_csv(BytesIO(content), encoding=encoding)
+    except Exception as exc:
+        raise ValidationError(f"Failed to read CSV: {exc}", field="file")
+
+
+def _read_excel(content: bytes) -> pd.DataFrame:
+    try:
+        return pd.read_excel(BytesIO(content))
+    except Exception as exc:
+        raise ValidationError(f"Failed to read Excel file: {exc}", field="file")
+
+
+def _parse_dataframe(df: pd.DataFrame) -> List[BatchTask]:
+    column_map = {str(col).strip().lower(): col for col in df.columns}
+    id_col = column_map.get("id")
+    task_col = column_map.get("task")
+    attachments_col = column_map.get("attachments")
+    vars_col = column_map.get("vars")
+
+    tasks: List[BatchTask] = []
+    seen_ids: set[str] = set()
+
+    for row_index, row in enumerate(df.to_dict(orient="records"), start=1):
+        task_prompt = _get_cell_text(row, task_col)
+        attachment_paths = _parse_json_list(row, attachments_col, row_index)
+        vars_override = _parse_json_dict(row, vars_col, row_index)
+
+        if not task_prompt and not attachment_paths:
+            raise ValidationError(
+                "Task and attachments cannot both be empty",
+                details={"row_index": row_index},
+            )
+
+        task_id = _get_cell_text(row, id_col)
+        if task_id:
+            if task_id in seen_ids:
+                raise ValidationError(
+                    "Duplicate ID in batch file",
+                    details={"row_index": row_index, "task_id": task_id},
+                )
+            seen_ids.add(task_id)
+
+        tasks.append(
+            BatchTask(
+                row_index=row_index,
+                task_id=task_id or None,
+                task_prompt=task_prompt,
+                attachment_paths=attachment_paths,
+                vars_override=vars_override,
+            )
+        )
+    return tasks
+
+
+def _get_cell_text(row: Dict[str, Any], column: Optional[str]) -> str:
+    if not column:
+        return ""
+    value = row.get(column)
+    if value is None:
+        return ""
+    if isinstance(value, float) and pd.isna(value):
+        return ""
+    if pd.isna(value):
+        return ""
+    return str(value).strip()
+
+
+def _parse_json_list(
+    row: Dict[str, Any],
+    column: Optional[str],
+    row_index: int,
+) -> List[str]:
+    if not column:
+        return []
+    raw_value = row.get(column)
+    if raw_value is None or (isinstance(raw_value, float) and pd.isna(raw_value)):
+        return []
+    if isinstance(raw_value, list):
+        return _ensure_string_list(raw_value, row_index, "Attachments")
+    if isinstance(raw_value, str):
+        if not raw_value.strip():
+            return []
+        try:
+            parsed = json.loads(raw_value)
+        except json.JSONDecodeError as exc:
+            raise ValidationError(
+                f"Invalid JSON in Attachments: {exc}",
+                details={"row_index": row_index},
+            )
+        return _ensure_string_list(parsed, row_index, "Attachments")
+    raise ValidationError(
+        "Attachments must be a JSON list",
+        details={"row_index": row_index},
+    )
+
+
+def _parse_json_dict(
+    row: Dict[str, Any],
+    column: Optional[str],
+    row_index: int,
+) -> Dict[str, Any]:
+    if not column:
+        return {}
+    raw_value = row.get(column)
+    if raw_value is None or (isinstance(raw_value, float) and pd.isna(raw_value)):
+        return {}
+    if isinstance(raw_value, dict):
+        return raw_value
+    if isinstance(raw_value, str):
+        if not raw_value.strip():
+            return {}
+        try:
+            parsed = json.loads(raw_value)
+        except json.JSONDecodeError as exc:
+            raise ValidationError(
+                f"Invalid JSON in Vars: {exc}",
+                details={"row_index": row_index},
+            )
+        if not isinstance(parsed, dict):
+            raise ValidationError(
+                "Vars must be a JSON object",
+                details={"row_index": row_index},
+            )
+        return parsed
+    raise ValidationError(
+        "Vars must be a JSON object",
+        details={"row_index": row_index},
+    )
+
+
+def _ensure_string_list(value: Any, row_index: int, field: str) -> List[str]:
+    if not isinstance(value, list):
+        raise ValidationError(
+            f"{field} must be a JSON list",
+            details={"row_index": row_index},
+        )
+    result: List[str] = []
+    for item in value:
+        if item is None or (isinstance(item, float) and pd.isna(item)):
+            continue
+        result.append(str(item))
+    return result

server/services/batch_run_service.py

@@ -0,0 +1,255 @@
+"""Batch workflow execution helpers."""
+
+import asyncio
+import csv
+import json
+import logging
+import re
+import time
+import uuid
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from check.check import load_config
+from entity.enums import LogLevel
+from entity.graph_config import GraphConfig
+from utils.exceptions import ValidationError
+from utils.task_input import TaskInputBuilder
+from workflow.graph import GraphExecutor
+from workflow.graph_context import GraphContext
+
+from server.services.batch_parser import BatchTask
+from server.services.workflow_storage import validate_workflow_filename
+from server.settings import WARE_HOUSE_DIR, YAML_DIR
+
+
+class BatchRunService:
+    """Runs batch workflows and reports progress over WebSocket."""
+
+    def __init__(self) -> None:
+        self.logger = logging.getLogger(__name__)
+
+    async def run_batch(
+        self,
+        session_id: str,
+        yaml_file: str,
+        tasks: List[BatchTask],
+        websocket_manager,
+        *,
+        max_parallel: int = 5,
+        file_base: str = "batch",
+        log_level: Optional[LogLevel] = None,
+    ) -> None:
+        batch_id = session_id
+        total = len(tasks)
+
+        await websocket_manager.send_message(
+            session_id,
+            {"type": "batch_started", "data": {"batch_id": batch_id, "total": total}},
+        )
+
+        semaphore = asyncio.Semaphore(max_parallel)
+        success_count = 0
+        failure_count = 0
+        result_rows: List[Dict[str, Any]] = []
+        result_lock = asyncio.Lock()
+
+        async def run_task(task: BatchTask) -> None:
+            nonlocal success_count, failure_count
+            task_id = task.task_id or str(uuid.uuid4())
+            task_dir = self._sanitize_label(f"{file_base}-{task_id}")
+
+            await websocket_manager.send_message(
+                session_id,
+                {
+                    "type": "batch_task_started",
+                    "data": {
+                        "row_index": task.row_index,
+                        "task_id": task_id,
+                        "task_dir": task_dir,
+                    },
+                },
+            )
+
+            try:
+                result = await asyncio.to_thread(
+                    self._run_single_task,
+                    session_id,
+                    yaml_file,
+                    task,
+                    task_dir,
+                    log_level,
+                )
+                success_count += 1
+                async with result_lock:
+                    result_rows.append(
+                        {
+                            "row_index": task.row_index,
+                            "task_id": task_id,
+                            "task_dir": task_dir,
+                            "status": "success",
+                            "duration_ms": result["duration_ms"],
+                            "token_usage": result["token_usage"],
+                            "graph_output": result["graph_output"],
+                            "results": result["results"],
+                            "error": "",
+                        }
+                    )
+                await websocket_manager.send_message(
+                    session_id,
+                    {
+                        "type": "batch_task_completed",
+                        "data": {
+                            "row_index": task.row_index,
+                            "task_id": task_id,
+                            "task_dir": task_dir,
+                            "results": result["results"],
+                            "token_usage": result["token_usage"],
+                            "duration_ms": result["duration_ms"],
+                        },
+                    },
+                )
+            except Exception as exc:
+                failure_count += 1
+                async with result_lock:
+                    result_rows.append(
+                        {
+                            "row_index": task.row_index,
+                            "task_id": task_id,
+                            "task_dir": task_dir,
+                            "status": "failed",
+                            "duration_ms": None,
+                            "token_usage": None,
+                            "graph_output": "",
+                            "results": None,
+                            "error": str(exc),
+                        }
+                    )
+                await websocket_manager.send_message(
+                    session_id,
+                    {
+                        "type": "batch_task_failed",
+                        "data": {
+                            "row_index": task.row_index,
+                            "task_id": task_id,
+                            "task_dir": task_dir,
+                            "error": str(exc),
+                        },
+                    },
+                )
+
+        async def run_with_limit(task: BatchTask) -> None:
+            async with semaphore:
+                await run_task(task)
+
+        await asyncio.gather(*(run_with_limit(task) for task in tasks))
+
+        self._write_batch_outputs(session_id, result_rows)
+
+        await websocket_manager.send_message(
+            session_id,
+            {
+                "type": "batch_completed",
+                "data": {
+                    "batch_id": batch_id,
+                    "total": total,
+                    "succeeded": success_count,
+                    "failed": failure_count,
+                },
+            },
+        )
+
+    def _write_batch_outputs(self, session_id: str, result_rows: List[Dict[str, Any]]) -> None:
+        output_root = WARE_HOUSE_DIR / f"session_{session_id}"
+        output_root.mkdir(parents=True, exist_ok=True)
+
+        csv_path = output_root / "batch_results.csv"
+        json_path = output_root / "batch_manifest.json"
+
+        fieldnames = [
+            "row_index",
+            "task_id",
+            "task_dir",
+            "status",
+            "duration_ms",
+            "token_usage",
+            "results",
+            "error",
+        ]
+
+        with csv_path.open("w", newline="", encoding="utf-8") as handle:
+            writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore")
+            writer.writeheader()
+            for row in result_rows:
+                row_copy = dict(row)
+                row_copy["token_usage"] = json.dumps(row_copy.get("token_usage"))
+                row_copy["results"] = row_copy.get("graph_output", "")
+                writer.writerow(row_copy)
+
+        with json_path.open("w", encoding="utf-8") as handle:
+            json.dump(result_rows, handle, ensure_ascii=True, indent=2)
+
+    def _run_single_task(
+        self,
+        session_id: str,
+        yaml_file: str,
+        task: BatchTask,
+        task_dir: str,
+        log_level: Optional[LogLevel],
+    ) -> Dict[str, Any]:
+        yaml_path = self._resolve_yaml_path(yaml_file)
+        design = load_config(yaml_path, vars_override=task.vars_override or None)
+        if any(node.type == "human" for node in design.graph.nodes):
+            raise ValidationError(
+                "Batch execution does not support human nodes",
+                details={"yaml_file": yaml_file},
+            )
+
+        output_root = WARE_HOUSE_DIR / f"session_{session_id}"
+        graph_config = GraphConfig.from_definition(
+            design.graph,
+            name=task_dir,
+            output_root=output_root,
+            source_path=str(yaml_path),
+            vars=design.vars,
+        )
+        graph_config.metadata["fixed_output_dir"] = True
+
+        if log_level:
+            graph_config.log_level = log_level
+            graph_config.definition.log_level = log_level
+
+        graph_context = GraphContext(config=graph_config)
+
+        start_time = time.perf_counter()
+        executor = GraphExecutor(graph_context, session_id=session_id)
+        task_input = self._build_task_input(executor.attachment_store, task)
+        executor._execute(task_input)
+        duration_ms = int((time.perf_counter() - start_time) * 1000)
+
+        return {
+            "results": executor.outputs,
+            "token_usage": executor.token_tracker.get_token_usage(),
+            "duration_ms": duration_ms,
+            "graph_output": executor.get_final_output(),
+        }
+
+    @staticmethod
+    def _build_task_input(attachment_store, task: BatchTask):
+        if task.attachment_paths:
+            builder = TaskInputBuilder(attachment_store)
+            return builder.build_from_file_paths(task.task_prompt, task.attachment_paths)
+        return task.task_prompt
+
+    @staticmethod
+    def _sanitize_label(value: str) -> str:
+        cleaned = re.sub(r"[^a-zA-Z0-9._-]+", "_", value)
+        return cleaned.strip("_") or "task"
+
+    @staticmethod
+    def _resolve_yaml_path(yaml_filename: str) -> Path:
+        safe_name = validate_workflow_filename(yaml_filename, require_yaml_extension=True)
+        yaml_path = YAML_DIR / safe_name
+        if not yaml_path.exists():
+            raise ValidationError("YAML file not found", details={"yaml_file": safe_name})
+        return yaml_path

server/services/message_handler.py

@@ -0,0 +1,91 @@
+import logging
+from typing import Any, Dict
+
+from utils.exceptions import ValidationError
+
+from server.services.session_execution import SessionExecutionController
+from server.services.session_store import WorkflowSessionStore
+
+
+class MessageHandler:
+    """Routes WebSocket messages to the appropriate handlers."""
+
+    def __init__(
+        self,
+        session_store: WorkflowSessionStore,
+        session_controller: SessionExecutionController,
+        workflow_run_service=None,
+    ) -> None:
+        self.session_store = session_store
+        self.session_controller = session_controller
+        self.workflow_run_service = workflow_run_service
+        self.logger = logging.getLogger(__name__)
+
+    async def handle_message(self, session_id: str, data: Dict[str, Any], websocket_manager):
+        message_type = data.get("type")
+        if message_type == "human_input":
+            await self._handle_human_input(session_id, data, websocket_manager)
+        elif message_type == "ping":
+            await self._handle_ping(session_id, websocket_manager)
+        elif message_type == "get_status":
+            await self._handle_get_status(session_id, websocket_manager)
+        elif message_type == "cancel":
+            await self._handle_cancel(session_id, websocket_manager)
+        else:
+            await websocket_manager.send_message(
+                session_id,
+                {"type": "error", "data": {"message": f"Unknown message type: {message_type}"}},
+            )
+
+    async def _handle_cancel(self, session_id: str, websocket_manager):
+        if self.workflow_run_service:
+            self.workflow_run_service.request_cancel(session_id, reason="User requested cancellation")
+        await websocket_manager.send_message(
+            session_id,
+            {"type": "input_received", "data": {"message": "Cancellation requested"}},
+        )
+
+    async def _handle_human_input(self, session_id: str, data: Dict[str, Any], websocket_manager):
+        try:
+            payload = data.get("data", {}) or {}
+            user_input = payload.get("input", "")
+            attachments = payload.get("attachments") or []
+
+            if not user_input and not attachments:
+                await websocket_manager.send_message(
+                    session_id,
+                    {"type": "error", "data": {"message": "Empty input"}},
+                )
+                return
+
+            self.session_controller.provide_human_input(
+                session_id,
+                {"text": user_input, "attachments": attachments},
+            )
+
+            await websocket_manager.send_message(
+                session_id,
+                {"type": "input_received", "data": {"message": "Input received"}},
+            )
+
+        except ValidationError as exc:
+            await websocket_manager.send_message(
+                session_id,
+                {"type": "error", "data": {"message": str(exc)}},
+            )
+        except Exception as exc:
+            self.logger.error("Error handling human input for session %s: %s", session_id, exc)
+            await websocket_manager.send_message(
+                session_id,
+                {"type": "error", "data": {"message": str(exc)}},
+            )
+
+    async def _handle_ping(self, session_id: str, websocket_manager):
+        await websocket_manager.handle_heartbeat(session_id)
+
+    async def _handle_get_status(self, session_id: str, websocket_manager):
+        session_info = self.session_store.get_session_info(session_id)
+        await websocket_manager.send_message(
+            session_id,
+            {"type": "status", "data": session_info or {"message": "Session not found"}},
+        )

server/services/prompt_channel.py

@@ -0,0 +1,122 @@
+"""PromptChannel implementation backed by WebSocket sessions."""
+
+import asyncio
+from typing import Any, Dict, List, Optional
+
+from entity.messages import MessageBlock
+from server.services.attachment_service import AttachmentService
+from server.services.session_execution import SessionExecutionController
+from utils.attachments import AttachmentStore
+from utils.exceptions import TimeoutError
+from utils.human_prompt import PromptChannel, PromptResult
+from utils.structured_logger import get_server_logger
+
+
+class WebPromptChannel(PromptChannel):
+    """Prompt channel that mediates through the WebSocket session controller."""
+
+    def __init__(
+        self,
+        *,
+        session_id: str,
+        session_controller: SessionExecutionController,
+        websocket_manager: Any,
+        attachment_service: AttachmentService,
+        attachment_store: AttachmentStore,
+    ) -> None:
+        self.session_id = session_id
+        self.session_controller = session_controller
+        self.websocket_manager = websocket_manager
+        self.attachment_service = attachment_service
+        self.attachment_store = attachment_store
+        try:
+            self._loop = asyncio.get_running_loop()
+        except RuntimeError:
+            self._loop = None
+
+    def request(
+        self,
+        *,
+        node_id: str,
+        task: str,
+        inputs: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> PromptResult:
+        preview = inputs or ""
+        payload = {
+            "input": preview,
+            "task_description": task,
+            **(metadata or {}),
+        }
+
+        self.session_controller.set_waiting_for_input(
+            self.session_id,
+            node_id,
+            payload,
+        )
+
+        self._notify_human_prompt(node_id, preview, task)
+
+        try:
+            human_response = self.session_controller.wait_for_human_input(self.session_id)
+        except TimeoutError:
+            raise
+        except Exception as exc:  # pragma: no cover - propagated upstream
+            logger = get_server_logger()
+            logger.log_exception(exc, "Error waiting for human input", node_id=node_id, session_id=self.session_id)
+            raise
+
+        response_text, attachment_ids = self._extract_response(human_response)
+        blocks = self._build_blocks(response_text, attachment_ids)
+        metadata_out = {
+            "attachment_count": len(attachment_ids),
+            "input_size": len(preview),
+        }
+        return PromptResult(text=response_text, blocks=blocks, metadata=metadata_out)
+
+    def _extract_response(self, payload: Any) -> tuple[str, List[str]]:
+        if isinstance(payload, dict):
+            response_text = payload.get("text") or ""
+            attachments = payload.get("attachments") or []
+            return response_text, attachments
+        if payload is None:
+            return "", []
+        return str(payload), []
+
+    def _build_blocks(self, text: str, attachment_ids: List[str]) -> List[MessageBlock]:
+        blocks: List[MessageBlock] = []
+        if text:
+            blocks.append(MessageBlock.text_block(text))
+        if attachment_ids:
+            blocks.extend(
+                self.attachment_service.build_attachment_blocks(
+                    self.session_id,
+                    attachment_ids,
+                    target_store=self.attachment_store,
+                )
+            )
+        if not blocks:
+            blocks.append(MessageBlock.text_block(""))
+        return blocks
+
+    def _notify_human_prompt(self, node_id: str, preview: str, task: str) -> None:
+        message = {
+            "type": "human_input_required",
+            "data": {
+                "node_id": node_id,
+                "input": preview,
+                "task_description": task,
+            },
+        }
+        if self._loop and self._loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(
+                self.websocket_manager.send_message(self.session_id, message),
+                self._loop,
+            )
+            try:
+                future.result()
+            except Exception:
+                # fallback to sync send to surface errors/logging
+                self.websocket_manager.send_message_sync(self.session_id, message)
+        else:
+            self.websocket_manager.send_message_sync(self.session_id, message)

server/services/session_execution.py

@@ -0,0 +1,148 @@
+"""Human input coordination for workflow sessions."""
+
+import concurrent.futures
+import logging
+import time
+from concurrent.futures import Future
+from typing import Any, Dict, Optional
+
+from utils.exceptions import ValidationError, TimeoutError as CustomTimeoutError, WorkflowCancelledError
+from utils.structured_logger import LogType, get_server_logger
+
+from .session_store import SessionStatus, WorkflowSessionStore
+
+
+class SessionExecutionController:
+    """Handles blocking wait/provide cycles for human input."""
+
+    def __init__(self, store: WorkflowSessionStore) -> None:
+        self.store = store
+        self.logger = logging.getLogger(__name__)
+
+    def set_waiting_for_input(self, session_id: str, node_id: str, input_data: Dict[str, Any]) -> None:
+        session = self.store.get_session(session_id)
+        if not session:
+            raise ValidationError("Session not found", details={"session_id": session_id})
+        session.waiting_for_input = True
+        session.current_node_id = node_id
+        session.pending_input_data = input_data
+        session.status = SessionStatus.WAITING_FOR_INPUT
+        session.human_input_future = Future()
+        session.human_input_value = None
+        self.logger.info("Session %s waiting for input at node %s", session_id, node_id)
+
+    def wait_for_human_input(self, session_id: str, timeout: float = 1800.0) -> Any:
+        session = self.store.get_session(session_id)
+        if not session:
+            logger = get_server_logger()
+            logger.warning(
+                "Session %s not found when waiting for human input", session_id, log_type=LogType.WORKFLOW
+            )
+            raise ValidationError("Session not found", details={"session_id": session_id})
+
+        future: Optional[Future] = session.human_input_future
+        if not session.waiting_for_input or future is None:
+            logger = get_server_logger()
+            logger.warning(
+                "Session %s is not waiting for input", session_id, log_type=LogType.WORKFLOW
+            )
+            raise ValidationError(
+                "Session is not waiting for input",
+                details={"session_id": session_id, "waiting_for_input": session.waiting_for_input},
+            )
+
+        start_time = time.time()
+        poll_interval = 1.0
+        try:
+            while True:
+                if session.cancel_event.is_set():
+                    raise WorkflowCancelledError("Workflow execution cancelled", workflow_id=session_id)
+
+                elapsed = time.time() - start_time
+                remaining = timeout - elapsed
+                if remaining <= 0:
+                    raise concurrent.futures.TimeoutError()
+
+                wait_time = min(poll_interval, remaining)
+                try:
+                    result = future.result(timeout=wait_time)
+                    logger = get_server_logger()
+                    input_length = 0
+                    if isinstance(result, dict):
+                        input_length = len(result.get("text") or "")
+                    elif result is not None:
+                        input_length = len(str(result))
+                    logger.info(
+                        "Human input received",
+                        log_type=LogType.WORKFLOW,
+                        session_id=session_id,
+                        input_length=input_length,
+                    )
+                    return result
+                except concurrent.futures.TimeoutError:
+                    continue
+        except concurrent.futures.TimeoutError:
+            self.logger.warning("Session %s human input timeout", session_id)
+            logger = get_server_logger()
+            logger.warning(
+                "Human input timeout",
+                log_type=LogType.WORKFLOW,
+                session_id=session_id,
+                timeout_duration=timeout,
+            )
+            raise CustomTimeoutError("Input timeout", operation="wait_for_human_input", timeout_duration=timeout)
+        finally:
+            session.waiting_for_input = False
+            session.current_node_id = None
+            session.pending_input_data = None
+            session.human_input_future = None
+
+    def provide_human_input(self, session_id: str, user_input: Any) -> None:
+        session = self.store.get_session(session_id)
+        if not session:
+            logger = get_server_logger()
+            logger.warning("Session %s not found when providing human input", session_id)
+            raise ValidationError(
+                "Session not found", details={"session_id": session_id, "input_provided": user_input is not None}
+            )
+
+        future: Optional[Future] = session.human_input_future
+        if not session.waiting_for_input or future is None:
+            logger = get_server_logger()
+            logger.warning("Session %s is not waiting for input when providing data", session_id)
+            raise ValidationError(
+                "Session is not waiting for input",
+                details={"session_id": session_id, "waiting_for_input": session.waiting_for_input},
+            )
+
+        future.set_result(user_input)
+        session.waiting_for_input = False
+        length = 0
+        if isinstance(user_input, dict):
+            length = len(user_input.get("text") or "")
+        elif user_input is not None:
+            length = len(str(user_input))
+        logger = get_server_logger()
+        logger.info(
+            "Human input provided",
+            log_type=LogType.WORKFLOW,
+            session_id=session_id,
+            input_length=length,
+        )
+
+    def cleanup_session(self, session_id: str) -> None:
+        session = self.store.get_session(session_id)
+        if not session:
+            return
+        future: Optional[Future] = session.human_input_future
+        if future and not future.done():
+            future.cancel()
+        promise = session.input_promise
+        if promise and not promise.done():
+            promise.cancel()
+        session.waiting_for_input = False
+        session.current_node_id = None
+        session.pending_input_data = None
+        session.human_input_future = None
+        session.human_input_value = None
+        self.logger.info("Session %s cleaned from execution controller", session_id)

server/services/session_store.py

@@ -0,0 +1,158 @@
+"""Session persistence primitives for workflow runs."""
+
+import logging
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from threading import Event
+from typing import Any, Dict, Optional
+
+from server.services.artifact_events import ArtifactEventQueue
+
+
+class SessionStatus(Enum):
+    """Lifecycle states for a workflow session."""
+
+    IDLE = "idle"
+    RUNNING = "running"
+    WAITING_FOR_INPUT = "waiting_for_input"
+    COMPLETED = "completed"
+    ERROR = "error"
+    CANCELLED = "cancelled"
+
+
+@dataclass
+class WorkflowSession:
+    """Mutable record describing a workflow session."""
+
+    session_id: str
+    yaml_file: str
+    task_prompt: str
+    task_attachments: list[str] = field(default_factory=list)
+    status: SessionStatus = SessionStatus.IDLE
+    created_at: float = field(default_factory=lambda: time.time())
+    updated_at: float = field(default_factory=lambda: time.time())
+
+    # Execution metadata
+    executor: Optional[Any] = None
+    graph: Optional[Any] = None
+    current_node_id: Optional[str] = None
+
+    # Human input tracking
+    waiting_for_input: bool = False
+    input_promise: Optional[Any] = None
+    pending_input_data: Optional[Dict[str, Any]] = None
+    human_input_future: Optional[Any] = None
+    human_input_value: Optional[str] = None
+
+    # Results + errors
+    results: Dict[str, Any] = field(default_factory=dict)
+    error_message: Optional[str] = None
+
+    # Artifact streaming
+    artifact_queue: ArtifactEventQueue = field(default_factory=ArtifactEventQueue)
+
+    # Cancellation tracking
+    cancel_event: Event = field(default_factory=Event)
+    cancel_reason: Optional[str] = None
+
+    # Message buffer for reconnection replay
+    message_buffer: list = field(default_factory=list)
+
+    MAX_BUFFER_SIZE: int = 1000
+
+    def append_message(self, message: Dict[str, Any]) -> None:
+        if len(self.message_buffer) >= self.MAX_BUFFER_SIZE:
+            self.message_buffer.pop(0)
+        self.message_buffer.append(message)
+
+
+class WorkflowSessionStore:
+    """In-memory registry that tracks workflow session metadata."""
+
+    def __init__(self) -> None:
+        self._sessions: Dict[str, WorkflowSession] = {}
+        self.logger = logging.getLogger(__name__)
+
+    def create_session(
+        self,
+        *,
+        yaml_file: str,
+        task_prompt: str,
+        session_id: str,
+        attachments: Optional[list[str]] = None,
+    ) -> WorkflowSession:
+        session = WorkflowSession(
+            session_id=session_id,
+            yaml_file=yaml_file,
+            task_prompt=task_prompt,
+            task_attachments=list(attachments or []),
+        )
+        self._sessions[session_id] = session
+        self.logger.info("Created session %s for workflow %s", session_id, yaml_file)
+        return session
+
+    def get_session(self, session_id: str) -> Optional[WorkflowSession]:
+        return self._sessions.get(session_id)
+
+    def has_session(self, session_id: str) -> bool:
+        return session_id in self._sessions
+
+    def update_session_status(self, session_id: str, status: SessionStatus, **kwargs: Any) -> None:
+        session = self._sessions.get(session_id)
+        if not session:
+            return
+        session.status = status
+        session.updated_at = time.time()
+        for key, value in kwargs.items():
+            if hasattr(session, key):
+                setattr(session, key, value)
+        self.logger.info("Updated session %s status to %s", session_id, status.value)
+
+    def set_session_error(self, session_id: str, error_message: str) -> None:
+        self.update_session_status(session_id, SessionStatus.ERROR, error_message=error_message)
+
+    def complete_session(self, session_id: str, results: Dict[str, Any]) -> None:
+        self.update_session_status(session_id, SessionStatus.COMPLETED, results=results)
+
+    def pop_session(self, session_id: str) -> Optional[WorkflowSession]:
+        return self._sessions.pop(session_id, None)
+
+    def get_session_info(self, session_id: str) -> Optional[Dict[str, Any]]:
+        session = self._sessions.get(session_id)
+        if not session:
+            return None
+        return {
+            "session_id": session.session_id,
+            "yaml_file": session.yaml_file,
+            "status": session.status.value,
+            "created_at": session.created_at,
+            "updated_at": session.updated_at,
+            "current_node_id": session.current_node_id,
+            "waiting_for_input": session.waiting_for_input,
+            "error_message": session.error_message,
+        }
+
+    def list_sessions(self) -> Dict[str, Dict[str, Any]]:
+        return {session_id: self.get_session_info(session_id) for session_id in self._sessions.keys()}
+
+    def get_artifact_queue(self, session_id: str) -> Optional[ArtifactEventQueue]:
+        session = self._sessions.get(session_id)
+        return session.artifact_queue if session else None
+
+    def get_session_snapshot(self, session_id: str) -> Optional[Dict[str, Any]]:
+        session = self._sessions.get(session_id)
+        if not session:
+            return None
+        return {
+            "session_id": session.session_id,
+            "yaml_file": session.yaml_file,
+            "task_prompt": session.task_prompt,
+            "status": session.status.value,
+            "current_node_id": session.current_node_id,
+            "created_at": session.created_at,
+            "updated_at": session.updated_at,
+            "waiting_for_input": session.waiting_for_input,
+            "error_message": session.error_message,
+            "message_count": len(session.message_buffer),
+        }

server/services/vuegraphs_storage.py

@@ -0,0 +1,62 @@
+"""
+SQLite-backed storage helpers for Vue graph editor payloads.
+"""
+
+import os
+import sqlite3
+from pathlib import Path
+from typing import Optional
+
+
+_INITIALIZED_PATHS: set[Path] = set()
+
+
+def _get_db_path() -> Path:
+    """Resolve the SQLite database path, allowing overrides via env."""
+    return Path(os.getenv("VUEGRAPHS_DB_PATH", "data/vuegraphs.db"))
+
+
+def _ensure_db_initialized() -> Path:
+    """Create the SQLite database and table if they do not already exist."""
+    db_path = _get_db_path()
+    if db_path not in _INITIALIZED_PATHS or not db_path.exists():
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        with sqlite3.connect(db_path) as connection:
+            connection.execute(
+                """
+                CREATE TABLE IF NOT EXISTS vuegraphs (
+                    filename TEXT PRIMARY KEY,
+                    content TEXT NOT NULL
+                )
+                """
+            )
+            connection.commit()
+        _INITIALIZED_PATHS.add(db_path)
+    return db_path
+
+
+def save_vuegraph_content(filename: str, content: str) -> None:
+    """Insert or update the stored content for the provided filename."""
+    db_path = _ensure_db_initialized()
+    with sqlite3.connect(db_path) as connection:
+        connection.execute(
+            """
+            INSERT INTO vuegraphs (filename, content)
+            VALUES (?, ?)
+            ON CONFLICT(filename) DO UPDATE SET content=excluded.content
+            """,
+            (filename, content),
+        )
+        connection.commit()
+
+
+def fetch_vuegraph_content(filename: str) -> Optional[str]:
+    """Return the stored content for filename, or None when absent."""
+    db_path = _ensure_db_initialized()
+    with sqlite3.connect(db_path) as connection:
+        cursor = connection.execute(
+            "SELECT content FROM vuegraphs WHERE filename = ?",
+            (filename,),
+        )
+        row = cursor.fetchone()
+        return row[0] if row else None

server/services/websocket_executor.py

@@ -0,0 +1,72 @@
+"""GraphExecutor variant that reports results over WebSocket."""
+
+import asyncio
+from typing import List
+
+from utils.logger import WorkflowLogger
+from workflow.graph import GraphExecutor
+from workflow.graph_context import GraphContext
+
+from server.services.attachment_service import AttachmentService
+from server.services.artifact_dispatcher import ArtifactDispatcher
+from server.services.prompt_channel import WebPromptChannel
+from server.services.session_store import WorkflowSessionStore
+from server.services.session_execution import SessionExecutionController
+from workflow.hooks.workspace_artifact import WorkspaceArtifact, WorkspaceArtifactHook
+
+
+class WebSocketGraphExecutor(GraphExecutor):
+    """GraphExecutor subclass that emits events via WebSocket."""
+
+    def __init__(
+        self,
+        graph: GraphContext,
+        session_id: str,
+        session_controller: SessionExecutionController,
+        attachment_service: AttachmentService,
+        websocket_manager,
+        session_store: WorkflowSessionStore,
+        cancel_event=None,
+    ):
+        self.session_id = session_id
+        self.session_controller = session_controller
+        self.attachment_service = attachment_service
+        self.websocket_manager = websocket_manager
+        self.session_store = session_store
+        self.results = {}
+        self.artifact_dispatcher = ArtifactDispatcher(session_id, session_store, websocket_manager)
+
+        def hook_factory(runtime_context):
+            prompt_channel = WebPromptChannel(
+                session_id=session_id,
+                session_controller=session_controller,
+                websocket_manager=websocket_manager,
+                attachment_service=attachment_service,
+                attachment_store=runtime_context.attachment_store,
+            )
+            return WorkspaceArtifactHook(
+                attachment_store=runtime_context.attachment_store,
+                emit_callback=self._handle_workspace_artifacts,
+                prompt_channel=prompt_channel,
+            )
+
+        super().__init__(
+            graph,
+            session_id=session_id,
+            workspace_hook_factory=hook_factory,
+            cancel_event=cancel_event,
+        )
+
+    def _create_logger(self) -> WorkflowLogger:
+        from server.services.websocket_logger import WebSocketLogger
+
+        return WebSocketLogger(self.websocket_manager, self.session_id, self.graph.name, self.graph.log_level)
+
+    async def execute_graph_async(self, task_prompt):
+        await asyncio.get_event_loop().run_in_executor(None, self._execute, task_prompt)
+
+    def get_results(self):
+        return self.outputs
+
+    def _handle_workspace_artifacts(self, artifacts: List[WorkspaceArtifact]) -> None:
+        self.artifact_dispatcher.emit_workspace_artifacts(artifacts)

server/services/websocket_logger.py

@@ -0,0 +1,30 @@
+import asyncio
+from typing import Any, Dict
+
+from entity.enums import LogLevel, EventType
+from utils.logger import WorkflowLogger, LogEntry
+from utils.structured_logger import get_workflow_logger
+
+
+class WebSocketLogger(WorkflowLogger):
+    """Workflow logger that also pushes entries via WebSocket."""
+
+    def __init__(self, websocket_manager, session_id: str, workflow_id: str = None, log_level: LogLevel = LogLevel.DEBUG):
+        super().__init__(workflow_id, log_level, log_to_console=False)
+        self.websocket_manager = websocket_manager
+        self.session_id = session_id
+
+    def add_log(self, level: LogLevel, message: str = None, node_id: str = None,
+                event_type: EventType = None, details: Dict[str, Any] = None,
+                duration: float = None) -> LogEntry | None:
+        log_entry = super().add_log(level, message, node_id, event_type, details, duration)
+        if not log_entry:
+            return None
+
+        # Send the message using the sync method which handles event loop properly
+        self.websocket_manager.send_message_sync(self.session_id, {
+            "type": "log",
+            "data": log_entry.to_dict()
+        })
+        
+        return log_entry

server/services/websocket_manager.py

@@ -0,0 +1,250 @@
+"""WebSocket connection manager used by FastAPI app."""
+
+import asyncio
+import json
+import logging
+import time
+import traceback
+import uuid
+from typing import Any, Dict, Optional
+
+from fastapi import WebSocket
+
+from server.services.message_handler import MessageHandler
+from server.services.attachment_service import AttachmentService
+from server.services.session_execution import SessionExecutionController
+from server.services.session_store import WorkflowSessionStore, SessionStatus
+from server.services.workflow_run_service import WorkflowRunService
+
+
+def _json_default(value):
+    to_dict = getattr(value, "to_dict", None)
+    if callable(to_dict):
+        try:
+            return to_dict()
+        except Exception:
+            pass
+    if hasattr(value, "__dict__"):
+        try:
+            return vars(value)
+        except Exception:
+            pass
+    return str(value)
+
+
+def _encode_ws_message(message: Any) -> str:
+    if isinstance(message, str):
+        return message
+    return json.dumps(message, default=_json_default)
+
+
+class WebSocketManager:
+    SESSION_TTL_SECONDS = 24 * 60 * 60  # 24 hours
+
+    def __init__(
+        self,
+        *,
+        session_store: WorkflowSessionStore | None = None,
+        session_controller: SessionExecutionController | None = None,
+        attachment_service: AttachmentService | None = None,
+        workflow_run_service: WorkflowRunService | None = None,
+    ):
+        self.active_connections: Dict[str, WebSocket] = {}
+        self.connection_timestamps: Dict[str, float] = {}
+        self._owner_loop: Optional[asyncio.AbstractEventLoop] = None
+        self._gc_task: Optional[asyncio.Task] = None
+        self.session_store = session_store or WorkflowSessionStore()
+        self.session_controller = session_controller or SessionExecutionController(self.session_store)
+        self.attachment_service = attachment_service or AttachmentService()
+        self.workflow_run_service = workflow_run_service or WorkflowRunService(
+            self.session_store,
+            self.session_controller,
+            self.attachment_service,
+        )
+        self.message_handler = MessageHandler(
+            self.session_store,
+            self.session_controller,
+            self.workflow_run_service,
+        )
+
+    async def connect(self, websocket: WebSocket, session_id: Optional[str] = None) -> str:
+        await websocket.accept()
+        # Capture the event loop that owns the WebSocket connections so that
+        # worker threads can safely schedule sends via run_coroutine_threadsafe.
+        if self._owner_loop is None:
+            self._owner_loop = asyncio.get_running_loop()
+
+        # --- Reconnect to existing session ---
+        if session_id and self.session_store.has_session(session_id):
+            # If an old WebSocket is still tied to this session, close it first
+            if session_id in self.active_connections:
+                old_ws = self.active_connections[session_id]
+                try:
+                    await old_ws.close(code=1000, reason="Replaced by new connection")
+                except Exception:
+                    pass
+
+            self.active_connections[session_id] = websocket
+            self.connection_timestamps[session_id] = time.time()
+            logging.info("WebSocket reconnected to existing session: %s", session_id)
+
+            # Always start the GC loop (idempotent)
+            self._start_gc()
+
+            # Send connection confirmation
+            await self._send_raw(
+                session_id,
+                {"type": "connection", "data": {"session_id": session_id, "status": "connected"}},
+            )
+
+            # Replay all buffered messages (snapshot to avoid including messages
+            # that arrive during replay)
+            session = self.session_store.get_session(session_id)
+            if session:
+                messages_to_replay = list(session.message_buffer)
+                for msg in messages_to_replay:
+                    await self._send_raw(session_id, msg)
+
+            # Send session state snapshot
+            snapshot = self.session_store.get_session_snapshot(session_id)
+            if snapshot:
+                await self._send_raw(session_id, {"type": "session_resumed", "data": snapshot})
+
+            return session_id
+
+        # --- New connection ---
+        if not session_id:
+            session_id = str(uuid.uuid4())
+        self.active_connections[session_id] = websocket
+        self.connection_timestamps[session_id] = time.time()
+        logging.info("WebSocket connected: %s", session_id)
+
+        # Always start the GC loop (idempotent)
+        self._start_gc()
+
+        await self.send_message(
+            session_id,
+            {
+                "type": "connection",
+                "data": {"session_id": session_id, "status": "connected"},
+            },
+        )
+        return session_id
+
+    def disconnect(self, session_id: str) -> None:
+        if session_id in self.active_connections:
+            del self.active_connections[session_id]
+        if session_id in self.connection_timestamps:
+            del self.connection_timestamps[session_id]
+        logging.info("WebSocket disconnected (session preserved): %s", session_id)
+
+    async def send_message(self, session_id: str, message: Dict[str, Any]) -> None:
+        # Buffer business messages for reconnection replay (exclude transport messages)
+        if message.get("type") not in ("connection", "pong"):
+            session = self.session_store.get_session(session_id)
+            if session:
+                session.append_message(message)
+
+        if session_id in self.active_connections:
+            websocket = self.active_connections[session_id]
+            try:
+                await websocket.send_text(_encode_ws_message(message))
+            except Exception as exc:
+                traceback.print_exc()
+                logging.error("Failed to send message to %s: %s", session_id, exc)
+
+    async def _send_raw(self, session_id: str, message: Dict[str, Any]) -> None:
+        """Send a message without buffering. Used for replay and connection management."""
+        if session_id in self.active_connections:
+            websocket = self.active_connections[session_id]
+            try:
+                await websocket.send_text(_encode_ws_message(message))
+            except Exception as exc:
+                traceback.print_exc()
+                logging.error("Failed to send raw message to %s: %s", session_id, exc)
+
+    def send_message_sync(self, session_id: str, message: Dict[str, Any]) -> None:
+        """Send a WebSocket message from any thread (including worker threads).
+
+        WebSocket objects are bound to the event loop that created them (the main
+        uvicorn loop).  Previous code called ``asyncio.run()`` from worker threads
+        which spins up a *new* event loop, causing ``RuntimeError: … attached to a
+        different loop`` or silent delivery failures.
+
+        The fix: always schedule the coroutine on the loop that owns the sockets
+        via ``asyncio.run_coroutine_threadsafe`` and wait for the result with a
+        short timeout so the caller knows if delivery failed.
+        """
+        loop = self._owner_loop
+        if loop is None or loop.is_closed():
+            logging.warning(
+                "Cannot send sync message to %s: owner event loop unavailable",
+                session_id,
+            )
+            return
+
+        future = asyncio.run_coroutine_threadsafe(
+            self.send_message(session_id, message), loop
+        )
+        try:
+            future.result(timeout=10)
+        except TimeoutError:
+            logging.warning(
+                "Timed out sending sync WS message to %s", session_id
+            )
+        except Exception as exc:
+            logging.error(
+                "Error sending sync WS message to %s: %s", session_id, exc
+            )
+
+    async def broadcast(self, message: Dict[str, Any]) -> None:
+        for session_id in list(self.active_connections.keys()):
+            await self.send_message(session_id, message)
+
+    async def handle_heartbeat(self, session_id: str) -> None:
+        if session_id in self.active_connections:
+            await self.send_message(
+                session_id,
+                {"type": "pong", "data": {"timestamp": time.time()}},
+            )
+        else:
+            logging.warning("Heartbeat request from disconnected session: %s", session_id)
+
+    async def handle_message(self, session_id: str, message: str) -> None:
+        try:
+            data = json.loads(message)
+            await self.message_handler.handle_message(session_id, data, self)
+        except json.JSONDecodeError:
+            await self.send_message(
+                session_id,
+                {"type": "error", "data": {"message": "Invalid JSON format"}},
+            )
+        except Exception as exc:
+            logging.error("Error handling message from %s: %s", session_id, exc)
+            await self.send_message(
+                session_id,
+                {"type": "error", "data": {"message": str(exc)}},
+            )
+
+    def _start_gc(self) -> None:
+        """Start the background GC task if not already running."""
+        if self._gc_task is not None and not self._gc_task.done():
+            return
+        loop = asyncio.get_running_loop()
+        self._gc_task = loop.create_task(self._gc_loop())
+
+    async def _gc_loop(self) -> None:
+        """Periodically clean up terminal sessions older than TTL."""
+        TERMINAL = {SessionStatus.COMPLETED, SessionStatus.ERROR, SessionStatus.CANCELLED}
+        while True:
+            await asyncio.sleep(3600)  # run every hour
+            now = time.time()
+            to_remove = []
+            for sid, session in self.session_store._sessions.items():
+                if session.status in TERMINAL:
+                    if now - session.updated_at > self.SESSION_TTL_SECONDS:
+                        to_remove.append(sid)
+            for sid in to_remove:
+                self.session_store.pop_session(sid)
+                self.attachment_service.cleanup_session(sid)
+                logging.info("GC: removed expired session %s", sid)

server/services/workflow_run_service.py

@@ -0,0 +1,293 @@
+"""Service responsible for executing workflows for WebSocket sessions."""
+
+import logging
+from pathlib import Path
+from typing import List, Optional, Union
+
+from check.check import load_config
+from entity.graph_config import GraphConfig
+from entity.messages import Message
+from entity.enums import LogLevel
+from utils.exceptions import ValidationError, WorkflowCancelledError
+from utils.structured_logger import get_server_logger, LogType
+from utils.task_input import TaskInputBuilder
+from workflow.graph_context import GraphContext
+
+from server.services.attachment_service import AttachmentService
+from server.services.session_execution import SessionExecutionController
+from server.services.session_store import SessionStatus, WorkflowSessionStore
+from server.services.websocket_executor import WebSocketGraphExecutor
+from server.services.workflow_storage import validate_workflow_filename
+from server.settings import WARE_HOUSE_DIR, YAML_DIR
+
+
+class WorkflowRunService:
+    def __init__(
+        self,
+        session_store: WorkflowSessionStore,
+        session_controller: SessionExecutionController,
+        attachment_service: AttachmentService,
+    ) -> None:
+        self.session_store = session_store
+        self.session_controller = session_controller
+        self.attachment_service = attachment_service
+        self.logger = logging.getLogger(__name__)
+
+    def request_cancel(self, session_id: str, *, reason: Optional[str] = None) -> bool:
+        session = self.session_store.get_session(session_id)
+        if not session:
+            return False
+
+        cancel_message = reason or "Cancellation requested"
+        session.cancel_reason = cancel_message
+        if not session.cancel_event.is_set():
+            session.cancel_event.set()
+            self.logger.info("Cancellation requested for session %s", session_id)
+
+        if session.executor:
+            try:
+                session.executor.request_cancel(cancel_message)
+            except Exception as exc:
+                self.logger.warning("Failed to propagate cancellation to executor for %s: %s", session_id, exc)
+
+        self.session_store.update_session_status(session_id, SessionStatus.CANCELLED, error_message=cancel_message)
+        return True
+
+    async def start_workflow(
+        self,
+        session_id: str,
+        yaml_file: str,
+        task_prompt: str,
+        websocket_manager,
+        *,
+        attachments: Optional[List[str]] = None,
+        log_level: Optional[LogLevel] = None,
+    ) -> None:
+        normalized_yaml_name = (yaml_file or "").strip()
+        try:
+            yaml_path = self._resolve_yaml_path(normalized_yaml_name)
+            normalized_yaml_name = yaml_path.name
+
+            attachments = attachments or []
+            if (not task_prompt or not task_prompt.strip()) and not attachments:
+                raise ValidationError(
+                    "Task prompt cannot be empty",
+                    details={"task_prompt_provided": bool(task_prompt)},
+                )
+
+            self.attachment_service.prepare_session_workspace(session_id)
+            self.session_store.create_session(
+                yaml_file=normalized_yaml_name,
+                task_prompt=task_prompt,
+                session_id=session_id,
+                attachments=attachments,
+            )
+            self.session_store.update_session_status(session_id, SessionStatus.RUNNING)
+
+            await websocket_manager.send_message(
+                session_id,
+                {
+                    "type": "workflow_started",
+                    "data": {"yaml_file": normalized_yaml_name, "task_prompt": task_prompt},
+                },
+            )
+
+            await self._execute_workflow_async(
+                session_id,
+                yaml_path,
+                task_prompt,
+                websocket_manager,
+                attachments,
+                log_level,
+            )
+        except ValidationError as exc:
+            self.logger.error(str(exc))
+            logger = get_server_logger()
+            logger.error(
+                "Workflow validation error",
+                log_type=LogType.WORKFLOW,
+                session_id=session_id,
+                yaml_file=normalized_yaml_name,
+                validation_details=getattr(exc, "details", None),
+            )
+            self.session_store.set_session_error(session_id, str(exc))
+            await websocket_manager.send_message(
+                session_id,
+                {"type": "error", "data": {"message": str(exc)}},
+            )
+        except Exception as exc:
+            self.logger.error(f"Error starting workflow for session {session_id}: {exc}")
+            logger = get_server_logger()
+            logger.log_exception(
+                exc,
+                "Error starting workflow",
+                session_id=session_id,
+                yaml_file=normalized_yaml_name,
+            )
+            self.session_store.set_session_error(session_id, str(exc))
+            await websocket_manager.send_message(
+                session_id,
+                {
+                    "type": "error",
+                    "data": {"message": f"Failed to start workflow: {exc}"},
+                },
+            )
+
+    async def _execute_workflow_async(
+        self,
+        session_id: str,
+        yaml_path: Path,
+        task_prompt: str,
+        websocket_manager,
+        attachments: List[str],
+        log_level: LogLevel,
+    ) -> None:
+        session = self.session_store.get_session(session_id)
+        cancel_event = session.cancel_event if session else None
+        try:
+            design = load_config(yaml_path)
+            graph_config = GraphConfig.from_definition(
+                design.graph,
+                name=f"session_{session_id}",
+                output_root=WARE_HOUSE_DIR,
+                source_path=str(yaml_path),
+                vars=design.vars,
+            )
+            if log_level:
+                graph_config.log_level = log_level
+                graph_config.definition.log_level = log_level
+            graph_context = GraphContext(config=graph_config)
+
+            executor = WebSocketGraphExecutor(
+                graph_context,
+                session_id,
+                self.session_controller,
+                self.attachment_service,
+                websocket_manager,
+                self.session_store,
+                cancel_event=cancel_event,
+            )
+
+            if session:
+                session.graph = graph_context
+                session.executor = executor
+                if session.cancel_event.is_set():
+                    executor.request_cancel(session.cancel_reason or "Cancellation requested")
+
+            task_input = self._build_initial_task_input(
+                session_id,
+                graph_context,
+                task_prompt,
+                attachments,
+                executor.attachment_store,
+            )
+
+            await executor.execute_graph_async(task_input)
+
+            # If cancellation was requested during execution but not raised inside threads,
+            # treat the run as cancelled to avoid conflicting status.
+            if cancel_event and cancel_event.is_set():
+                reason = session.cancel_reason if session else "Cancellation requested"
+                raise WorkflowCancelledError(reason, workflow_id=graph_context.name)
+
+            results = executor.get_results()
+            self.session_store.complete_session(session_id, results)
+
+            await websocket_manager.send_message(
+                session_id,
+                {
+                    "type": "workflow_completed",
+                    "data": {
+                        "results": results,
+                        "summary": graph_context.final_message(),
+                        "token_usage": executor.token_tracker.get_token_usage(),
+                    },
+                },
+            )
+
+            logger = get_server_logger()
+            logger.info(
+                "Workflow execution completed successfully",
+                log_type=LogType.WORKFLOW,
+                session_id=session_id,
+                yaml_path=str(yaml_path),
+                result_count=len(results) if isinstance(results, dict) else 0,
+            )
+        except WorkflowCancelledError as exc:
+            reason = str(exc)
+            self.session_store.update_session_status(session_id, SessionStatus.CANCELLED, error_message=reason)
+            await websocket_manager.send_message(
+                session_id,
+                {
+                    "type": "workflow_cancelled",
+                    "data": {"message": reason},
+                },
+            )
+            logger = get_server_logger()
+            logger.info(
+                "Workflow execution cancelled",
+                log_type=LogType.WORKFLOW,
+                session_id=session_id,
+                yaml_path=str(yaml_path),
+                cancellation_reason=reason,
+            )
+        except ValidationError as exc:
+            self.session_store.set_session_error(session_id, str(exc))
+            await websocket_manager.send_message(
+                session_id,
+                {"type": "error", "data": {"message": str(exc)}},
+            )
+            logger = get_server_logger()
+            logger.error(
+                "Workflow validation error",
+                log_type=LogType.WORKFLOW,
+                session_id=session_id,
+                yaml_path=str(yaml_path),
+                validation_details=getattr(exc, "details", None),
+            )
+        except Exception as exc:
+            self.session_store.set_session_error(session_id, str(exc))
+            await websocket_manager.send_message(
+                session_id,
+                {"type": "error", "data": {"message": f"Workflow execution error: {exc}"}},
+            )
+            logger = get_server_logger()
+            logger.log_exception(
+                exc,
+                f"Error executing workflow for session {session_id}",
+                session_id=session_id,
+                yaml_path=str(yaml_path),
+            )
+        finally:
+            session_ref = self.session_store.get_session(session_id)
+            if session_ref:
+                session_ref.executor = None
+                session_ref.graph = None
+            self.session_controller.cleanup_session(session_id)
+
+    def _build_initial_task_input(
+        self,
+        session_id: str,
+        graph_context: GraphContext,
+        prompt: str,
+        attachment_ids: List[str],
+        store,
+    ) -> Union[List[Message], str]:
+        if not attachment_ids:
+            return prompt
+
+        blocks = self.attachment_service.build_attachment_blocks(
+            session_id,
+            attachment_ids,
+            target_store=store,
+        )
+        return TaskInputBuilder(store).build_from_blocks(prompt, blocks)
+
+    def _resolve_yaml_path(self, yaml_filename: str) -> Path:
+        """Validate and resolve YAML paths inside the configured directory."""
+
+        safe_name = validate_workflow_filename(yaml_filename, require_yaml_extension=True)
+        yaml_path = YAML_DIR / safe_name
+        if not yaml_path.exists():
+            raise ValidationError("YAML file not found", details={"yaml_file": safe_name})
+        return yaml_path

server/services/workflow_storage.py

@@ -0,0 +1,207 @@
+"""Utilities for validating and persisting workflow YAML files."""
+
+import re
+from pathlib import Path
+from typing import Any, Tuple
+
+import yaml
+
+from check.check import check_config
+from utils.exceptions import (
+    ResourceConflictError,
+    ResourceNotFoundError,
+    SecurityError,
+    ValidationError,
+    WorkflowExecutionError,
+)
+from utils.structured_logger import get_server_logger, LogType
+
+
+def _update_workflow_id(content: str, workflow_id: str) -> str:
+    pattern = re.compile(r"^(id:\\s*).*$", re.MULTILINE)
+    match = pattern.search(content)
+    if match:
+        return pattern.sub(rf"\\1{workflow_id}", content, count=1)
+
+    lines = content.splitlines()
+    insert_index = 0
+    if lines and lines[0].strip() == "---":
+        insert_index = 1
+    lines.insert(insert_index, f"id: {workflow_id}")
+    updated = "\n".join(lines)
+    if content.endswith("\n"):
+        updated += "\n"
+    return updated
+
+
+def validate_workflow_filename(filename: str, *, require_yaml_extension: bool = True) -> str:
+    """Sanitize workflow filenames and guard against traversal attempts."""
+
+    value = (filename or "").strip()
+    if not value:
+        raise ValidationError("Filename cannot be empty", field="filename")
+
+    if ".." in value or value.startswith(("/", "\\")):
+        logger = get_server_logger()
+        logger.log_security_event(
+            "PATH_TRAVERSAL_ATTEMPT",
+            f"Suspicious filename detected: {value}",
+            details={"received_filename": value},
+        )
+        raise SecurityError("Invalid filename format", details={"filename": value})
+
+    if not re.match(r"^[a-zA-Z0-9._-]+$", value):
+        raise ValidationError(
+            "Invalid filename: only letters, digits, dots, underscores, and hyphens are allowed",
+            field="filename",
+        )
+
+    if require_yaml_extension and not value.endswith((".yaml", ".yml")):
+        raise ValidationError("Filename must end with .yaml or .yml", field="filename")
+
+    return Path(value).name
+
+
+def validate_workflow_content(filename: str, content: str) -> Tuple[str, Any]:
+    safe_filename = validate_workflow_filename(filename, require_yaml_extension=True)
+
+    try:
+        yaml_content = yaml.safe_load(content)
+        if yaml_content is None:
+            raise ValidationError("YAML content is empty", field="content")
+
+        errors = check_config(yaml_content)
+        if errors:
+            raise ValidationError(f"YAML validation errors:\n{errors}", field="content")
+    except yaml.YAMLError as exc:
+        logger = get_server_logger()
+        logger.warning("Invalid YAML content in upload", details={"error": str(exc)})
+        raise ValidationError(f"Invalid YAML syntax: {exc}", field="content")
+
+    return safe_filename, yaml_content
+
+
+def persist_workflow(
+    safe_filename: str,
+    content: str,
+    yaml_content: Any,
+    *,
+    action: str,
+    directory: Path,
+) -> None:
+    save_path = directory / safe_filename
+    logger = get_server_logger()
+
+    try:
+        save_path.write_text(content, encoding="utf-8")
+    except Exception as exc:
+        logger.log_exception(exc, f"Failed to save workflow file {safe_filename}")
+        raise WorkflowExecutionError(
+            "Failed to save workflow file", details={"filename": safe_filename}
+        )
+
+    logger.info(
+        "Workflow file persisted",
+        log_type=LogType.WORKFLOW,
+        filename=safe_filename,
+        action=action,
+    )
+
+
+def rename_workflow(source_filename: str, target_filename: str, *, directory: Path) -> None:
+    source_safe = validate_workflow_filename(source_filename, require_yaml_extension=True)
+    target_safe = validate_workflow_filename(target_filename, require_yaml_extension=True)
+
+    if source_safe == target_safe:
+        raise ValidationError("Source and target filenames must be different", field="new_filename")
+
+    source_path = directory / source_safe
+    target_path = directory / target_safe
+
+    if not source_path.exists() or not source_path.is_file():
+        raise ResourceNotFoundError(
+            "Workflow file not found",
+            resource_type="workflow",
+            resource_id=source_safe,
+        )
+
+    if target_path.exists():
+        raise ResourceConflictError(
+            "Target workflow already exists",
+            resource_type="workflow",
+            resource_id=target_safe,
+        )
+
+    logger = get_server_logger()
+    try:
+        source_path.rename(target_path)
+    except Exception as exc:
+        logger.log_exception(exc, f"Failed to rename workflow file {source_safe} to {target_safe}")
+        raise WorkflowExecutionError(
+            "Failed to rename workflow file",
+            details={"source": source_safe, "target": target_safe},
+        )
+
+    try:
+        new_workflow_id = Path(target_safe).stem
+        content = target_path.read_text(encoding="utf-8")
+        updated = _update_workflow_id(content, new_workflow_id)
+        if updated != content:
+            target_path.write_text(updated, encoding="utf-8")
+    except Exception as exc:
+        logger.log_exception(exc, f"Failed to update workflow id after rename to {target_safe}")
+        raise WorkflowExecutionError(
+            "Failed to update workflow id after rename",
+            details={"target": target_safe},
+        )
+
+    logger.info(
+        "Workflow file renamed",
+        log_type=LogType.WORKFLOW,
+        source=source_safe,
+        target=target_safe,
+        action="rename",
+    )
+
+
+def copy_workflow(source_filename: str, target_filename: str, *, directory: Path) -> None:
+    source_safe = validate_workflow_filename(source_filename, require_yaml_extension=True)
+    target_safe = validate_workflow_filename(target_filename, require_yaml_extension=True)
+
+    if source_safe == target_safe:
+        raise ValidationError("Source and target filenames must be different", field="new_filename")
+
+    source_path = directory / source_safe
+    target_path = directory / target_safe
+
+    if not source_path.exists() or not source_path.is_file():
+        raise ResourceNotFoundError(
+            "Workflow file not found",
+            resource_type="workflow",
+            resource_id=source_safe,
+        )
+
+    if target_path.exists():
+        raise ResourceConflictError(
+            "Target workflow already exists",
+            resource_type="workflow",
+            resource_id=target_safe,
+        )
+
+    logger = get_server_logger()
+    try:
+        target_path.write_text(source_path.read_text(encoding="utf-8"), encoding="utf-8")
+    except Exception as exc:
+        logger.log_exception(exc, f"Failed to copy workflow file {source_safe} to {target_safe}")
+        raise WorkflowExecutionError(
+            "Failed to copy workflow file",
+            details={"source": source_safe, "target": target_safe},
+        )
+
+    logger.info(
+        "Workflow file copied",
+        log_type=LogType.WORKFLOW,
+        source=source_safe,
+        target=target_safe,
+        action="copy",
+    )