chore: import upstream snapshot with attribution

invokeai/app/services/session_processor/session_processor_base.py

@@ -0,0 +1,153 @@
+from abc import ABC, abstractmethod
+from threading import Event
+from typing import Optional, Protocol
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, BaseInvocationOutput
+from invokeai.app.services.invocation_services import InvocationServices
+from invokeai.app.services.session_processor.session_processor_common import SessionProcessorStatus
+from invokeai.app.services.session_queue.session_queue_common import SessionQueueItem
+from invokeai.app.util.profiler import Profiler
+
+
+class SessionRunnerBase(ABC):
+    """
+    Base class for session runner.
+    """
+
+    @abstractmethod
+    def start(self, services: InvocationServices, cancel_event: Event, profiler: Optional[Profiler] = None) -> None:
+        """Starts the session runner.
+
+        Args:
+            services: The invocation services.
+            cancel_event: The cancel event.
+            profiler: The profiler to use for session profiling via cProfile. Omit to disable profiling. Basic session
+                stats will be still be recorded and logged when profiling is disabled.
+        """
+        pass
+
+    @abstractmethod
+    def run(self, queue_item: SessionQueueItem) -> None:
+        """Runs a session.
+
+        Args:
+            queue_item: The session to run.
+        """
+        pass
+
+    @abstractmethod
+    def run_node(self, invocation: BaseInvocation, queue_item: SessionQueueItem) -> None:
+        """Run a single node in the graph.
+
+        Args:
+            invocation: The invocation to run.
+            queue_item: The session queue item.
+        """
+        pass
+
+
+class SessionProcessorBase(ABC):
+    """
+    Base class for session processor.
+
+    The session processor is responsible for executing sessions. It runs a simple polling loop,
+    checking the session queue for new sessions to execute. It must coordinate with the
+    invocation queue to ensure only one session is executing at a time.
+    """
+
+    @abstractmethod
+    def resume(self) -> SessionProcessorStatus:
+        """Starts or resumes the session processor"""
+        pass
+
+    @abstractmethod
+    def pause(self) -> SessionProcessorStatus:
+        """Pauses the session processor"""
+        pass
+
+    @abstractmethod
+    def get_status(self) -> SessionProcessorStatus:
+        """Gets the status of the session processor"""
+        pass
+
+
+class OnBeforeRunNode(Protocol):
+    def __call__(self, invocation: BaseInvocation, queue_item: SessionQueueItem) -> None:
+        """Callback to run before executing a node.
+
+        Args:
+            invocation: The invocation that will be executed.
+            queue_item: The session queue item.
+        """
+        ...
+
+
+class OnAfterRunNode(Protocol):
+    def __call__(self, invocation: BaseInvocation, queue_item: SessionQueueItem, output: BaseInvocationOutput) -> None:
+        """Callback to run before executing a node.
+
+        Args:
+            invocation: The invocation that was executed.
+            queue_item: The session queue item.
+        """
+        ...
+
+
+class OnNodeError(Protocol):
+    def __call__(
+        self,
+        invocation: BaseInvocation,
+        queue_item: SessionQueueItem,
+        error_type: str,
+        error_message: str,
+        error_traceback: str,
+    ) -> None:
+        """Callback to run when a node has an error.
+
+        Args:
+            invocation: The invocation that errored.
+            queue_item: The session queue item.
+            error_type: The type of error, e.g. "ValueError".
+            error_message: The error message, e.g. "Invalid value".
+            error_traceback: The stringified error traceback.
+        """
+        ...
+
+
+class OnBeforeRunSession(Protocol):
+    def __call__(self, queue_item: SessionQueueItem) -> None:
+        """Callback to run before executing a session.
+
+        Args:
+            queue_item: The session queue item.
+        """
+        ...
+
+
+class OnAfterRunSession(Protocol):
+    def __call__(self, queue_item: SessionQueueItem) -> None:
+        """Callback to run after executing a session.
+
+        Args:
+            queue_item: The session queue item.
+        """
+        ...
+
+
+class OnNonFatalProcessorError(Protocol):
+    def __call__(
+        self,
+        queue_item: Optional[SessionQueueItem],
+        error_type: str,
+        error_message: str,
+        error_traceback: str,
+    ) -> None:
+        """Callback to run when a non-fatal error occurs in the processor.
+
+        Args:
+            queue_item: The session queue item, if one was being executed when the error occurred.
+            error_type: The type of error, e.g. "ValueError".
+            error_message: The error message, e.g. "Invalid value".
+            error_traceback: The stringified error traceback.
+        """
+        ...

invokeai/app/services/session_processor/session_processor_common.py

@@ -0,0 +1,33 @@
+from PIL.Image import Image as PILImageType
+from pydantic import BaseModel, Field
+
+from invokeai.backend.util.util import image_to_dataURL
+
+
+class SessionProcessorStatus(BaseModel):
+    is_started: bool = Field(description="Whether the session processor is started")
+    is_processing: bool = Field(description="Whether a session is being processed")
+
+
+class CanceledException(Exception):
+    """Execution canceled by user."""
+
+    pass
+
+
+class ProgressImage(BaseModel):
+    """The progress image sent intermittently during processing"""
+
+    width: int = Field(ge=1, description="The effective width of the image in pixels")
+    height: int = Field(ge=1, description="The effective height of the image in pixels")
+    dataURL: str = Field(description="The image data as a b64 data URL")
+
+    @classmethod
+    def build(cls, image: PILImageType, size: tuple[int, int] | None = None) -> "ProgressImage":
+        """Build a ProgressImage from a PIL image"""
+
+        return cls(
+            width=size[0] if size else image.width,
+            height=size[1] if size else image.height,
+            dataURL=image_to_dataURL(image, image_format="JPEG"),
+        )

invokeai/app/services/session_processor/session_processor_default.py

@@ -0,0 +1,552 @@
+import gc
+import traceback
+from contextlib import suppress
+from threading import BoundedSemaphore, Thread
+from threading import Event as ThreadEvent
+from typing import Optional
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, BaseInvocationOutput
+from invokeai.app.invocations.call_saved_workflow import CallSavedWorkflowInvocation
+from invokeai.app.services.events.events_common import (
+    BatchEnqueuedEvent,
+    FastAPIEvent,
+    QueueClearedEvent,
+    QueueItemStatusChangedEvent,
+    register_events,
+)
+from invokeai.app.services.invocation_stats.invocation_stats_common import GESStatsNotFoundError
+from invokeai.app.services.invoker import Invoker
+from invokeai.app.services.session_processor.session_processor_base import (
+    InvocationServices,
+    OnAfterRunNode,
+    OnAfterRunSession,
+    OnBeforeRunNode,
+    OnBeforeRunSession,
+    OnNodeError,
+    OnNonFatalProcessorError,
+    SessionProcessorBase,
+    SessionRunnerBase,
+)
+from invokeai.app.services.session_processor.session_processor_common import CanceledException, SessionProcessorStatus
+from invokeai.app.services.session_processor.workflow_call_runtime import (
+    WorkflowCallCoordinator,
+    WorkflowCallQueueLifecycle,
+)
+from invokeai.app.services.session_queue.session_queue_common import SessionQueueItem, SessionQueueItemNotFoundError
+from invokeai.app.services.shared.graph import NodeInputError
+from invokeai.app.services.shared.invocation_context import InvocationContextData, build_invocation_context
+from invokeai.app.util.profiler import Profiler
+
+
+class DefaultSessionRunner(SessionRunnerBase):
+    """Processes a single session's invocations."""
+
+    def __init__(
+        self,
+        on_before_run_session_callbacks: Optional[list[OnBeforeRunSession]] = None,
+        on_before_run_node_callbacks: Optional[list[OnBeforeRunNode]] = None,
+        on_after_run_node_callbacks: Optional[list[OnAfterRunNode]] = None,
+        on_node_error_callbacks: Optional[list[OnNodeError]] = None,
+        on_after_run_session_callbacks: Optional[list[OnAfterRunSession]] = None,
+    ):
+        """
+        Args:
+            on_before_run_session_callbacks: Callbacks to run before the session starts.
+            on_before_run_node_callbacks: Callbacks to run before each node starts.
+            on_after_run_node_callbacks: Callbacks to run after each node completes.
+            on_node_error_callbacks: Callbacks to run when a node errors.
+            on_after_run_session_callbacks: Callbacks to run after the session completes.
+        """
+
+        self._on_before_run_session_callbacks = on_before_run_session_callbacks or []
+        self._on_before_run_node_callbacks = on_before_run_node_callbacks or []
+        self._on_after_run_node_callbacks = on_after_run_node_callbacks or []
+        self._on_node_error_callbacks = on_node_error_callbacks or []
+        self._on_after_run_session_callbacks = on_after_run_session_callbacks or []
+        self.workflow_call_coordinator = WorkflowCallCoordinator(self)
+        self.workflow_call_queue_lifecycle = WorkflowCallQueueLifecycle(self)
+
+    def start(self, services: InvocationServices, cancel_event: ThreadEvent, profiler: Optional[Profiler] = None):
+        self._services = services
+        self._cancel_event = cancel_event
+        self._profiler = profiler
+
+    def _is_canceled(self) -> bool:
+        """Check if the cancel event is set. This is also passed to the invocation context builder and called during
+        denoising to check if the session has been canceled."""
+        return self._cancel_event.is_set()
+
+    def _run_session_loop(self, queue_item: SessionQueueItem) -> None:
+        # Loop over invocations until the session is complete or canceled
+        while True:
+            try:
+                invocation = queue_item.session.next()
+            # Anything other than a `NodeInputError` is handled as a processor error
+            except NodeInputError as e:
+                error_type = e.__class__.__name__
+                error_message = str(e)
+                error_traceback = traceback.format_exc()
+                self._on_node_error(
+                    invocation=e.node,
+                    queue_item=queue_item,
+                    error_type=error_type,
+                    error_message=error_message,
+                    error_traceback=error_traceback,
+                )
+                break
+
+            if invocation is None or self._is_canceled():
+                break
+
+            self.run_node(invocation, queue_item)
+
+            # The session is complete if all invocations have been run or there is an error on the session.
+            # At this time, the queue item may be canceled, but the object itself here won't be updated yet. We must
+            # use the cancel event to check if the session is canceled.
+            if (
+                queue_item.session.is_complete()
+                or self._is_canceled()
+                or queue_item.status in ["failed", "canceled", "completed"]
+            ):
+                break
+
+    def run(self, queue_item: SessionQueueItem):
+        # Exceptions raised outside `run_node` are handled by the processor. There is no need to catch them here.
+
+        self._on_before_run_session(queue_item=queue_item)
+        self._run_session_loop(queue_item)
+        self._on_after_run_session(queue_item=queue_item)
+
+    def run_node(self, invocation: BaseInvocation, queue_item: SessionQueueItem):
+        try:
+            # Any unhandled exception in this scope is an invocation error & will fail the graph
+            with self._services.performance_statistics.collect_stats(invocation, queue_item.session_id):
+                self._on_before_run_node(invocation, queue_item)
+
+                data = InvocationContextData(
+                    invocation=invocation,
+                    source_invocation_id=queue_item.session.prepared_source_mapping[invocation.id],
+                    queue_item=queue_item,
+                )
+                context = build_invocation_context(
+                    data=data,
+                    services=self._services,
+                    is_canceled=self._is_canceled,
+                )
+
+                if isinstance(invocation, CallSavedWorkflowInvocation):
+                    workflow_record = invocation.validate_selected_workflow(context)
+                    self.workflow_call_coordinator.begin_workflow_call_boundary(invocation, queue_item, workflow_record)
+                    return
+
+                # Invoke the node
+                output = invocation.invoke_internal(context=context, services=self._services)
+                # Save output and history
+                queue_item.session.complete(invocation.id, output)
+
+                self._on_after_run_node(invocation, queue_item, output)
+
+        except CanceledException:
+            # A CanceledException is raised during the denoising step callback if the cancel event is set. We don't need
+            # to do any handling here, and no error should be set - just pass and the cancellation will be handled
+            # correctly in the next iteration of the session runner loop.
+            #
+            # See the comment in the processor's `_on_queue_item_status_changed()` method for more details on how we
+            # handle cancellation.
+            pass
+        except Exception as e:
+            error_type = e.__class__.__name__
+            error_message = str(e)
+            error_traceback = traceback.format_exc()
+            self._on_node_error(
+                invocation=invocation,
+                queue_item=queue_item,
+                error_type=error_type,
+                error_message=error_message,
+                error_traceback=error_traceback,
+            )
+
+    def _on_before_run_session(self, queue_item: SessionQueueItem) -> None:
+        """Called before a session is run.
+
+        - Start the profiler if profiling is enabled.
+        - Run any callbacks registered for this event.
+        """
+
+        self._services.logger.debug(
+            f"On before run session: queue item {queue_item.item_id}, session {queue_item.session_id}"
+        )
+
+        # If profiling is enabled, start the profiler
+        if self._profiler is not None:
+            self._profiler.start(profile_id=queue_item.session_id)
+
+        for callback in self._on_before_run_session_callbacks:
+            callback(queue_item=queue_item)
+
+    def _on_after_run_session(self, queue_item: SessionQueueItem) -> None:
+        """Called after a session is run.
+
+        - Stop the profiler if profiling is enabled.
+        - Update the queue item's session object in the database.
+        - If not already canceled or failed, complete the queue item.
+        - Log and reset performance statistics.
+        - Run any callbacks registered for this event.
+        """
+
+        self._services.logger.debug(
+            f"On after run session: queue item {queue_item.item_id}, session {queue_item.session_id}"
+        )
+
+        # If we are profiling, stop the profiler and dump the profile & stats
+        if self._profiler is not None:
+            profile_path = self._profiler.stop()
+            stats_path = profile_path.with_suffix(".json")
+            self._services.performance_statistics.dump_stats(
+                graph_execution_state_id=queue_item.session.id, output_path=stats_path
+            )
+
+        try:
+            # Update the queue item with the completed session. If the queue item has been removed from the queue,
+            # we'll get a SessionQueueItemNotFoundError and we can ignore it. This can happen if the queue is cleared
+            # while the session is running.
+            queue_item = self._services.session_queue.set_queue_item_session(queue_item.item_id, queue_item.session)
+
+            # The queue item may have been canceled or failed while the session was running. We should only complete it
+            # if it is not already canceled or failed.
+            if queue_item.status not in ["canceled", "failed"] and queue_item.session.is_complete():
+                queue_item = self._services.session_queue.complete_queue_item(queue_item.item_id)
+
+            # We'll get a GESStatsNotFoundError if we try to log stats for an untracked graph, but in the processor
+            # we don't care about that - suppress the error.
+            with suppress(GESStatsNotFoundError):
+                self._services.performance_statistics.log_stats(queue_item.session.id)
+                self._services.performance_statistics.reset_stats(queue_item.session.id)
+
+            for callback in self._on_after_run_session_callbacks:
+                callback(queue_item=queue_item)
+        except SessionQueueItemNotFoundError:
+            pass
+
+    def _on_before_run_node(self, invocation: BaseInvocation, queue_item: SessionQueueItem):
+        """Called before a node is run.
+
+        - Emits an invocation started event.
+        - Run any callbacks registered for this event.
+        """
+
+        self._services.logger.debug(
+            f"On before run node: queue item {queue_item.item_id}, session {queue_item.session_id}, node {invocation.id} ({invocation.get_type()})"
+        )
+
+        # Send starting event
+        self._services.events.emit_invocation_started(queue_item=queue_item, invocation=invocation)
+
+        for callback in self._on_before_run_node_callbacks:
+            callback(invocation=invocation, queue_item=queue_item)
+
+    def _on_after_run_node(
+        self, invocation: BaseInvocation, queue_item: SessionQueueItem, output: BaseInvocationOutput
+    ):
+        """Called after a node is run.
+
+        - Emits an invocation complete event.
+        - Run any callbacks registered for this event.
+        """
+
+        self._services.logger.debug(
+            f"On after run node: queue item {queue_item.item_id}, session {queue_item.session_id}, node {invocation.id} ({invocation.get_type()})"
+        )
+
+        # Send complete event on successful runs
+        self._services.events.emit_invocation_complete(invocation=invocation, queue_item=queue_item, output=output)
+
+        for callback in self._on_after_run_node_callbacks:
+            callback(invocation=invocation, queue_item=queue_item, output=output)
+
+    def _on_node_error(
+        self,
+        invocation: BaseInvocation,
+        queue_item: SessionQueueItem,
+        error_type: str,
+        error_message: str,
+        error_traceback: str,
+    ):
+        """Called when a node errors. Node errors may occur when running or preparing the node..
+
+        - Set the node error on the session object.
+        - Log the error.
+        - Fail the queue item.
+        - Emits an invocation error event.
+        - Run any callbacks registered for this event.
+        """
+
+        self._services.logger.debug(
+            f"On node error: queue item {queue_item.item_id}, session {queue_item.session_id}, node {invocation.id} ({invocation.get_type()})"
+        )
+
+        # Node errors do not get the full traceback. Only the queue item gets the full traceback.
+        node_error = f"{error_type}: {error_message}"
+        queue_item.session.set_node_error(invocation.id, node_error)
+        self._services.logger.error(
+            f"Error while invoking session {queue_item.session_id}, invocation {invocation.id} ({invocation.get_type()}): {error_message}"
+        )
+        self._services.logger.error(error_traceback)
+
+        # Fail the queue item
+        queue_item = self._services.session_queue.set_queue_item_session(queue_item.item_id, queue_item.session)
+        queue_item = self._services.session_queue.fail_queue_item(
+            queue_item.item_id, error_type, error_message, error_traceback
+        )
+
+        # Send error event
+        self._services.events.emit_invocation_error(
+            queue_item=queue_item,
+            invocation=invocation,
+            error_type=error_type,
+            error_message=error_message,
+            error_traceback=error_traceback,
+        )
+
+        for callback in self._on_node_error_callbacks:
+            callback(
+                invocation=invocation,
+                queue_item=queue_item,
+                error_type=error_type,
+                error_message=error_message,
+                error_traceback=error_traceback,
+            )
+
+
+class DefaultSessionProcessor(SessionProcessorBase):
+    def __init__(
+        self,
+        session_runner: Optional[SessionRunnerBase] = None,
+        on_non_fatal_processor_error_callbacks: Optional[list[OnNonFatalProcessorError]] = None,
+        thread_limit: int = 1,
+        polling_interval: int = 1,
+    ) -> None:
+        super().__init__()
+
+        self.session_runner = session_runner if session_runner else DefaultSessionRunner()
+        self.workflow_call_queue_lifecycle = self.session_runner.workflow_call_queue_lifecycle
+        self._on_non_fatal_processor_error_callbacks = on_non_fatal_processor_error_callbacks or []
+        self._thread_limit = thread_limit
+        self._polling_interval = polling_interval
+
+    def start(self, invoker: Invoker) -> None:
+        self._invoker: Invoker = invoker
+        self._queue_item: Optional[SessionQueueItem] = None
+        self._invocation: Optional[BaseInvocation] = None
+
+        self._resume_event = ThreadEvent()
+        self._stop_event = ThreadEvent()
+        self._poll_now_event = ThreadEvent()
+        self._cancel_event = ThreadEvent()
+
+        register_events(QueueClearedEvent, self._on_queue_cleared)
+        register_events(BatchEnqueuedEvent, self._on_batch_enqueued)
+        register_events(QueueItemStatusChangedEvent, self._on_queue_item_status_changed)
+
+        self._thread_semaphore = BoundedSemaphore(self._thread_limit)
+
+        # If profiling is enabled, create a profiler. The same profiler will be used for all sessions. Internally,
+        # the profiler will create a new profile for each session.
+        self._profiler = (
+            Profiler(
+                logger=self._invoker.services.logger,
+                output_dir=self._invoker.services.configuration.profiles_path,
+                prefix=self._invoker.services.configuration.profile_prefix,
+            )
+            if self._invoker.services.configuration.profile_graphs
+            else None
+        )
+
+        self.session_runner.start(services=invoker.services, cancel_event=self._cancel_event, profiler=self._profiler)
+        self._thread = Thread(
+            name="session_processor",
+            target=self._process,
+            daemon=True,
+            kwargs={
+                "stop_event": self._stop_event,
+                "poll_now_event": self._poll_now_event,
+                "resume_event": self._resume_event,
+                "cancel_event": self._cancel_event,
+            },
+        )
+        self._thread.start()
+
+    def stop(self, *args, **kwargs) -> None:
+        self._stop_event.set()
+        # Cancel any in-progress generation so that long-running nodes (e.g. denoising) stop at
+        # the next step boundary instead of running to completion. Without this, the generation
+        # thread may still be executing CUDA operations when Python teardown begins, which can
+        # cause a C++ std::terminate() crash ("terminate called without an active exception").
+        self._cancel_event.set()
+        # Wake the thread if it is sleeping in poll_now_event.wait() or blocked in resume_event.wait() (paused).
+        self._poll_now_event.set()
+        self._resume_event.set()
+
+    def _poll_now(self) -> None:
+        self._poll_now_event.set()
+
+    async def _on_queue_cleared(self, event: FastAPIEvent[QueueClearedEvent]) -> None:
+        if self._queue_item and self._queue_item.queue_id == event[1].queue_id:
+            self._cancel_event.set()
+            self._poll_now()
+
+    async def _on_batch_enqueued(self, event: FastAPIEvent[BatchEnqueuedEvent]) -> None:
+        self._poll_now()
+
+    async def _on_queue_item_status_changed(self, event: FastAPIEvent[QueueItemStatusChangedEvent]) -> None:
+        # Make sure the cancel event is for the currently processing queue item
+        if self._queue_item and self._queue_item.item_id != event[1].item_id:
+            return
+        if self._queue_item and event[1].status in ["completed", "failed", "canceled"]:
+            # When the queue item is canceled via HTTP, the queue item status is set to `"canceled"` and this event is
+            # emitted. We need to respond to this event and stop graph execution. This is done by setting the cancel
+            # event, which the session runner checks between invocations. If set, the session runner loop is broken.
+            #
+            # Long-running nodes that cannot be interrupted easily present a challenge. `denoise_latents` is one such
+            # node, but it gets a step callback, called on each step of denoising. This callback checks if the queue item
+            # is canceled, and if it is, raises a `CanceledException` to stop execution immediately.
+            if event[1].status == "canceled":
+                self._cancel_event.set()
+            self._poll_now()
+
+    def resume(self) -> SessionProcessorStatus:
+        if not self._resume_event.is_set():
+            self._resume_event.set()
+        return self.get_status()
+
+    def pause(self) -> SessionProcessorStatus:
+        if self._resume_event.is_set():
+            self._resume_event.clear()
+        return self.get_status()
+
+    def get_status(self) -> SessionProcessorStatus:
+        return SessionProcessorStatus(
+            is_started=self._resume_event.is_set(),
+            is_processing=self._queue_item is not None,
+        )
+
+    def _is_image_move_maintenance_active(self) -> bool:
+        image_moves = getattr(self._invoker.services, "image_moves", None)
+        return image_moves is not None and image_moves.is_maintenance_active()
+
+    def _process(
+        self,
+        stop_event: ThreadEvent,
+        poll_now_event: ThreadEvent,
+        resume_event: ThreadEvent,
+        cancel_event: ThreadEvent,
+    ):
+        try:
+            # Any unhandled exception in this block is a fatal processor error and will stop the processor.
+            self._thread_semaphore.acquire()
+            stop_event.clear()
+            resume_event.set()
+            cancel_event.clear()
+
+            while not stop_event.is_set():
+                poll_now_event.clear()
+                try:
+                    # Any unhandled exception in this block is a nonfatal processor error and will be handled.
+                    # If we are paused, wait for resume event
+                    resume_event.wait()
+
+                    if self._is_image_move_maintenance_active():
+                        self._invoker.services.logger.debug("Image storage maintenance is active")
+                        poll_now_event.wait(self._polling_interval)
+                        continue
+
+                    # Get the next session to process
+                    self._queue_item = self._invoker.services.session_queue.dequeue()
+
+                    if self._queue_item is None:
+                        # The queue was empty, wait for next polling interval or event to try again
+                        self._invoker.services.logger.debug("Waiting for next polling interval or event")
+                        poll_now_event.wait(self._polling_interval)
+                        continue
+
+                    # GC-ing here can reduce peak memory usage of the invoke process by freeing allocated memory blocks.
+                    # Most queue items take seconds to execute, so the relative cost of a GC is very small.
+                    # Python will never cede allocated memory back to the OS, so anything we can do to reduce the peak
+                    # allocation is well worth it.
+                    gc.collect()
+
+                    self._invoker.services.logger.info(
+                        f"Executing queue item {self._queue_item.item_id}, session {self._queue_item.session_id}"
+                    )
+                    cancel_event.clear()
+
+                    # Run the graph
+                    self.workflow_call_queue_lifecycle.run_queue_item(self._queue_item)
+
+                except Exception as e:
+                    error_type = e.__class__.__name__
+                    error_message = str(e)
+                    error_traceback = traceback.format_exc()
+                    self._on_non_fatal_processor_error(
+                        queue_item=self._queue_item,
+                        error_type=error_type,
+                        error_message=error_message,
+                        error_traceback=error_traceback,
+                    )
+                    # Wait for next polling interval or event to try again
+                    poll_now_event.wait(self._polling_interval)
+                    continue
+        except Exception as e:
+            # Fatal error in processor, log and pass - we're done here
+            error_type = e.__class__.__name__
+            error_message = str(e)
+            error_traceback = traceback.format_exc()
+            self._invoker.services.logger.error(f"Fatal Error in session processor {error_type}: {error_message}")
+            self._invoker.services.logger.error(error_traceback)
+            pass
+        finally:
+            stop_event.clear()
+            poll_now_event.clear()
+            self._queue_item = None
+            self._thread_semaphore.release()
+
+    def _on_non_fatal_processor_error(
+        self,
+        queue_item: Optional[SessionQueueItem],
+        error_type: str,
+        error_message: str,
+        error_traceback: str,
+    ) -> None:
+        """Called when a non-fatal error occurs in the processor.
+
+        - Log the error.
+        - If a queue item is provided, update the queue item with the completed session & fail it.
+        - Run any callbacks registered for this event.
+        """
+
+        self._invoker.services.logger.error(f"Non-fatal error in session processor {error_type}: {error_message}")
+        self._invoker.services.logger.error(error_traceback)
+
+        if queue_item is not None:
+            try:
+                queue_item = self._invoker.services.session_queue.set_queue_item_session(
+                    queue_item.item_id, queue_item.session
+                )
+                queue_item = self._invoker.services.session_queue.fail_queue_item(
+                    item_id=queue_item.item_id,
+                    error_type=error_type,
+                    error_message=error_message,
+                    error_traceback=error_traceback,
+                )
+            except SessionQueueItemNotFoundError:
+                self._invoker.services.logger.warning(
+                    f"Could not mark queue item {queue_item.item_id} as failed because it no longer exists in the database."
+                )
+
+        for callback in self._on_non_fatal_processor_error_callbacks:
+            callback(
+                queue_item=queue_item,
+                error_type=error_type,
+                error_message=error_message,
+                error_traceback=error_traceback,
+            )

invokeai/app/services/session_processor/workflow_call_batch.py

@@ -0,0 +1,725 @@
+from __future__ import annotations
+
+import copy
+import json
+import random
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from dynamicprompts.generators import CombinatorialPromptGenerator, RandomPromptGenerator
+
+from invokeai.app.invocations.fields import ImageField
+from invokeai.app.services.board_records.board_records_common import BoardRecordOrderBy, BoardVisibility
+from invokeai.app.services.image_records.image_records_common import ASSETS_CATEGORIES, IMAGE_CATEGORIES
+from invokeai.app.services.session_queue.session_queue_common import (
+    Batch,
+    BatchDatum,
+    NodeFieldValue,
+    TooManySessionsError,
+    calc_session_count,
+    create_session_nfv_tuples,
+)
+from invokeai.app.services.shared.graph import GraphExecutionState, WorkflowCallFrame
+from invokeai.app.services.shared.sqlite.sqlite_common import SQLiteDirection
+from invokeai.app.services.shared.workflow_graph_builder import (
+    UnsupportedWorkflowNodeError,
+    apply_workflow_inputs_to_workflow,
+    build_graph_from_workflow,
+)
+
+BATCH_FIELD_NAMES = {
+    "image_batch": "images",
+    "string_batch": "strings",
+    "integer_batch": "integers",
+    "float_batch": "floats",
+}
+SUPPORTED_BATCH_TYPES = set(BATCH_FIELD_NAMES)
+SUPPORTED_BATCH_GROUP_IDS = {
+    "None",
+    "Group 1",
+    "Group 2",
+    "Group 3",
+    "Group 4",
+    "Group 5",
+}
+CONNECTOR_INPUT_HANDLE = "in"
+
+
+@dataclass(frozen=True)
+class WorkflowCallChildSessionResult:
+    session: GraphExecutionState
+    field_values: list[NodeFieldValue] | None = None
+
+
+def _is_mapping(value: Any) -> bool:
+    return isinstance(value, Mapping)
+
+
+def _is_invocation_node(node: Any) -> bool:
+    return _is_mapping(node) and node.get("type") == "invocation" and _is_mapping(node.get("data"))
+
+
+def _is_connector_node(node: Any) -> bool:
+    return _is_mapping(node) and node.get("type") == "connector"
+
+
+def workflow_contains_supported_batch_nodes(workflow: Mapping[str, Any]) -> bool:
+    workflow_nodes = workflow.get("nodes", [])
+    if not isinstance(workflow_nodes, Sequence):
+        return False
+    return any(
+        _is_invocation_node(node) and node["data"].get("type") in SUPPORTED_BATCH_TYPES for node in workflow_nodes
+    )
+
+
+def _get_workflow_nodes(workflow: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]:
+    workflow_nodes = workflow.get("nodes", [])
+    if not isinstance(workflow_nodes, Sequence):
+        return {}
+    return {node["id"]: node for node in workflow_nodes if _is_mapping(node) and isinstance(node.get("id"), str)}
+
+
+def _get_default_edges(workflow: Mapping[str, Any]) -> list[Mapping[str, Any]]:
+    workflow_edges = workflow.get("edges", [])
+    if not isinstance(workflow_edges, Sequence):
+        return []
+    return [edge for edge in workflow_edges if _is_mapping(edge) and edge.get("type") == "default"]
+
+
+def _get_connector_input_edge(
+    connector_id: str, workflow_edges: Sequence[Mapping[str, Any]]
+) -> Mapping[str, Any] | None:
+    return next(
+        (
+            edge
+            for edge in workflow_edges
+            if edge.get("target") == connector_id and edge.get("targetHandle") == CONNECTOR_INPUT_HANDLE
+        ),
+        None,
+    )
+
+
+def _resolve_connector_source(
+    connector_id: str, workflow_nodes: Mapping[str, Mapping[str, Any]], workflow_edges: Sequence[Mapping[str, Any]]
+) -> tuple[str, str] | None:
+    visited: set[str] = set()
+
+    def resolve(node_id: str) -> tuple[str, str] | None:
+        if node_id in visited:
+            return None
+        visited.add(node_id)
+
+        incoming_edge = _get_connector_input_edge(node_id, workflow_edges)
+        if incoming_edge is None:
+            return None
+
+        source_id = incoming_edge.get("source")
+        source_handle = incoming_edge.get("sourceHandle")
+        if not isinstance(source_id, str) or not isinstance(source_handle, str):
+            return None
+
+        source_node = workflow_nodes.get(source_id)
+        if source_node is None:
+            return None
+
+        if _is_invocation_node(source_node):
+            return (source_id, source_handle)
+
+        if _is_connector_node(source_node):
+            return resolve(source_id)
+
+        return None
+
+    return resolve(connector_id)
+
+
+def _build_child_graph_workflow(workflow: Mapping[str, Any], used_generator_node_ids: set[str]) -> dict[str, Any]:
+    workflow_nodes = workflow.get("nodes", [])
+    workflow_edges = workflow.get("edges", [])
+    if not isinstance(workflow_nodes, list) or not isinstance(workflow_edges, list):
+        raise UnsupportedWorkflowNodeError("call_saved_workflow child workflow is malformed")
+
+    filtered_nodes = [
+        node
+        for node in workflow_nodes
+        if not (
+            _is_invocation_node(node)
+            and (
+                node["data"].get("type") in SUPPORTED_BATCH_TYPES
+                or (isinstance(node.get("id"), str) and node["id"] in used_generator_node_ids)
+            )
+        )
+    ]
+    filtered_node_ids = {node["id"] for node in filtered_nodes if _is_mapping(node) and isinstance(node.get("id"), str)}
+    filtered_edges = [
+        edge
+        for edge in workflow_edges
+        if _is_mapping(edge)
+        and edge.get("type") == "default"
+        and edge.get("source") in filtered_node_ids
+        and edge.get("target") in filtered_node_ids
+    ]
+    return {**workflow, "nodes": filtered_nodes, "edges": filtered_edges}
+
+
+def _reject_unrelated_generator_nodes(workflow: Mapping[str, Any], used_generator_node_ids: set[str]) -> None:
+    workflow_nodes = workflow.get("nodes", [])
+    if not isinstance(workflow_nodes, list):
+        raise UnsupportedWorkflowNodeError("call_saved_workflow child workflow is malformed")
+
+    unrelated_generator_nodes: list[tuple[str, str]] = []
+    for node in workflow_nodes:
+        if not _is_invocation_node(node):
+            continue
+
+        node_data = node["data"]
+        node_id = node_data.get("id")
+        node_type = node_data.get("type")
+        if not isinstance(node_id, str) or not isinstance(node_type, str):
+            continue
+        if node_type.endswith("_generator") and node_id not in used_generator_node_ids:
+            unrelated_generator_nodes.append((node_type, node_id))
+
+    if unrelated_generator_nodes:
+        unsupported_nodes = ", ".join(
+            f"'{node_type}' (node '{node_id}')" for node_type, node_id in unrelated_generator_nodes
+        )
+        raise UnsupportedWorkflowNodeError(
+            "call_saved_workflow does not yet support child workflows that mix supported batch nodes with "
+            f"unrelated generator nodes: {unsupported_nodes}"
+        )
+
+
+def _get_batch_group_id(node_data: Mapping[str, Any]) -> str:
+    inputs = node_data.get("inputs")
+    if not _is_mapping(inputs):
+        return "None"
+    batch_group_input = inputs.get("batch_group_id")
+    if not _is_mapping(batch_group_input):
+        return "None"
+    batch_group_id = batch_group_input.get("value")
+    if not isinstance(batch_group_id, str):
+        return "None"
+    if batch_group_id not in SUPPORTED_BATCH_GROUP_IDS:
+        raise UnsupportedWorkflowNodeError(f"Unsupported batch group id '{batch_group_id}' in called workflow")
+    return batch_group_id
+
+
+def _get_batch_items(node_data: Mapping[str, Any], field_name: str) -> list[Any]:
+    inputs = node_data.get("inputs")
+    if not _is_mapping(inputs):
+        raise UnsupportedWorkflowNodeError("call_saved_workflow batch child workflow node inputs are malformed")
+    batch_input = inputs.get(field_name)
+    if not _is_mapping(batch_input):
+        raise UnsupportedWorkflowNodeError(
+            f"call_saved_workflow batch child workflow node is missing required '{field_name}' input"
+        )
+    batch_items = batch_input.get("value")
+    if not isinstance(batch_items, list):
+        raise UnsupportedWorkflowNodeError(
+            f"call_saved_workflow batch child workflow node '{node_data.get('id')}' must provide a direct list for '{field_name}'"
+        )
+    return batch_items
+
+
+def _parse_split_values(input_value: str, split_on: str) -> list[str]:
+    if split_on == "":
+        return [input_value]
+    try:
+        return input_value.split(json.loads(f'"{split_on}"'))
+    except Exception:
+        return input_value.split(split_on)
+
+
+def _resolve_float_generator(value: Mapping[str, Any]) -> list[float]:
+    generator_type = value.get("type")
+    if generator_type == "float_generator_arithmetic_sequence":
+        start = float(value.get("start", 0))
+        step = float(value.get("step", 1))
+        count = int(value.get("count", 10))
+        if step == 0:
+            return [start]
+        return [start + i * step for i in range(count)]
+    if generator_type == "float_generator_linear_distribution":
+        start = float(value.get("start", 0))
+        end = float(value.get("end", 1))
+        count = int(value.get("count", 10))
+        if count == 1:
+            return [start]
+        return [start + (end - start) * (i / (count - 1)) for i in range(count)]
+    if generator_type == "float_generator_random_distribution_uniform":
+        minimum = float(value.get("min", 0))
+        maximum = float(value.get("max", 1))
+        count = int(value.get("count", 10))
+        if "values" in value and isinstance(value["values"], list):
+            return [float(v) for v in value["values"]]
+        rng = random.Random(value.get("seed"))
+        return [rng.random() * (maximum - minimum) + minimum for _ in range(count)]
+    if generator_type == "float_generator_parse_string":
+        if "values" in value and isinstance(value["values"], list):
+            return [float(v) for v in value["values"]]
+        split_values = _parse_split_values(str(value.get("input", "")), str(value.get("splitOn", ",")))
+        return [float(v.strip()) for v in split_values if v.strip()]
+    raise UnsupportedWorkflowNodeError(f"Unsupported float generator type '{generator_type}'")
+
+
+def _resolve_integer_generator(value: Mapping[str, Any]) -> list[int]:
+    generator_type = value.get("type")
+    if generator_type == "integer_generator_arithmetic_sequence":
+        start = int(value.get("start", 0))
+        step = int(value.get("step", 1))
+        count = int(value.get("count", 10))
+        if step == 0:
+            return [start]
+        return [start + i * step for i in range(count)]
+    if generator_type == "integer_generator_linear_distribution":
+        start = int(value.get("start", 0))
+        end = int(value.get("end", 10))
+        count = int(value.get("count", 10))
+        if count == 1:
+            return [start]
+        return [start + round((end - start) * (i / (count - 1))) for i in range(count)]
+    if generator_type == "integer_generator_random_distribution_uniform":
+        minimum = int(value.get("min", 0))
+        maximum = int(value.get("max", 10))
+        count = int(value.get("count", 10))
+        rng = random.Random(value.get("seed"))
+        return [int(rng.random() * (maximum - minimum + 1)) + minimum for _ in range(count)]
+    if generator_type == "integer_generator_parse_string":
+        split_values = _parse_split_values(str(value.get("input", "")), str(value.get("splitOn", ",")))
+        return [int(v.strip()) for v in split_values if v.strip()]
+    raise UnsupportedWorkflowNodeError(f"Unsupported integer generator type '{generator_type}'")
+
+
+def _resolve_string_generator(value: Mapping[str, Any]) -> list[str]:
+    generator_type = value.get("type")
+    if generator_type == "string_generator_parse_string":
+        return [v for v in _parse_split_values(str(value.get("input", "")), str(value.get("splitOn", ","))) if v]
+    if generator_type == "string_generator_dynamic_prompts_combinatorial":
+        generator = CombinatorialPromptGenerator()
+        return list(generator.generate(str(value.get("input", "")), max_prompts=int(value.get("maxPrompts", 10))))
+    if generator_type == "string_generator_dynamic_prompts_random":
+        seed = value.get("seed")
+        if seed is None:
+            seed = random.randint(0, 2**31 - 1)
+        generator = RandomPromptGenerator(seed=int(seed))
+        return list(generator.generate(str(value.get("input", "")), num_images=int(value.get("count", 10))))
+    raise UnsupportedWorkflowNodeError(f"Unsupported string generator type '{generator_type}'")
+
+
+def _assert_user_can_access_board(board_id: str, services: Any, user_id: str | None) -> None:
+    if not user_id:
+        return
+
+    board_records = getattr(services, "board_records", None)
+    if board_records is None or not hasattr(board_records, "get"):
+        return
+
+    users = getattr(services, "users", None)
+    user = users.get(user_id) if users is not None and hasattr(users, "get") else None
+    is_admin = bool(user and getattr(user, "is_admin", False))
+    if is_admin:
+        return
+
+    try:
+        board_record = board_records.get(board_id)
+    except Exception as e:
+        raise UnsupportedWorkflowNodeError(
+            f"call_saved_workflow could not access board '{board_id}' for image generator expansion"
+        ) from e
+
+    if getattr(board_record, "user_id", None) == user_id:
+        return
+
+    board_visibility = getattr(board_record, "board_visibility", BoardVisibility.Private)
+    if isinstance(board_visibility, str):
+        try:
+            board_visibility = BoardVisibility(board_visibility)
+        except ValueError:
+            board_visibility = BoardVisibility.Private
+    if board_visibility in {BoardVisibility.Shared, BoardVisibility.Public}:
+        return
+
+    if hasattr(board_records, "get_all"):
+        try:
+            accessible_boards = board_records.get_all(
+                user_id=user_id,
+                is_admin=False,
+                order_by=BoardRecordOrderBy.Name,
+                direction=SQLiteDirection.Ascending,
+                include_archived=True,
+            )
+        except Exception:
+            accessible_boards = []
+        if any(getattr(board, "board_id", None) == board_id for board in accessible_boards):
+            return
+
+    raise UnsupportedWorkflowNodeError(
+        f"call_saved_workflow caller does not have access to board '{board_id}' for image generator expansion"
+    )
+
+
+def _resolve_image_generator(value: Mapping[str, Any], services: Any, user_id: str | None) -> list[ImageField]:
+    generator_type = value.get("type")
+    if generator_type != "image_generator_images_from_board":
+        raise UnsupportedWorkflowNodeError(f"Unsupported image generator type '{generator_type}'")
+    board_id = value.get("board_id")
+    if not isinstance(board_id, str) or not board_id:
+        return []
+    _assert_user_can_access_board(board_id, services, user_id)
+    category = value.get("category", "images")
+    categories = IMAGE_CATEGORIES if category == "images" else ASSETS_CATEGORIES
+    image_names = services.board_images.get_all_board_image_names_for_board(
+        board_id=board_id,
+        categories=categories,
+        is_intermediate=False,
+    )
+    return [ImageField(image_name=image_name) for image_name in image_names]
+
+
+def _get_declared_generator_item_count(value: Mapping[str, Any]) -> int | None:
+    explicit_values = value.get("values")
+    if isinstance(explicit_values, list):
+        return len(explicit_values)
+    if value.get("type") == "string_generator_dynamic_prompts_combinatorial":
+        return int(value.get("maxPrompts", 10))
+    if value.get("type") in {
+        "float_generator_arithmetic_sequence",
+        "float_generator_linear_distribution",
+        "float_generator_random_distribution_uniform",
+        "integer_generator_arithmetic_sequence",
+        "integer_generator_linear_distribution",
+        "integer_generator_random_distribution_uniform",
+        "string_generator_dynamic_prompts_random",
+    }:
+        return int(value.get("count", 10))
+    return None
+
+
+def _resolve_generator_items(
+    generator_node: Mapping[str, Any], services: Any, user_id: str | None, maximum_items: int
+) -> list[Any]:
+    generator_node_data = generator_node["data"]
+    node_type = generator_node_data.get("type")
+    inputs = generator_node_data.get("inputs")
+    if not isinstance(node_type, str) or not _is_mapping(inputs):
+        raise UnsupportedWorkflowNodeError("call_saved_workflow generator node is malformed")
+    generator_input = inputs.get("generator")
+    if not _is_mapping(generator_input):
+        raise UnsupportedWorkflowNodeError(
+            f"call_saved_workflow generator node '{generator_node_data.get('id')}' is missing generator input"
+        )
+    generator_value = generator_input.get("value")
+    if not _is_mapping(generator_value):
+        raise UnsupportedWorkflowNodeError(
+            f"call_saved_workflow generator node '{generator_node_data.get('id')}' has invalid generator value"
+        )
+    declared_item_count = _get_declared_generator_item_count(generator_value)
+    if declared_item_count is not None and declared_item_count > maximum_items:
+        raise TooManySessionsError("call_saved_workflow exceeds remaining queue capacity for child workflow executions")
+    if node_type == "integer_generator":
+        return _resolve_integer_generator(generator_value)
+    if node_type == "float_generator":
+        return _resolve_float_generator(generator_value)
+    if node_type == "string_generator":
+        return _resolve_string_generator(generator_value)
+    if node_type == "image_generator":
+        return _resolve_image_generator(generator_value, services, user_id)
+    raise UnsupportedWorkflowNodeError(f"Unsupported generator node type '{node_type}'")
+
+
+def _get_generator_placeholder_items(generator_node: Mapping[str, Any]) -> list[Any]:
+    generator_node_data = generator_node["data"]
+    node_type = generator_node_data.get("type")
+    if node_type == "integer_generator":
+        return [0]
+    if node_type == "float_generator":
+        return [0.0]
+    if node_type == "string_generator":
+        return [""]
+    if node_type == "image_generator":
+        return [ImageField(image_name="compatibility-placeholder")]
+    raise UnsupportedWorkflowNodeError(f"Unsupported generator node type '{node_type}'")
+
+
+def _get_outgoing_default_edges(
+    node_id: str, source_handle: str, workflow_edges: Sequence[Mapping[str, Any]]
+) -> list[Mapping[str, Any]]:
+    return [
+        edge
+        for edge in workflow_edges
+        if edge.get("source") == node_id and edge.get("sourceHandle") == source_handle and edge.get("type") == "default"
+    ]
+
+
+def _resolve_connector_destinations(
+    connector_id: str, workflow_nodes: Mapping[str, Mapping[str, Any]], workflow_edges: Sequence[Mapping[str, Any]]
+) -> list[tuple[str, str]]:
+    visited: set[str] = set()
+    destinations: list[tuple[str, str]] = []
+    stack = [connector_id]
+    while stack:
+        current_id = stack.pop()
+        if current_id in visited:
+            continue
+        visited.add(current_id)
+        outgoing_edges = _get_outgoing_default_edges(current_id, "out", workflow_edges)
+        for edge in outgoing_edges:
+            target_id = edge.get("target")
+            target_handle = edge.get("targetHandle")
+            if not isinstance(target_id, str) or not isinstance(target_handle, str):
+                continue
+            target_node = workflow_nodes.get(target_id)
+            if target_node is None:
+                continue
+            if _is_invocation_node(target_node):
+                destinations.append((target_id, target_handle))
+            elif _is_connector_node(target_node):
+                stack.append(target_id)
+    return destinations
+
+
+def _resolve_batch_destinations(
+    node_id: str,
+    source_handle: str,
+    workflow_nodes: Mapping[str, Mapping[str, Any]],
+    workflow_edges: Sequence[Mapping[str, Any]],
+) -> list[tuple[str, str]]:
+    destinations: list[tuple[str, str]] = []
+    for edge in _get_outgoing_default_edges(node_id, source_handle, workflow_edges):
+        target_id = edge.get("target")
+        target_handle = edge.get("targetHandle")
+        if not isinstance(target_id, str) or not isinstance(target_handle, str):
+            continue
+        target_node = workflow_nodes.get(target_id)
+        if target_node is None:
+            continue
+        if _is_invocation_node(target_node):
+            destinations.append((target_id, target_handle))
+        elif _is_connector_node(target_node):
+            destinations.extend(_resolve_connector_destinations(target_id, workflow_nodes, workflow_edges))
+    return destinations
+
+
+def _normalize_batch_item_for_destination(destination_field: str, batch_items: list[Any]) -> list[Any]:
+    if destination_field == "collection":
+        return [[item] for item in batch_items]
+    return batch_items
+
+
+def _resolve_batch_items_from_inputs(
+    node_id: str,
+    field_name: str,
+    workflow_edges: Sequence[Mapping[str, Any]],
+    workflow_nodes: Mapping[str, Mapping[str, Any]],
+) -> list[Any] | None:
+    incoming_edges = [
+        edge
+        for edge in workflow_edges
+        if edge.get("target") == node_id and edge.get("targetHandle") == field_name and edge.get("type") == "default"
+    ]
+    if not incoming_edges:
+        return None
+    incoming_source_ids = [edge.get("source") for edge in incoming_edges if isinstance(edge.get("source"), str)]
+    if len(incoming_source_ids) != 1:
+        raise UnsupportedWorkflowNodeError(
+            f"call_saved_workflow does not yet support multiple connected batch inputs on node '{node_id}'"
+        )
+    source_id = incoming_source_ids[0]
+    source_node = workflow_nodes.get(source_id)
+    if _is_invocation_node(source_node) and source_node["data"].get("type", "").endswith("_generator"):
+        return source_id
+    if _is_connector_node(source_node):
+        resolved_source = _resolve_connector_source(source_id, workflow_nodes, workflow_edges)
+        if resolved_source is not None:
+            resolved_source_id, _resolved_source_handle = resolved_source
+            resolved_source_node = workflow_nodes.get(resolved_source_id)
+            if _is_invocation_node(resolved_source_node) and resolved_source_node["data"].get("type", "").endswith(
+                "_generator"
+            ):
+                return resolved_source_id
+    raise UnsupportedWorkflowNodeError(
+        f"call_saved_workflow does not yet support connected batch child workflow inputs on node '{node_id}'"
+    )
+
+
+def build_batch_child_workflow_session_results(
+    *,
+    parent_session: GraphExecutionState,
+    workflow: Mapping[str, Any],
+    workflow_inputs: Mapping[str, Any],
+    call_frame: WorkflowCallFrame,
+    maximum_children: int,
+    services: Any = None,
+    user_id: str | None = None,
+    resolve_generator_items: bool = True,
+) -> list[GraphExecutionState]:
+    mutable_workflow = copy.deepcopy(workflow)
+    apply_workflow_inputs_to_workflow(mutable_workflow, workflow_inputs)
+
+    workflow_nodes = _get_workflow_nodes(mutable_workflow)
+    workflow_edges = _get_default_edges(mutable_workflow)
+
+    batch_data_by_group: dict[str, list[BatchDatum]] = {}
+    used_generator_node_ids: set[str] = set()
+    for node in workflow_nodes.values():
+        if not _is_invocation_node(node):
+            continue
+        node_data = node["data"]
+        node_id = node_data.get("id")
+        node_type = node_data.get("type")
+        if not isinstance(node_id, str) or not isinstance(node_type, str):
+            continue
+        if node_type.endswith("_generator"):
+            continue
+        if node_type not in SUPPORTED_BATCH_TYPES:
+            continue
+
+        field_name = BATCH_FIELD_NAMES[node_type]
+        generator_source_id = _resolve_batch_items_from_inputs(node_id, field_name, workflow_edges, workflow_nodes)
+        if generator_source_id is not None:
+            generator_node = workflow_nodes.get(generator_source_id)
+            if generator_node is None:
+                raise UnsupportedWorkflowNodeError(
+                    f"call_saved_workflow generator-backed batch child workflow is missing generator node '{generator_source_id}'"
+                )
+            generator_node_type = generator_node["data"].get("type") if _is_invocation_node(generator_node) else None
+            if generator_node_type == "image_generator" and services is None and resolve_generator_items:
+                raise UnsupportedWorkflowNodeError(
+                    "call_saved_workflow image-generator-backed batch child workflows require runtime services"
+                )
+            batch_items = (
+                _resolve_generator_items(generator_node, services, user_id, maximum_children)
+                if resolve_generator_items
+                else _get_generator_placeholder_items(generator_node)
+            )
+            used_generator_node_ids.add(generator_source_id)
+            if not batch_items:
+                raise UnsupportedWorkflowNodeError(
+                    f"call_saved_workflow generator-backed batch child workflow node '{generator_source_id}' produced no batch items"
+                )
+        else:
+            batch_items = _get_batch_items(node_data, field_name)
+            if not batch_items:
+                raise UnsupportedWorkflowNodeError(
+                    f"call_saved_workflow batch child workflow node '{node_id}' must provide at least one batch item"
+                )
+        batch_group_id = _get_batch_group_id(node_data)
+        destinations = _resolve_batch_destinations(node_id, field_name, workflow_nodes, workflow_edges)
+        if not destinations:
+            raise UnsupportedWorkflowNodeError(
+                f"call_saved_workflow batch child workflow node '{node_id}' is not connected to any invocation input"
+            )
+        group_batch_data = batch_data_by_group.setdefault(batch_group_id, [])
+        for destination_node_id, destination_field in destinations:
+            group_batch_data.append(
+                BatchDatum(
+                    node_path=destination_node_id,
+                    field_name=destination_field,
+                    items=_normalize_batch_item_for_destination(destination_field, batch_items),
+                )
+            )
+
+    if not batch_data_by_group:
+        raise UnsupportedWorkflowNodeError("call_saved_workflow batch child workflow contains no supported batch nodes")
+
+    _reject_unrelated_generator_nodes(mutable_workflow, used_generator_node_ids)
+    sanitized_workflow = _build_child_graph_workflow(mutable_workflow, used_generator_node_ids)
+    child_graph = build_graph_from_workflow(sanitized_workflow)
+    batch_data = [[datum] for datum in batch_data_by_group.pop("None", [])]
+    batch_data.extend(batch_data_by_group.values())
+    batch = Batch(graph=child_graph, data=batch_data)
+    if calc_session_count(batch) > maximum_children:
+        raise TooManySessionsError("call_saved_workflow exceeds remaining queue capacity for child workflow executions")
+
+    child_session_results: list[WorkflowCallChildSessionResult] = []
+    for session_id, session_json, field_values_json in create_session_nfv_tuples(batch, maximum_children):
+        generated_session = GraphExecutionState.model_validate_json(session_json)
+        child_session = parent_session.create_child_workflow_execution_state(generated_session.graph, call_frame)
+        child_session.id = session_id
+        field_values = [NodeFieldValue.model_validate(field_value) for field_value in json.loads(field_values_json)]
+        child_session_results.append(WorkflowCallChildSessionResult(session=child_session, field_values=field_values))
+    return child_session_results
+
+
+def build_batch_child_workflow_sessions(
+    *,
+    parent_session: GraphExecutionState,
+    workflow: Mapping[str, Any],
+    workflow_inputs: Mapping[str, Any],
+    call_frame: WorkflowCallFrame,
+    maximum_children: int,
+    services: Any = None,
+    user_id: str | None = None,
+    resolve_generator_items: bool = True,
+) -> list[GraphExecutionState]:
+    return [
+        child_result.session
+        for child_result in build_batch_child_workflow_session_results(
+            parent_session=parent_session,
+            workflow=workflow,
+            workflow_inputs=workflow_inputs,
+            call_frame=call_frame,
+            maximum_children=maximum_children,
+            services=services,
+            user_id=user_id,
+            resolve_generator_items=resolve_generator_items,
+        )
+    ]
+
+
+def build_child_workflow_session_results(
+    *,
+    parent_session: GraphExecutionState,
+    workflow: Mapping[str, Any],
+    workflow_inputs: Mapping[str, Any],
+    call_frame: WorkflowCallFrame,
+    maximum_children: int,
+    services: Any = None,
+    user_id: str | None = None,
+    resolve_generator_items: bool = True,
+) -> list[WorkflowCallChildSessionResult]:
+    if workflow_contains_supported_batch_nodes(workflow):
+        return build_batch_child_workflow_session_results(
+            parent_session=parent_session,
+            workflow=workflow,
+            workflow_inputs=workflow_inputs,
+            call_frame=call_frame,
+            maximum_children=maximum_children,
+            services=services,
+            user_id=user_id,
+            resolve_generator_items=resolve_generator_items,
+        )
+
+    mutable_workflow = copy.deepcopy(workflow)
+    apply_workflow_inputs_to_workflow(mutable_workflow, workflow_inputs)
+    child_graph = build_graph_from_workflow(mutable_workflow)
+    child_session = parent_session.create_child_workflow_execution_state(child_graph, call_frame)
+    return [WorkflowCallChildSessionResult(session=child_session)]
+
+
+def build_child_workflow_sessions(
+    *,
+    parent_session: GraphExecutionState,
+    workflow: Mapping[str, Any],
+    workflow_inputs: Mapping[str, Any],
+    call_frame: WorkflowCallFrame,
+    maximum_children: int,
+    services: Any = None,
+    user_id: str | None = None,
+    resolve_generator_items: bool = True,
+) -> list[GraphExecutionState]:
+    return [
+        child_result.session
+        for child_result in build_child_workflow_session_results(
+            parent_session=parent_session,
+            workflow=workflow,
+            workflow_inputs=workflow_inputs,
+            call_frame=call_frame,
+            maximum_children=maximum_children,
+            services=services,
+            user_id=user_id,
+            resolve_generator_items=resolve_generator_items,
+        )
+    ]

invokeai/app/services/session_processor/workflow_call_runtime.py

@@ -0,0 +1,294 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from invokeai.app.invocations.call_saved_workflow import (
+    CallSavedWorkflowInvocation,
+    is_call_saved_workflow_dynamic_input,
+)
+from invokeai.app.invocations.workflow_return import WorkflowReturnOutput
+from invokeai.app.services.session_processor.workflow_call_batch import build_child_workflow_session_results
+from invokeai.app.services.session_queue.session_queue_common import (
+    NodeFieldValue,
+    SessionQueueItem,
+    SessionQueueItemNotFoundError,
+)
+from invokeai.app.services.shared.graph import GraphExecutionState
+
+if TYPE_CHECKING:
+    from invokeai.app.services.session_processor.session_processor_default import DefaultSessionRunner
+
+
+class WorkflowCallCoordinator:
+    """Coordinates call-specific workflow setup."""
+
+    def __init__(self, session_runner: DefaultSessionRunner) -> None:
+        self._session_runner = session_runner
+
+    def _collect_call_saved_workflow_inputs(
+        self, invocation: CallSavedWorkflowInvocation, queue_item: SessionQueueItem
+    ) -> dict[str, Any]:
+        workflow_inputs = dict(invocation.workflow_inputs)
+        for edge in queue_item.session.execution_graph._get_input_edges(invocation.id):
+            if not is_call_saved_workflow_dynamic_input(edge.destination.field):
+                continue
+            if edge.source.node_id not in queue_item.session.results:
+                continue
+            workflow_inputs[edge.destination.field] = getattr(
+                queue_item.session.results[edge.source.node_id], edge.source.field
+            )
+        return workflow_inputs
+
+    @staticmethod
+    def build_child_queue_item(
+        queue_item: SessionQueueItem,
+        child_session: GraphExecutionState,
+        field_values: list[NodeFieldValue] | None = None,
+    ) -> SessionQueueItem:
+        workflow_call_execution = queue_item.session.waiting_workflow_call_execution
+        if workflow_call_execution is None:
+            raise ValueError("Parent queue item is missing active workflow call execution metadata.")
+        root_item_id = getattr(queue_item, "root_item_id", None) or queue_item.item_id
+        child_updates = {
+            "session": child_session,
+            "session_id": child_session.id,
+            "workflow_call_id": workflow_call_execution.id,
+            "parent_item_id": queue_item.item_id,
+            "parent_session_id": queue_item.session_id,
+            "root_item_id": root_item_id,
+            "workflow_call_depth": workflow_call_execution.depth,
+            "field_values": field_values,
+        }
+        if hasattr(queue_item, "model_copy"):
+            return queue_item.model_copy(update=child_updates)
+
+        child_queue_item = type(queue_item).__new__(type(queue_item))
+        child_queue_item.__dict__ = {**queue_item.__dict__, **child_updates}
+        return child_queue_item
+
+    def begin_workflow_call_boundary(
+        self,
+        invocation: CallSavedWorkflowInvocation,
+        queue_item: SessionQueueItem,
+        workflow_record,
+    ) -> SessionQueueItem:
+        queue_status = self._session_runner._services.session_queue.get_queue_status(queue_item.queue_id)
+        remaining_queue_capacity = self._session_runner._services.configuration.max_queue_size - queue_status.pending
+        if remaining_queue_capacity <= 0:
+            raise ValueError("call_saved_workflow exceeds remaining queue capacity for child workflow executions")
+
+        call_frame = queue_item.session.build_workflow_call_frame(invocation.id, invocation.workflow_id)
+        workflow_inputs = self._collect_call_saved_workflow_inputs(invocation, queue_item)
+        child_session_results = build_child_workflow_session_results(
+            parent_session=queue_item.session,
+            workflow=workflow_record.workflow.model_dump(),
+            workflow_inputs=workflow_inputs,
+            call_frame=call_frame,
+            maximum_children=remaining_queue_capacity,
+            services=self._session_runner._services,
+            user_id=getattr(queue_item, "user_id", None),
+        )
+        child_sessions = [child_result.session for child_result in child_session_results]
+        if len(child_sessions) > remaining_queue_capacity:
+            raise ValueError("call_saved_workflow exceeds remaining queue capacity for child workflow executions")
+        queue_item.session.begin_waiting_on_workflow_call(call_frame)
+        queue_item.session.attach_waiting_workflow_call_child_sessions(child_sessions)
+        child_queue_item = None
+        enqueued_child_item_ids: list[int] = []
+        try:
+            self._session_runner._services.session_queue.set_queue_item_session(queue_item.item_id, queue_item.session)
+            for child_result in child_session_results:
+                child_queue_item = self._session_runner._services.session_queue.enqueue_workflow_call_child(
+                    parent_queue_item=queue_item,
+                    child_session=child_result.session,
+                    field_values=child_result.field_values,
+                )
+                enqueued_child_item_ids.append(child_queue_item.item_id)
+            queue_item.session.set_waiting_workflow_call_child_item_ids(enqueued_child_item_ids)
+            self._session_runner._services.session_queue.set_queue_item_session(queue_item.item_id, queue_item.session)
+            self._session_runner._services.session_queue.suspend_queue_item(queue_item.item_id)
+        except Exception as e:
+            if enqueued_child_item_ids:
+                self._session_runner._services.session_queue.delete_queue_items_by_id(enqueued_child_item_ids)
+            queue_item.session.end_waiting_on_workflow_call(status="failed", error_message=str(e))
+            raise
+        queue_item.status = "waiting"
+        if child_queue_item is None:
+            raise ValueError("Workflow call did not produce any child executions.")
+        return child_queue_item
+
+
+class WorkflowCallQueueLifecycle:
+    """Coordinates queue-visible child workflow execution and parent lifecycle transitions."""
+
+    def __init__(self, session_runner: DefaultSessionRunner) -> None:
+        self._session_runner = session_runner
+
+    @staticmethod
+    def get_waiting_workflow_call_invocation(queue_item: SessionQueueItem) -> CallSavedWorkflowInvocation:
+        waiting_frame = queue_item.session.waiting_workflow_call
+        if waiting_frame is None:
+            raise ValueError("Execution state is not waiting on a workflow call.")
+        invocation = queue_item.session.execution_graph.nodes.get(waiting_frame.prepared_call_node_id)
+        if not isinstance(invocation, CallSavedWorkflowInvocation):
+            raise ValueError("Waiting workflow call frame does not point to a call_saved_workflow invocation.")
+        return invocation
+
+    @staticmethod
+    def get_child_workflow_return_output(child_session: GraphExecutionState) -> WorkflowReturnOutput:
+        workflow_return_node_ids = [
+            node_id for node_id, node in child_session.graph.nodes.items() if node.get_type() == "workflow_return"
+        ]
+        if not workflow_return_node_ids:
+            raise ValueError("The selected saved workflow must contain exactly one workflow_return node.")
+        if len(workflow_return_node_ids) > 1:
+            raise ValueError("The selected saved workflow must not contain more than one workflow_return node.")
+
+        workflow_return_node_id = workflow_return_node_ids[0]
+        prepared_return_node_ids = child_session.source_prepared_mapping.get(workflow_return_node_id, set())
+        if len(prepared_return_node_ids) != 1:
+            raise ValueError(
+                "The selected saved workflow produced an unsupported number of workflow_return executions."
+            )
+
+        prepared_return_node_id = next(iter(prepared_return_node_ids))
+        output = child_session.results.get(prepared_return_node_id)
+        if not isinstance(output, WorkflowReturnOutput):
+            raise ValueError("The selected saved workflow did not produce a valid workflow_return output.")
+
+        return output
+
+    def resume_waiting_workflow_call(self, queue_item: SessionQueueItem) -> None:
+        invocation = self.get_waiting_workflow_call_invocation(queue_item)
+        child_session = queue_item.session.waiting_workflow_call_child_session
+        if child_session is None:
+            raise ValueError("Execution state is waiting on a workflow call but has no attached child session.")
+        output = self.get_child_workflow_return_output(child_session)
+        queue_item.session.end_waiting_on_workflow_call(status="completed")
+        queue_item.session.complete(invocation.id, output)
+        self._session_runner._on_after_run_node(invocation, queue_item, output)
+
+    def fail_waiting_workflow_call(self, queue_item: SessionQueueItem, error_message: str) -> None:
+        invocation = self.get_waiting_workflow_call_invocation(queue_item)
+        queue_item.session.end_waiting_on_workflow_call(status="failed", error_message=error_message)
+        self._session_runner._on_node_error(
+            invocation=invocation,
+            queue_item=queue_item,
+            error_type="ValueError",
+            error_message=error_message,
+            error_traceback=error_message,
+        )
+
+    def _get_parent_queue_item(self, child_queue_item: SessionQueueItem) -> SessionQueueItem | None:
+        parent_item_id = child_queue_item.parent_item_id
+        if parent_item_id is None:
+            raise ValueError("Child workflow queue item is missing parent_item_id metadata.")
+        try:
+            return self._session_runner._services.session_queue.get_queue_item(parent_item_id)
+        except SessionQueueItemNotFoundError:
+            # The parent may have been deleted while the child was running (e.g. the queue was cleared).
+            return None
+
+    def _resume_parent_from_completed_child(self, child_queue_item: SessionQueueItem) -> None:
+        parent_queue_item = self._get_parent_queue_item(child_queue_item)
+        if parent_queue_item is None or parent_queue_item.status in ("completed", "failed", "canceled"):
+            return
+        try:
+            output = self.get_child_workflow_return_output(child_queue_item.session)
+            should_resume_parent, aggregated_values = (
+                parent_queue_item.session.record_waiting_workflow_call_child_completion(
+                    child_queue_item.item_id, output.values
+                )
+            )
+        except Exception as e:
+            workflow_call_execution = parent_queue_item.session.waiting_workflow_call_execution
+            if workflow_call_execution is not None:
+                self._session_runner._services.session_queue.cancel_workflow_call_children(
+                    workflow_call_execution.id,
+                    exclude_item_ids={child_queue_item.item_id},
+                )
+            self.fail_waiting_workflow_call(parent_queue_item, str(e))
+            try:
+                parent_queue_item = self._session_runner._services.session_queue.get_queue_item(
+                    parent_queue_item.item_id
+                )
+            except SessionQueueItemNotFoundError:
+                return
+            if getattr(parent_queue_item, "parent_item_id", None) is not None:
+                self._fail_parent_from_failed_child(parent_queue_item)
+            return
+        if not should_resume_parent:
+            self._session_runner._services.session_queue.set_queue_item_session(
+                parent_queue_item.item_id, parent_queue_item.session
+            )
+            return
+        parent_queue_item.session.waiting_workflow_call_child_session = child_queue_item.session
+        waiting_invocation = self.get_waiting_workflow_call_invocation(parent_queue_item)
+        parent_queue_item.session.end_waiting_on_workflow_call(status="completed")
+        parent_output = WorkflowReturnOutput(values=aggregated_values)
+        parent_queue_item.session.complete(waiting_invocation.id, parent_output)
+        self._session_runner._on_after_run_node(waiting_invocation, parent_queue_item, parent_output)
+        parent_queue_item = self._session_runner._services.session_queue.set_queue_item_session(
+            parent_queue_item.item_id, parent_queue_item.session
+        )
+        if parent_queue_item.session.is_complete():
+            parent_queue_item = self._session_runner._services.session_queue.complete_queue_item(
+                parent_queue_item.item_id
+            )
+            if getattr(parent_queue_item, "parent_item_id", None) is not None:
+                self._resume_parent_from_completed_child(parent_queue_item)
+            return
+        self._session_runner._services.session_queue.resume_queue_item(parent_queue_item.item_id)
+
+    def _fail_parent_from_failed_child(self, child_queue_item: SessionQueueItem) -> None:
+        parent_queue_item = self._get_parent_queue_item(child_queue_item)
+        if parent_queue_item is None or parent_queue_item.status in ("completed", "failed", "canceled"):
+            return
+        waiting_frame = parent_queue_item.session.waiting_workflow_call
+        if waiting_frame is None:
+            raise ValueError("Parent queue item is missing workflow call waiting state.")
+        workflow_call_execution = parent_queue_item.session.waiting_workflow_call_execution
+        if workflow_call_execution is not None:
+            self._session_runner._services.session_queue.cancel_workflow_call_children(
+                workflow_call_execution.id,
+                exclude_item_ids={child_queue_item.item_id},
+            )
+        child_error_message = getattr(child_queue_item, "error_message", None) or (
+            f"The selected saved workflow '{waiting_frame.workflow_id}' failed during child execution."
+        )
+        self.fail_waiting_workflow_call(parent_queue_item, child_error_message)
+        try:
+            parent_queue_item = self._session_runner._services.session_queue.get_queue_item(parent_queue_item.item_id)
+        except SessionQueueItemNotFoundError:
+            return
+        if getattr(parent_queue_item, "parent_item_id", None) is not None:
+            self._fail_parent_from_failed_child(parent_queue_item)
+
+    def _cancel_parent_from_canceled_child(self, child_queue_item: SessionQueueItem) -> None:
+        parent_queue_item = self._get_parent_queue_item(child_queue_item)
+        if parent_queue_item is None or parent_queue_item.status == "canceled":
+            return
+        self._session_runner._services.session_queue.cancel_queue_item(parent_queue_item.item_id)
+
+    def run_queue_item(self, queue_item: SessionQueueItem) -> None:
+        self._session_runner.run(queue_item)
+        try:
+            updated_queue_item = self._session_runner._services.session_queue.get_queue_item(queue_item.item_id)
+        except SessionQueueItemNotFoundError:
+            # The queue item was deleted while it was running (e.g. the queue was cleared or the current item
+            # was deleted). There is no parent lifecycle work to do for a deleted item.
+            return
+        if getattr(updated_queue_item, "parent_item_id", None) is None:
+            return
+        try:
+            if updated_queue_item.status == "completed":
+                self._resume_parent_from_completed_child(updated_queue_item)
+            elif updated_queue_item.status == "failed":
+                self._fail_parent_from_failed_child(updated_queue_item)
+            elif updated_queue_item.status == "canceled":
+                self._cancel_parent_from_canceled_child(updated_queue_item)
+        except SessionQueueItemNotFoundError:
+            # An item in the parent chain was deleted after we looked it up but before we mutated it
+            # (the queue mutations re-read the row and raise when it has disappeared). The chain is
+            # being torn down; there is nothing left to update.
+            return