chore: import upstream snapshot with attribution
Validate YAML Workflows / Validate YAML Configuration Files (push) Waiting to run
Validate YAML Workflows / Validate YAML Configuration Files (push) Waiting to run
This commit is contained in:
wehub-resource-sync
Executable
Executable
+218
@@ -0,0 +1,218 @@
|
||||
"""Cycle detection and management for workflow graphs."""
|
||||
|
||||
from typing import Dict, List, Set, Optional, Any
|
||||
from dataclasses import dataclass, field
|
||||
from entity.configs import Node
|
||||
|
||||
|
||||
@dataclass
|
||||
class CycleInfo:
|
||||
"""Information about a detected cycle in the workflow graph."""
|
||||
cycle_id: str
|
||||
nodes: Set[str] # Node IDs in the cycle
|
||||
entry_nodes: Set[str] # Nodes that can enter the cycle (kept for compatibility)
|
||||
exit_edges: List[Dict[str, Any]] # Edges that can exit the cycle
|
||||
iteration_count: int = 0
|
||||
max_iterations: Optional[int] = None # Safety limit
|
||||
is_active: bool = False
|
||||
execution_state: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# New fields for refactored cycle execution
|
||||
initial_node: Optional[str] = None # The unique initial node when first entering the cycle
|
||||
configured_entry_node: Optional[str] = None # User-configured entry node (if any)
|
||||
max_iterations_default: int = 100 # Default maximum iterations if max_iterations is None
|
||||
|
||||
def add_node(self, node_id: str) -> None:
|
||||
"""Add a node to the cycle."""
|
||||
self.nodes.add(node_id)
|
||||
|
||||
def add_entry_node(self, node_id: str) -> None:
|
||||
"""Add an entry node to the cycle."""
|
||||
self.entry_nodes.add(node_id)
|
||||
|
||||
def add_exit_edge(self, edge_config: Dict[str, Any]) -> None:
|
||||
"""Add an exit edge configuration."""
|
||||
self.exit_edges.append(edge_config)
|
||||
|
||||
def increment_iteration(self) -> None:
|
||||
"""Increment the iteration counter."""
|
||||
self.iteration_count += 1
|
||||
|
||||
def is_within_iteration_limit(self) -> bool:
|
||||
"""Check if the cycle should continue executing."""
|
||||
max_iter = self.max_iterations if self.max_iterations is not None else self.max_iterations_default
|
||||
return self.iteration_count < max_iter
|
||||
|
||||
def get_max_iterations(self) -> int:
|
||||
"""Get the effective maximum iterations."""
|
||||
return self.max_iterations if self.max_iterations is not None else self.max_iterations_default
|
||||
|
||||
def reset_iteration_count(self) -> None:
|
||||
"""Reset the iteration counter."""
|
||||
self.iteration_count = 0
|
||||
|
||||
def is_node_in_cycle(self, node_id: str) -> bool:
|
||||
"""Check if a node is part of this cycle."""
|
||||
return node_id in self.nodes
|
||||
|
||||
def is_entry_node(self, node_id: str) -> bool:
|
||||
"""Check if a node is an entry node for this cycle."""
|
||||
return node_id in self.entry_nodes
|
||||
|
||||
|
||||
class CycleDetector:
|
||||
"""Detects cycles in workflow graphs using Tarjan's algorithm."""
|
||||
|
||||
def __init__(self):
|
||||
self.index_counter = 0
|
||||
self.index: Dict[str, int] = {}
|
||||
self.low_link: Dict[str, int] = {}
|
||||
self.stack: List[str] = []
|
||||
self.on_stack: Set[str] = set()
|
||||
self.cycles: List[Set[str]] = []
|
||||
|
||||
def detect_cycles(self, nodes: Dict[str, Node]) -> List[Set[str]]:
|
||||
"""Detect all cycles in the graph using Tarjan's strongly connected components' algorithm."""
|
||||
self.cycles.clear()
|
||||
self.index_counter = 0
|
||||
self.index.clear()
|
||||
self.low_link.clear()
|
||||
self.stack.clear()
|
||||
self.on_stack.clear()
|
||||
|
||||
for node_id in nodes:
|
||||
if node_id not in self.index:
|
||||
self._strong_connect(node_id, nodes)
|
||||
|
||||
# return [cycle for cycle in self.cycles if len(cycle) > 1 or self._has_self_loop(next(iter(cycle)), nodes)]
|
||||
return self.cycles
|
||||
|
||||
def _has_self_loop(self, node_id: str, nodes: Dict[str, Node]) -> bool:
|
||||
"""Check if a node has a self-loop."""
|
||||
node = nodes.get(node_id)
|
||||
if not node:
|
||||
return False
|
||||
return any(edge_link.target.id == node_id for edge_link in node.iter_outgoing_edges())
|
||||
|
||||
def _strong_connect(self, node_id: str, nodes: Dict[str, Node]) -> None:
|
||||
"""Recursive part of Tarjan's algorithm."""
|
||||
self.index[node_id] = self.index_counter
|
||||
self.low_link[node_id] = self.index_counter
|
||||
self.index_counter += 1
|
||||
self.stack.append(node_id)
|
||||
self.on_stack.add(node_id)
|
||||
|
||||
node = nodes.get(node_id)
|
||||
if not node:
|
||||
return
|
||||
|
||||
# Consider successors of node
|
||||
for edge_link in node.iter_outgoing_edges():
|
||||
successor_id = edge_link.target.id
|
||||
|
||||
if successor_id not in self.index:
|
||||
self._strong_connect(successor_id, nodes)
|
||||
self.low_link[node_id] = min(self.low_link[node_id], self.low_link[successor_id])
|
||||
elif successor_id in self.on_stack:
|
||||
self.low_link[node_id] = min(self.low_link[node_id], self.index[successor_id])
|
||||
|
||||
# If node is a root node, pop the stack and generate an SCC
|
||||
if self.low_link[node_id] == self.index[node_id]:
|
||||
cycle = set()
|
||||
while True:
|
||||
w = self.stack.pop()
|
||||
self.on_stack.remove(w)
|
||||
cycle.add(w)
|
||||
if w == node_id:
|
||||
break
|
||||
|
||||
if len(cycle) > 1 or self._has_self_loop(node_id, nodes):
|
||||
self.cycles.append(cycle)
|
||||
|
||||
|
||||
class CycleManager:
|
||||
"""Manages execution of cycles in the workflow graph."""
|
||||
|
||||
def __init__(self):
|
||||
self.cycles: Dict[str, CycleInfo] = {}
|
||||
self.node_to_cycle: Dict[str, str] = {} # Maps node ID to cycle ID
|
||||
self.active_cycles: Set[str] = set()
|
||||
|
||||
def initialize_cycles(self, cycles: List[Set[str]], nodes: Dict[str, Node]) -> None:
|
||||
"""Initialize cycle information from detected cycles."""
|
||||
self.cycles.clear()
|
||||
self.node_to_cycle.clear()
|
||||
|
||||
for i, cycle_nodes in enumerate(cycles):
|
||||
cycle_id = f"cycle_{i}_{cycle_nodes}"
|
||||
cycle_info = CycleInfo(
|
||||
cycle_id=cycle_id,
|
||||
nodes=set(cycle_nodes),
|
||||
entry_nodes=set(),
|
||||
exit_edges=[]
|
||||
)
|
||||
|
||||
# Find entry nodes and exit edges
|
||||
self._analyze_cycle_structure(cycle_info, nodes)
|
||||
|
||||
self.cycles[cycle_id] = cycle_info
|
||||
|
||||
# Map nodes to their cycle
|
||||
for node_id in cycle_nodes:
|
||||
self.node_to_cycle[node_id] = cycle_id
|
||||
|
||||
def _analyze_cycle_structure(self, cycle: CycleInfo, nodes: Dict[str, Node]) -> None:
|
||||
"""Analyze cycle structure to find entry nodes and exit edges."""
|
||||
cycle_nodes = cycle.nodes
|
||||
|
||||
# Find entry nodes (nodes with predecessors outside the cycle)
|
||||
def judge_entry_predecessor(_predecessor: Node, _predecessor_id: str) -> bool:
|
||||
if _predecessor_id in cycle_nodes:
|
||||
return False
|
||||
for _edge_link in _predecessor.iter_outgoing_edges():
|
||||
if _edge_link.target.id == node_id:
|
||||
if _edge_link.trigger:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
return False
|
||||
|
||||
for node_id in cycle_nodes:
|
||||
node = nodes.get(node_id)
|
||||
if not node:
|
||||
continue
|
||||
|
||||
for predecessor in node.predecessors:
|
||||
if judge_entry_predecessor(predecessor, predecessor.id):
|
||||
cycle.add_entry_node(node_id)
|
||||
break
|
||||
|
||||
# Find exit edges (edges from cycle nodes to nodes outside the cycle)
|
||||
for node_id in cycle_nodes:
|
||||
node = nodes.get(node_id)
|
||||
if not node:
|
||||
continue
|
||||
|
||||
for edge_link in node.iter_outgoing_edges():
|
||||
if edge_link.target.id not in cycle_nodes and edge_link.trigger:
|
||||
exit_edge = {
|
||||
"from": node_id,
|
||||
"to": edge_link.target.id,
|
||||
"condition": edge_link.condition,
|
||||
"trigger": edge_link.trigger,
|
||||
"config": edge_link.config
|
||||
}
|
||||
cycle.add_exit_edge(exit_edge)
|
||||
|
||||
def activate_cycle(self, cycle_id: str) -> None:
|
||||
"""Activate a cycle for execution."""
|
||||
if cycle_id in self.cycles:
|
||||
self.cycles[cycle_id].is_active = True
|
||||
self.active_cycles.add(cycle_id)
|
||||
|
||||
def deactivate_cycle(self, cycle_id: str) -> None:
|
||||
"""Deactivate a cycle."""
|
||||
if cycle_id in self.cycles:
|
||||
self.cycles[cycle_id].is_active = False
|
||||
self.cycles[cycle_id].iteration_count = 0
|
||||
self.active_cycles.discard(cycle_id)
|
||||
Executable
Executable
+614
@@ -0,0 +1,614 @@
|
||||
"""Cycle executor that runs workflow graphs containing loops."""
|
||||
|
||||
import copy
|
||||
import threading
|
||||
from typing import Dict, List, Callable, Any, Set, Optional
|
||||
|
||||
from entity.configs import Node, EdgeLink
|
||||
from utils.log_manager import LogManager
|
||||
from workflow.cycle_manager import CycleManager
|
||||
from workflow.executor.parallel_executor import ParallelExecutor
|
||||
from workflow.topology_builder import GraphTopologyBuilder
|
||||
|
||||
|
||||
class CycleExecutor:
|
||||
"""Execute workflow graphs that contain cycles.
|
||||
|
||||
Features:
|
||||
- Scheduling is based on "super nodes"
|
||||
- Parallel execution inside cycles
|
||||
- Automatic detection of exit conditions
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
log_manager: LogManager,
|
||||
nodes: Dict[str, Node],
|
||||
cycle_execution_order: List[Dict[str, Any]],
|
||||
cycle_manager: CycleManager,
|
||||
execute_node_func: Callable[[Node], None],
|
||||
):
|
||||
"""Initialize the cycle executor.
|
||||
|
||||
Args:
|
||||
log_manager: Logger instance
|
||||
nodes: Mapping of node ids to nodes
|
||||
cycle_execution_order: Super-node execution order with cycles
|
||||
cycle_manager: Cycle manager coordinating iterations
|
||||
execute_node_func: Callable that executes a single node
|
||||
"""
|
||||
self.log_manager = log_manager
|
||||
self.nodes = nodes
|
||||
self.cycle_execution_order = cycle_execution_order
|
||||
self.cycle_manager = cycle_manager
|
||||
self.execute_node_func = execute_node_func
|
||||
self.parallel_executor = ParallelExecutor(log_manager, nodes)
|
||||
|
||||
def execute(self) -> None:
|
||||
"""Run the workflow that contains cycles."""
|
||||
self.log_manager.debug("Executing graph with cycles using super-node scheduler")
|
||||
|
||||
for layer_idx, layer_items in enumerate(self.cycle_execution_order):
|
||||
self.log_manager.debug(f"Executing super-node layer {layer_idx} with {len(layer_items)} items")
|
||||
self._execute_super_layer(layer_items)
|
||||
|
||||
def _execute_super_layer(self, layer_items: List[Dict[str, Any]]) -> None:
|
||||
"""Execute a single super-node layer."""
|
||||
self._execute_super_layer_parallel(layer_items)
|
||||
|
||||
def _execute_super_layer_parallel(self, layer_items: List[Dict[str, Any]]) -> None:
|
||||
"""Execute a super-node layer in parallel."""
|
||||
def item_desc_func(item: Dict[str, Any]) -> str:
|
||||
if item["type"] == "cycle":
|
||||
return f"cycle {item['cycle_id']}"
|
||||
elif item["type"] == "node":
|
||||
# New format
|
||||
return f"node {item['node_id']}"
|
||||
else:
|
||||
# Old format: "layer"
|
||||
return f"node {item['nodes'][0]}"
|
||||
|
||||
self.parallel_executor.execute_items_parallel(
|
||||
layer_items,
|
||||
self._execute_super_item,
|
||||
item_desc_func
|
||||
)
|
||||
|
||||
def _execute_super_item(self, item: Dict[str, Any]) -> None:
|
||||
"""Execute a single super-node item (node or cycle)."""
|
||||
if item["type"] == "layer":
|
||||
# Old format: {"type": "layer", "nodes": [node_id]}
|
||||
self._execute_single_node(item["nodes"][0])
|
||||
elif item["type"] == "node":
|
||||
# New format from GraphTopologyBuilder: {"type": "node", "node_id": "..."}
|
||||
self._execute_single_node(item["node_id"])
|
||||
elif item["type"] == "cycle":
|
||||
self._execute_cycle(item)
|
||||
|
||||
def _execute_single_node(self, node_id: str) -> None:
|
||||
"""Execute a non-cycle node."""
|
||||
self.log_manager.debug(f"Executing non-cycle node: {node_id}")
|
||||
|
||||
node = self.nodes[node_id]
|
||||
if node.is_triggered():
|
||||
self.execute_node_func(node)
|
||||
else:
|
||||
self.log_manager.warning(f"Node {node_id} is not triggered, skipping execution")
|
||||
|
||||
def _execute_cycle(self, cycle_info: Dict[str, Any]) -> None:
|
||||
"""Execute a cycle using the multi-iteration logic."""
|
||||
cycle_id = cycle_info["cycle_id"]
|
||||
nodes = cycle_info["nodes"]
|
||||
|
||||
self.log_manager.debug(f"Executing cycle {cycle_id} with nodes: {nodes}")
|
||||
|
||||
# Step 2: Validate cycle entry uniqueness
|
||||
try:
|
||||
initial_node_id = self._validate_cycle_entry(cycle_id, nodes)
|
||||
except ValueError as e:
|
||||
self.log_manager.error(str(e))
|
||||
raise
|
||||
|
||||
if initial_node_id is None:
|
||||
self.log_manager.debug(
|
||||
f"Cycle {cycle_id} has no triggered entry node in this pass; skipping execution"
|
||||
)
|
||||
return
|
||||
|
||||
# Store initial node in cycle_manager
|
||||
self.cycle_manager.cycles[cycle_id].initial_node = initial_node_id
|
||||
self.log_manager.debug(f"Cycle {cycle_id} initial node: {initial_node_id}")
|
||||
|
||||
# Activate cycle
|
||||
self.cycle_manager.activate_cycle(cycle_id)
|
||||
|
||||
# Step 4: Execute cycle with iterations
|
||||
self._execute_cycle_with_iterations(
|
||||
cycle_id,
|
||||
nodes,
|
||||
initial_node_id,
|
||||
max_iterations=self.cycle_manager.cycles[cycle_id].get_max_iterations()
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
self.cycle_manager.deactivate_cycle(cycle_id)
|
||||
self.log_manager.debug(f"Cycle {cycle_id} completed")
|
||||
|
||||
# ==================== New Methods for Refactored Cycle Execution ====================
|
||||
|
||||
def _validate_cycle_entry(self, cycle_id: str, nodes: List[str]) -> str | None:
|
||||
"""
|
||||
Validate that exactly one node in the cycle is triggered by external edges.
|
||||
|
||||
Args:
|
||||
cycle_id: The cycle ID
|
||||
nodes: List of node IDs in the cycle
|
||||
|
||||
Returns:
|
||||
The ID of the unique initial node
|
||||
|
||||
Raises:
|
||||
ValueError: If no node or multiple nodes are triggered
|
||||
"""
|
||||
triggered_nodes: List[str] = []
|
||||
|
||||
for node_id in nodes:
|
||||
node = self.nodes[node_id]
|
||||
# Check if any external predecessor (node outside the cycle) triggers this node
|
||||
for predecessor in node.predecessors:
|
||||
if predecessor.id not in nodes: # External node
|
||||
edge = predecessor.find_outgoing_edge(node_id)
|
||||
if edge and edge.trigger and edge.triggered:
|
||||
triggered_nodes.append(node_id)
|
||||
break
|
||||
|
||||
cycle_info = self.cycle_manager.cycles.get(cycle_id)
|
||||
configured_entry = cycle_info.configured_entry_node if cycle_info else None
|
||||
|
||||
if len(triggered_nodes) == 0:
|
||||
if configured_entry:
|
||||
return configured_entry
|
||||
return None
|
||||
elif len(triggered_nodes) > 1:
|
||||
raise ValueError(
|
||||
f"Cycle {cycle_id} has multiple triggered entry nodes: {triggered_nodes}. "
|
||||
"Only one entry node must be triggered when entering a cycle."
|
||||
)
|
||||
entry_node = triggered_nodes[0]
|
||||
if configured_entry and entry_node != configured_entry:
|
||||
raise ValueError(
|
||||
f"Cycle {cycle_id} entry mismatch: configured '{configured_entry}' "
|
||||
f"but triggered '{entry_node}'",
|
||||
)
|
||||
|
||||
return entry_node
|
||||
|
||||
def _execute_cycle_with_iterations(
|
||||
self,
|
||||
cycle_id: str,
|
||||
cycle_nodes: List[str],
|
||||
initial_node_id: str,
|
||||
max_iterations: int,
|
||||
) -> Set[str]:
|
||||
"""
|
||||
Execute a cycle with multiple iterations.
|
||||
|
||||
Args:
|
||||
cycle_id: Cycle ID
|
||||
cycle_nodes: List of all nodes in the cycle
|
||||
initial_node_id: Initial node ID
|
||||
max_iterations: Maximum number of iterations
|
||||
|
||||
Returns:
|
||||
A tuple of two sets:
|
||||
- exit_nodes: nodes triggered outside the *current* cycle scope
|
||||
- external_nodes: subset of exit_nodes that are also outside the
|
||||
provided parent_cycle_nodes scope
|
||||
"""
|
||||
iteration = 0
|
||||
|
||||
while iteration < max_iterations:
|
||||
self.log_manager.debug(
|
||||
f"Cycle {cycle_id} iteration {iteration + 1}/{max_iterations}"
|
||||
)
|
||||
|
||||
# Step 1: Detect nested cycles in the scoped subgraph
|
||||
inner_cycles = self._detect_cycles_in_scope(cycle_nodes, initial_node_id)
|
||||
|
||||
# Build topological layers (whether there are nested cycles or not)
|
||||
execution_layers = self._build_topological_layers_in_scope(
|
||||
cycle_nodes, initial_node_id, inner_cycles,
|
||||
is_first_iteration=(iteration == 0)
|
||||
)
|
||||
|
||||
# Execute the topological layers
|
||||
external_nodes = self._execute_scope_layers(
|
||||
execution_layers,
|
||||
cycle_id,
|
||||
cycle_nodes,
|
||||
initial_node_id=initial_node_id,
|
||||
is_first_iteration=(iteration == 0)
|
||||
)
|
||||
|
||||
if external_nodes:
|
||||
self.log_manager.debug(
|
||||
f"Cycle {cycle_id} exited - external nodes triggered: {sorted(external_nodes)}"
|
||||
)
|
||||
return external_nodes
|
||||
|
||||
# Step 4: Check if initial node is retriggered
|
||||
if not self._is_initial_node_retriggered(initial_node_id, cycle_nodes):
|
||||
self.log_manager.debug(
|
||||
f"Cycle {cycle_id} completed - initial node not retriggered"
|
||||
)
|
||||
break
|
||||
|
||||
iteration += 1
|
||||
|
||||
if iteration >= max_iterations:
|
||||
self.log_manager.warning(
|
||||
f"Cycle {cycle_id} reached max iterations ({max_iterations})"
|
||||
)
|
||||
return set()
|
||||
def _detect_cycles_in_scope(
|
||||
self,
|
||||
scope_nodes: List[str],
|
||||
initial_node_id: str
|
||||
) -> List[Set[str]]:
|
||||
"""
|
||||
Detect nested cycles within the scoped subgraph.
|
||||
|
||||
Constructs a subgraph containing only:
|
||||
1. Nodes in scope_nodes
|
||||
2. Edges where both source and target are in scope_nodes
|
||||
3. Initial node's incoming edges are REMOVED (to break the outer cycle)
|
||||
|
||||
Args:
|
||||
scope_nodes: List of node IDs in the current scope
|
||||
initial_node_id: Initial node ID (whose incoming edges are removed)
|
||||
|
||||
Returns:
|
||||
List of detected nested cycles (excluding the current cycle itself)
|
||||
"""
|
||||
# Build scoped nodes with initial node's incoming edges removed
|
||||
scoped_nodes = self._build_scoped_nodes(scope_nodes, clear_entry_node=initial_node_id)
|
||||
|
||||
# Use GraphTopologyBuilder to detect cycles
|
||||
all_cycles = GraphTopologyBuilder.detect_cycles(scoped_nodes)
|
||||
|
||||
# Filter out single-node "cycles" (unless they have self-loops)
|
||||
nested_cycles = [
|
||||
cycle for cycle in all_cycles
|
||||
if len(cycle) > 1
|
||||
]
|
||||
|
||||
return nested_cycles
|
||||
|
||||
def _build_scoped_nodes(
|
||||
self,
|
||||
scope_nodes: List[str],
|
||||
clear_entry_node: Optional[str] = None
|
||||
) -> Dict[str, Node]:
|
||||
"""
|
||||
Build a scoped subgraph containing only nodes and edges within the scope.
|
||||
|
||||
Args:
|
||||
scope_nodes: List of node IDs in the scope
|
||||
clear_entry_node: If specified, this node's incoming edges will be removed
|
||||
(used to break the outer cycle when detecting nested cycles)
|
||||
|
||||
Returns:
|
||||
Dictionary of scoped nodes
|
||||
"""
|
||||
scoped_nodes = {}
|
||||
scope_nodes_set = set(scope_nodes)
|
||||
|
||||
for node_id in scope_nodes:
|
||||
original_node = self.nodes[node_id]
|
||||
# Shallow copy the node
|
||||
scoped_node = copy.copy(original_node)
|
||||
|
||||
# Filter outgoing edges: only keep edges where target is in scope AND trigger=true
|
||||
# Special case: if target is clear_entry_node, remove this edge
|
||||
scoped_edges = [
|
||||
edge_link for edge_link in original_node.iter_outgoing_edges()
|
||||
if edge_link.target.id in scope_nodes_set
|
||||
and edge_link.trigger
|
||||
and edge_link.target.id != clear_entry_node # Remove edges to entry node
|
||||
]
|
||||
scoped_node._outgoing_edges = scoped_edges
|
||||
|
||||
# Filter predecessors: only keep predecessors in scope AND with trigger=true edge
|
||||
# Special case: if this node is clear_entry_node, clear all predecessors
|
||||
if node_id == clear_entry_node:
|
||||
scoped_node.predecessors = []
|
||||
else:
|
||||
scoped_predecessors = []
|
||||
for pred in original_node.predecessors:
|
||||
if pred.id in scope_nodes_set:
|
||||
# Check if the edge from pred to node has trigger=true
|
||||
edge = pred.find_outgoing_edge(node_id)
|
||||
if edge and edge.trigger:
|
||||
scoped_predecessors.append(pred)
|
||||
scoped_node.predecessors = scoped_predecessors
|
||||
|
||||
# Filter successors: only keep successors in scope AND with trigger=true edge
|
||||
# Special case: remove clear_entry_node from successors
|
||||
scoped_successors = [
|
||||
succ for succ in original_node.successors
|
||||
if succ.id in scope_nodes_set
|
||||
and succ.id != clear_entry_node # Remove entry node from successors
|
||||
and any(
|
||||
edge_link.target.id == succ.id and edge_link.trigger
|
||||
for edge_link in original_node.iter_outgoing_edges()
|
||||
)
|
||||
]
|
||||
scoped_node.successors = scoped_successors
|
||||
|
||||
scoped_nodes[node_id] = scoped_node
|
||||
|
||||
return scoped_nodes
|
||||
|
||||
def _build_topological_layers_in_scope(
|
||||
self,
|
||||
scope_nodes: List[str],
|
||||
initial_node_id: str,
|
||||
inner_cycles: List[Set[str]],
|
||||
is_first_iteration: bool = False
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Build topological execution order for the scoped subgraph.
|
||||
|
||||
Args:
|
||||
scope_nodes: List of node IDs in the scope
|
||||
initial_node_id: Initial node ID
|
||||
inner_cycles: List of nested cycles detected in the scope
|
||||
is_first_iteration: Whether this is the first iteration (affects initial node handling)
|
||||
|
||||
Returns:
|
||||
List of execution layers, each containing execution items
|
||||
"""
|
||||
# Build scoped nodes WITHOUT clearing entry node
|
||||
# We want to keep all edges intact for execution
|
||||
scoped_nodes = self._build_scoped_nodes(scope_nodes, clear_entry_node=None)
|
||||
|
||||
# Handle entry points based on iteration:
|
||||
# - First iteration: manually clear initial node's predecessors (for in_degree calculation only)
|
||||
# - Subsequent iterations: clear predecessors for all triggered nodes
|
||||
if is_first_iteration:
|
||||
# Clear initial node's predecessors to make it an entry point
|
||||
if initial_node_id in scoped_nodes:
|
||||
scoped_nodes[initial_node_id].predecessors = []
|
||||
else:
|
||||
# Subsequent iterations: clear predecessors for all triggered nodes
|
||||
for node_id in scope_nodes:
|
||||
if self.nodes[node_id].is_triggered():
|
||||
scoped_nodes[node_id].predecessors = []
|
||||
|
||||
# Extract scoped edges from scoped_nodes (not original nodes)
|
||||
# This ensures consistency with the filtered graph structure
|
||||
scoped_edges = []
|
||||
|
||||
# Collect nodes whose incoming edges should be excluded
|
||||
# (to break cycles in topological sorting)
|
||||
exclude_targets = set()
|
||||
if is_first_iteration:
|
||||
# First iteration: exclude edges to initial_node
|
||||
exclude_targets.add(initial_node_id)
|
||||
else:
|
||||
# Subsequent iterations: exclude edges to all triggered nodes
|
||||
for node_id in scope_nodes:
|
||||
if self.nodes[node_id].is_triggered():
|
||||
exclude_targets.add(node_id)
|
||||
|
||||
for node_id in scope_nodes:
|
||||
# Use scoped_node to get filtered edges
|
||||
scoped_node = scoped_nodes.get(node_id)
|
||||
if scoped_node:
|
||||
for edge_link in scoped_node.iter_outgoing_edges():
|
||||
# Exclude edges pointing to nodes in exclude_targets
|
||||
if edge_link.target.id in exclude_targets:
|
||||
continue
|
||||
scoped_edges.append({
|
||||
"from": node_id,
|
||||
"to": edge_link.target.id,
|
||||
"trigger": edge_link.trigger,
|
||||
"condition": edge_link.condition
|
||||
})
|
||||
|
||||
# Use GraphTopologyBuilder to build execution order
|
||||
if not inner_cycles:
|
||||
# No nested cycles, use DAG layers
|
||||
layers = GraphTopologyBuilder.build_dag_layers(scoped_nodes)
|
||||
return layers
|
||||
else:
|
||||
# Has nested cycles, use super-node approach
|
||||
super_graph = GraphTopologyBuilder.create_super_node_graph(
|
||||
scoped_nodes, scoped_edges, inner_cycles
|
||||
)
|
||||
layers = GraphTopologyBuilder.topological_sort_super_nodes(
|
||||
super_graph, inner_cycles
|
||||
)
|
||||
return layers
|
||||
|
||||
def _execute_scope_layers(
|
||||
self,
|
||||
execution_layers: List[List[Dict[str, Any]]],
|
||||
parent_cycle_id: str,
|
||||
parent_cycle_nodes: List[str],
|
||||
initial_node_id: Optional[str] = None,
|
||||
is_first_iteration: bool = False
|
||||
) -> Set[str]:
|
||||
"""
|
||||
Execute scoped layers with parallelism, supporting nested cycles.
|
||||
|
||||
Args:
|
||||
execution_layers: List of execution layers
|
||||
parent_cycle_id: Parent cycle ID
|
||||
parent_cycle_nodes: List of nodes in the parent cycle
|
||||
initial_node_id: Initial node ID (for first iteration special handling)
|
||||
is_first_iteration: Whether this is the first iteration
|
||||
|
||||
Returns:
|
||||
external_nodes: subset of exit_nodes outside parent_cycle_nodes_set
|
||||
"""
|
||||
scope_node_set = set(parent_cycle_nodes)
|
||||
external_nodes: Set[str] = set()
|
||||
stop_event = threading.Event()
|
||||
result_lock = threading.Lock()
|
||||
|
||||
def record_external(nodes: Set[str]) -> None:
|
||||
nonlocal external_nodes
|
||||
if not nodes:
|
||||
return
|
||||
with result_lock:
|
||||
if nodes:
|
||||
external_nodes.update(nodes)
|
||||
stop_event.set()
|
||||
|
||||
def item_desc(item: Dict[str, Any]) -> str:
|
||||
if item["type"] == "node":
|
||||
return f"node {item['node_id']}"
|
||||
if item["type"] == "cycle":
|
||||
return f"cycle {item['cycle_id']}"
|
||||
return "layer_item"
|
||||
|
||||
for layer in execution_layers:
|
||||
if stop_event.is_set():
|
||||
break
|
||||
|
||||
def executor_func(item: Dict[str, Any]) -> None:
|
||||
if stop_event.is_set():
|
||||
return
|
||||
|
||||
if item["type"] == "node":
|
||||
_node_id = item["node_id"]
|
||||
force_execute = is_first_iteration and (_node_id == initial_node_id)
|
||||
targets = self._execute_single_cycle_node_in_scope(
|
||||
_node_id,
|
||||
scope_node_set,
|
||||
force_execute=force_execute
|
||||
)
|
||||
if targets:
|
||||
record_external(targets)
|
||||
|
||||
elif item["type"] == "cycle":
|
||||
inner_cycle_nodes = item["nodes"]
|
||||
inner_cycle_id = item["cycle_id"]
|
||||
|
||||
self.log_manager.debug(
|
||||
f"Executing nested cycle {inner_cycle_id} within cycle {parent_cycle_id}"
|
||||
)
|
||||
|
||||
try:
|
||||
inner_initial_node = self._validate_cycle_entry(
|
||||
inner_cycle_id, inner_cycle_nodes
|
||||
)
|
||||
except ValueError as e:
|
||||
self.log_manager.error(str(e))
|
||||
raise
|
||||
|
||||
if inner_initial_node is None:
|
||||
self.log_manager.debug(
|
||||
f"Nested cycle {inner_cycle_id} has no triggered entry; skipping"
|
||||
)
|
||||
return
|
||||
|
||||
inner_external_nodes = self._execute_cycle_with_iterations(
|
||||
inner_cycle_id,
|
||||
inner_cycle_nodes,
|
||||
inner_initial_node,
|
||||
max_iterations=100,
|
||||
)
|
||||
|
||||
if inner_external_nodes:
|
||||
filtered = {
|
||||
node
|
||||
for node in inner_external_nodes
|
||||
if node not in scope_node_set
|
||||
}
|
||||
if filtered:
|
||||
record_external(filtered)
|
||||
|
||||
self.parallel_executor.execute_items_parallel(
|
||||
layer,
|
||||
executor_func,
|
||||
item_desc
|
||||
)
|
||||
|
||||
if stop_event.is_set():
|
||||
break
|
||||
|
||||
if external_nodes:
|
||||
for node_id in scope_node_set:
|
||||
self.nodes[node_id].reset_triggers()
|
||||
|
||||
return external_nodes
|
||||
|
||||
def _execute_single_cycle_node_in_scope(
|
||||
self,
|
||||
node_id: str,
|
||||
scope_node_set: Set[str],
|
||||
force_execute: bool = False
|
||||
) -> Set[str]:
|
||||
"""
|
||||
Execute a single node within a cycle scope.
|
||||
|
||||
Args:
|
||||
node_id: Node ID to execute
|
||||
scope_node_set: Nodes that belong to the current scoped cycle
|
||||
force_execute: If True, execute even if not triggered (for initial node in first iteration)
|
||||
|
||||
Returns:
|
||||
Set of node IDs triggered outside the current scoped cycle
|
||||
"""
|
||||
node = self.nodes[node_id]
|
||||
|
||||
# Check if node is triggered (unless force_execute is True)
|
||||
if not force_execute:
|
||||
if not node.is_triggered():
|
||||
return set()
|
||||
|
||||
# Reset edge triggers
|
||||
for edge_link in node.iter_outgoing_edges():
|
||||
edge_link.triggered = False
|
||||
|
||||
# Execute the node
|
||||
self.execute_node_func(node)
|
||||
|
||||
# Check if any external node was triggered
|
||||
external_targets: Set[str] = set()
|
||||
for edge_link in node.iter_outgoing_edges():
|
||||
if edge_link.target.id not in scope_node_set and edge_link.triggered:
|
||||
self.log_manager.debug(
|
||||
f"Node {node_id} triggered external node {edge_link.target.id}"
|
||||
)
|
||||
external_targets.add(edge_link.target.id)
|
||||
|
||||
return external_targets
|
||||
|
||||
def _is_initial_node_retriggered(
|
||||
self,
|
||||
initial_node_id: str,
|
||||
cycle_nodes: List[str]
|
||||
) -> bool:
|
||||
"""
|
||||
Check if the initial node is retriggered by any internal edge (from within the cycle).
|
||||
|
||||
Args:
|
||||
initial_node_id: Initial node ID
|
||||
cycle_nodes: List of nodes in the cycle
|
||||
|
||||
Returns:
|
||||
True if the initial node is retriggered by an internal edge
|
||||
"""
|
||||
initial_node = self.nodes[initial_node_id]
|
||||
|
||||
for predecessor in initial_node.predecessors:
|
||||
# Only check predecessors within the cycle
|
||||
if predecessor.id in cycle_nodes:
|
||||
edge = predecessor.find_outgoing_edge(initial_node_id)
|
||||
if edge and edge.trigger and edge.triggered:
|
||||
return True
|
||||
|
||||
return False
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
"""Executor for DAG (Directed Acyclic Graph) workflows."""
|
||||
|
||||
from typing import Dict, List, Callable
|
||||
|
||||
from entity.configs import Node
|
||||
from utils.log_manager import LogManager
|
||||
from workflow.executor.parallel_executor import ParallelExecutor
|
||||
|
||||
|
||||
class DAGExecutor:
|
||||
"""Execute DAG workflows.
|
||||
|
||||
Features:
|
||||
- Execute layer by layer following the topology
|
||||
- Support parallel execution inside a layer
|
||||
- Serialize Human nodes automatically
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
log_manager: LogManager,
|
||||
nodes: Dict[str, Node],
|
||||
layers: List[List[str]],
|
||||
execute_node_func: Callable[[Node], None]
|
||||
):
|
||||
"""Initialize the executor.
|
||||
|
||||
Args:
|
||||
log_manager: Logger instance
|
||||
nodes: Mapping of node ids to ``Node`` objects
|
||||
layers: Topological layers
|
||||
execute_node_func: Callable used to execute a single node
|
||||
"""
|
||||
self.log_manager = log_manager
|
||||
self.nodes = nodes
|
||||
self.layers = layers
|
||||
self.execute_node_func = execute_node_func
|
||||
self.parallel_executor = ParallelExecutor(log_manager, nodes)
|
||||
|
||||
def execute(self) -> None:
|
||||
"""Execute the DAG workflow."""
|
||||
for layer_idx, layer_nodes in enumerate(self.layers):
|
||||
self.log_manager.debug(f"Executing Layer {layer_idx} with nodes: {layer_nodes}")
|
||||
self._execute_layer(layer_nodes)
|
||||
|
||||
def _execute_layer(self, layer_nodes: List[str]) -> None:
|
||||
"""Execute a single topological layer."""
|
||||
def execute_if_triggered(node_id: str) -> None:
|
||||
node = self.nodes[node_id]
|
||||
if node.is_triggered():
|
||||
self.execute_node_func(node)
|
||||
else:
|
||||
self.log_manager.debug(f"Node {node_id} skipped - not triggered")
|
||||
|
||||
self.parallel_executor.execute_nodes_parallel(layer_nodes, execute_if_triggered)
|
||||
Executable
+399
@@ -0,0 +1,399 @@
|
||||
"""Dynamic edge executor for edge-level Map and Tree execution.
|
||||
|
||||
Handles dynamic node expansion based on edge-level dynamic configuration.
|
||||
When a message passes through an edge with dynamic config, the target node
|
||||
is virtually expanded into multiple instances based on split results.
|
||||
"""
|
||||
|
||||
import concurrent.futures
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
from entity.configs import Node
|
||||
from entity.configs.edge.dynamic_edge_config import DynamicEdgeConfig
|
||||
from entity.messages import Message, MessageRole
|
||||
from runtime.node.splitter import create_splitter_from_config, group_messages
|
||||
from utils.log_manager import LogManager
|
||||
|
||||
|
||||
class DynamicEdgeExecutor:
|
||||
"""Execute edge-level dynamic expansion.
|
||||
|
||||
When an edge has dynamic configuration, this executor:
|
||||
1. Splits the payload passing through the edge
|
||||
2. Executes the target node for each split unit
|
||||
3. Collects and returns results (flat for Map, reduced for Tree)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
log_manager: LogManager,
|
||||
node_executor_func: Callable[[Node, List[Message]], List[Message]],
|
||||
):
|
||||
"""Initialize the dynamic edge executor.
|
||||
|
||||
Args:
|
||||
log_manager: Logger instance
|
||||
node_executor_func: Function to execute a node with inputs
|
||||
"""
|
||||
self.log_manager = log_manager
|
||||
self.node_executor_func = node_executor_func
|
||||
|
||||
def execute(
|
||||
self,
|
||||
target_node: Node,
|
||||
payload: Message,
|
||||
dynamic_config: DynamicEdgeConfig,
|
||||
static_inputs: Optional[List[Message]] = None,
|
||||
) -> List[Message]:
|
||||
"""Execute dynamic expansion for an edge.
|
||||
|
||||
Args:
|
||||
target_node: The node to execute (will be used as template)
|
||||
payload: The message passing through the edge
|
||||
dynamic_config: Edge dynamic configuration
|
||||
static_inputs: Optional static inputs from non-dynamic edges
|
||||
|
||||
Returns:
|
||||
List of output messages from all executions
|
||||
"""
|
||||
split_config = dynamic_config.split
|
||||
|
||||
# Create splitter based on config
|
||||
splitter = create_splitter_from_config(split_config)
|
||||
|
||||
# Split the payload into execution units
|
||||
execution_units = splitter.split([payload])
|
||||
|
||||
if not execution_units:
|
||||
self.log_manager.debug(
|
||||
f"Dynamic edge -> {target_node.id}: no execution units after split"
|
||||
)
|
||||
return []
|
||||
|
||||
self.log_manager.info(
|
||||
f"Dynamic edge -> {target_node.id}: splitting into {len(execution_units)} parallel units"
|
||||
)
|
||||
|
||||
if dynamic_config.is_map():
|
||||
return self._execute_map(
|
||||
target_node, execution_units, dynamic_config, static_inputs
|
||||
)
|
||||
elif dynamic_config.is_tree():
|
||||
return self._execute_tree(
|
||||
target_node, execution_units, dynamic_config, static_inputs
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown dynamic type: {dynamic_config.type}")
|
||||
|
||||
def execute_from_inputs(
|
||||
self,
|
||||
target_node: Node,
|
||||
inputs: List[Message],
|
||||
dynamic_config: DynamicEdgeConfig,
|
||||
static_inputs: Optional[List[Message]] = None,
|
||||
) -> List[Message]:
|
||||
"""Execute dynamic expansion using all collected inputs.
|
||||
|
||||
This method is called from _execute_node when a node has incoming edges
|
||||
with dynamic configuration. All inputs are already collected and passed here.
|
||||
|
||||
Args:
|
||||
target_node: The node to execute
|
||||
inputs: Dynamic edge inputs to be split
|
||||
dynamic_config: Edge dynamic configuration
|
||||
static_inputs: Non-dynamic edge inputs to be replicated to all units
|
||||
|
||||
Returns:
|
||||
List of output messages from all executions
|
||||
"""
|
||||
split_config = dynamic_config.split
|
||||
static_inputs = static_inputs or []
|
||||
|
||||
# Create splitter based on config
|
||||
splitter = create_splitter_from_config(split_config)
|
||||
|
||||
# Split only dynamic inputs into execution units
|
||||
execution_units = splitter.split(inputs)
|
||||
|
||||
if not execution_units:
|
||||
self.log_manager.debug(
|
||||
f"Dynamic node {target_node.id}: no execution units after split"
|
||||
)
|
||||
# If no dynamic inputs but have static inputs, execute once with static inputs
|
||||
if static_inputs:
|
||||
return self.node_executor_func(target_node, static_inputs)
|
||||
return []
|
||||
|
||||
self.log_manager.info(
|
||||
f"Dynamic node {target_node.id}: splitting {len(inputs)} dynamic inputs into "
|
||||
f"{len(execution_units)} parallel units ({dynamic_config.type} mode)"
|
||||
+ (f", with {len(static_inputs)} static inputs replicated to each" if static_inputs else "")
|
||||
)
|
||||
|
||||
if dynamic_config.is_map():
|
||||
return self._execute_map(
|
||||
target_node, execution_units, dynamic_config, static_inputs
|
||||
)
|
||||
elif dynamic_config.is_tree():
|
||||
return self._execute_tree(
|
||||
target_node, execution_units, dynamic_config, static_inputs
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown dynamic type: {dynamic_config.type}")
|
||||
|
||||
def _execute_map(
|
||||
self,
|
||||
target_node: Node,
|
||||
execution_units: List[List[Message]],
|
||||
dynamic_config: DynamicEdgeConfig,
|
||||
static_inputs: Optional[List[Message]] = None,
|
||||
) -> List[Message]:
|
||||
"""Execute in Map mode (fan-out only).
|
||||
|
||||
Args:
|
||||
target_node: Target node template
|
||||
execution_units: Split message units
|
||||
dynamic_config: Dynamic configuration
|
||||
static_inputs: Static inputs to copy to all units
|
||||
|
||||
Returns:
|
||||
Flat list of all output messages
|
||||
"""
|
||||
map_config = dynamic_config.as_map_config()
|
||||
max_parallel = map_config.max_parallel
|
||||
all_outputs: List[Message] = []
|
||||
static_inputs = static_inputs or []
|
||||
|
||||
if len(execution_units) == 1:
|
||||
# Single unit - execute directly
|
||||
unit_inputs = list(static_inputs) + execution_units[0]
|
||||
outputs = self._execute_unit(target_node, unit_inputs, 0)
|
||||
all_outputs.extend(outputs)
|
||||
else:
|
||||
# Multiple units - parallel execution
|
||||
effective_workers = min(len(execution_units), max_parallel)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=effective_workers) as executor:
|
||||
futures: Dict[concurrent.futures.Future, int] = {}
|
||||
|
||||
for idx, unit in enumerate(execution_units):
|
||||
unit_inputs = list(static_inputs) + unit
|
||||
future = executor.submit(
|
||||
self._execute_unit, target_node, unit_inputs, idx
|
||||
)
|
||||
futures[future] = idx
|
||||
|
||||
results_by_idx: Dict[int, List[Message]] = {}
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
idx = futures[future]
|
||||
try:
|
||||
result = future.result()
|
||||
results_by_idx[idx] = result
|
||||
self.log_manager.debug(
|
||||
f"Dynamic edge -> {target_node.id}#{idx}: "
|
||||
f"completed with {len(result)} outputs"
|
||||
)
|
||||
except Exception as e:
|
||||
self.log_manager.error(
|
||||
f"Dynamic edge -> {target_node.id}#{idx}: "
|
||||
f"failed with error: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
# Combine results in original order
|
||||
for idx in range(len(execution_units)):
|
||||
if idx in results_by_idx:
|
||||
all_outputs.extend(results_by_idx[idx])
|
||||
|
||||
self.log_manager.info(
|
||||
f"Dynamic edge -> {target_node.id}: "
|
||||
f"Map completed with {len(all_outputs)} total outputs"
|
||||
)
|
||||
|
||||
return all_outputs
|
||||
|
||||
def _execute_tree(
|
||||
self,
|
||||
target_node: Node,
|
||||
execution_units: List[List[Message]],
|
||||
dynamic_config: DynamicEdgeConfig,
|
||||
static_inputs: Optional[List[Message]] = None,
|
||||
) -> List[Message]:
|
||||
"""Execute in Tree mode (fan-out + reduce).
|
||||
|
||||
Args:
|
||||
target_node: Target node template
|
||||
execution_units: Split message units
|
||||
dynamic_config: Dynamic configuration
|
||||
static_inputs: Static inputs (used in first layer only)
|
||||
|
||||
Returns:
|
||||
Single-element list with the final reduced result
|
||||
"""
|
||||
tree_config = dynamic_config.as_tree_config()
|
||||
if tree_config is None:
|
||||
raise ValueError(f"Invalid tree configuration for edge -> {target_node.id}")
|
||||
|
||||
group_size = tree_config.group_size
|
||||
max_parallel = tree_config.max_parallel
|
||||
static_inputs = static_inputs or []
|
||||
|
||||
# Flatten execution units to individual messages
|
||||
current_messages: List[Message] = []
|
||||
for unit in execution_units:
|
||||
current_messages.extend(unit)
|
||||
|
||||
if not current_messages:
|
||||
return []
|
||||
|
||||
self.log_manager.info(
|
||||
f"Dynamic edge -> {target_node.id}: "
|
||||
f"Tree starting with {len(current_messages)} inputs, group_size={group_size}"
|
||||
)
|
||||
|
||||
layer = 0
|
||||
is_first_layer = True
|
||||
|
||||
# Reduction loop
|
||||
while len(current_messages) > 1:
|
||||
layer += 1
|
||||
|
||||
# Group messages
|
||||
groups = group_messages(current_messages, group_size)
|
||||
|
||||
self.log_manager.debug(
|
||||
f"Dynamic edge -> {target_node.id} layer {layer}: "
|
||||
f"processing {len(groups)} groups"
|
||||
)
|
||||
|
||||
layer_outputs: List[Message] = []
|
||||
|
||||
if len(groups) == 1:
|
||||
# Single group - execute directly
|
||||
group_inputs = groups[0]
|
||||
if is_first_layer:
|
||||
group_inputs = list(static_inputs) + group_inputs
|
||||
outputs = self._execute_group(target_node, group_inputs, layer, 0)
|
||||
layer_outputs.extend(outputs)
|
||||
else:
|
||||
# Multiple groups - parallel execution
|
||||
effective_workers = min(len(groups), max_parallel)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=effective_workers) as executor:
|
||||
futures: Dict[concurrent.futures.Future, int] = {}
|
||||
|
||||
for idx, group in enumerate(groups):
|
||||
group_inputs = group
|
||||
if is_first_layer:
|
||||
group_inputs = list(static_inputs) + group_inputs
|
||||
future = executor.submit(
|
||||
self._execute_group, target_node, group_inputs, layer, idx
|
||||
)
|
||||
futures[future] = idx
|
||||
|
||||
results_by_idx: Dict[int, List[Message]] = {}
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
idx = futures[future]
|
||||
try:
|
||||
result = future.result()
|
||||
results_by_idx[idx] = result
|
||||
except Exception as e:
|
||||
self.log_manager.error(
|
||||
f"Dynamic edge -> {target_node.id}#{layer}-{idx}: "
|
||||
f"failed with error: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
for idx in range(len(groups)):
|
||||
if idx in results_by_idx:
|
||||
layer_outputs.extend(results_by_idx[idx])
|
||||
|
||||
self.log_manager.debug(
|
||||
f"Dynamic edge -> {target_node.id} layer {layer}: "
|
||||
f"produced {len(layer_outputs)} outputs"
|
||||
)
|
||||
|
||||
current_messages = layer_outputs
|
||||
is_first_layer = False
|
||||
|
||||
# Safety check
|
||||
if layer > 100:
|
||||
self.log_manager.error(
|
||||
f"Dynamic edge -> {target_node.id}: exceeded maximum layers"
|
||||
)
|
||||
break
|
||||
|
||||
self.log_manager.info(
|
||||
f"Dynamic edge -> {target_node.id}: "
|
||||
f"Tree completed after {layer} layers with {len(current_messages)} output(s)"
|
||||
)
|
||||
|
||||
return current_messages
|
||||
|
||||
def _execute_unit(
|
||||
self,
|
||||
node: Node,
|
||||
unit_inputs: List[Message],
|
||||
unit_index: int,
|
||||
) -> List[Message]:
|
||||
"""Execute a single map unit."""
|
||||
self.log_manager.debug(
|
||||
f"Dynamic edge -> {node.id}#{unit_index}: "
|
||||
f"executing with {len(unit_inputs)} inputs"
|
||||
)
|
||||
|
||||
# Tag inputs with unit index
|
||||
# Clone messages first to avoid mutating shared inputs in parallel threads
|
||||
unit_inputs = [msg.clone() for msg in unit_inputs]
|
||||
for msg in unit_inputs:
|
||||
metadata = dict(msg.metadata)
|
||||
metadata["dynamic_edge_unit_index"] = unit_index
|
||||
msg.metadata = metadata
|
||||
|
||||
# Execute using node executor
|
||||
outputs = self.node_executor_func(node, unit_inputs)
|
||||
|
||||
# Tag outputs with unit index
|
||||
for msg in outputs:
|
||||
metadata = dict(msg.metadata)
|
||||
metadata["dynamic_edge_unit_index"] = unit_index
|
||||
msg.metadata = metadata
|
||||
|
||||
return outputs
|
||||
|
||||
def _execute_group(
|
||||
self,
|
||||
node: Node,
|
||||
group_inputs: List[Message],
|
||||
layer: int,
|
||||
group_index: int,
|
||||
) -> List[Message]:
|
||||
"""Execute a single tree group."""
|
||||
instance_id = f"{node.id}#{layer}-{group_index}"
|
||||
|
||||
self.log_manager.debug(
|
||||
f"Dynamic edge -> {instance_id}: executing with {len(group_inputs)} inputs"
|
||||
)
|
||||
|
||||
# Tag inputs
|
||||
# Clone messages first to avoid mutating shared inputs in parallel threads
|
||||
group_inputs = [msg.clone() for msg in group_inputs]
|
||||
for msg in group_inputs:
|
||||
metadata = dict(msg.metadata)
|
||||
metadata["dynamic_edge_tree_layer"] = layer
|
||||
metadata["dynamic_edge_tree_group"] = group_index
|
||||
msg.metadata = metadata
|
||||
|
||||
# Execute
|
||||
outputs = self.node_executor_func(node, group_inputs)
|
||||
|
||||
# Tag outputs
|
||||
for msg in outputs:
|
||||
metadata = dict(msg.metadata)
|
||||
metadata["dynamic_edge_tree_layer"] = layer
|
||||
metadata["dynamic_edge_tree_group"] = group_index
|
||||
metadata["dynamic_edge_instance_id"] = instance_id
|
||||
msg.metadata = metadata
|
||||
msg.role = MessageRole.USER # Mark as user-generated
|
||||
|
||||
return outputs
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
"""Parallel execution helpers that eliminate duplicated code."""
|
||||
|
||||
import concurrent.futures
|
||||
from typing import Any, Callable, List, Tuple
|
||||
|
||||
from utils.log_manager import LogManager
|
||||
|
||||
|
||||
class ParallelExecutor:
|
||||
"""Manage parallel execution for workflow nodes.
|
||||
|
||||
Provides shared logic for parallel batches and serializes Human nodes when needed.
|
||||
"""
|
||||
|
||||
def __init__(self, log_manager: LogManager, nodes_dict: dict):
|
||||
"""Initialize the parallel executor.
|
||||
|
||||
Args:
|
||||
log_manager: Logger instance
|
||||
nodes_dict: Mapping of ``node_id`` to ``Node``
|
||||
"""
|
||||
self.log_manager = log_manager
|
||||
self.nodes_dict = nodes_dict
|
||||
|
||||
def execute_items_parallel(
|
||||
self,
|
||||
items: List[Any],
|
||||
executor_func: Callable,
|
||||
item_desc_func: Callable[[Any], str],
|
||||
has_blocking_func: Callable[[Any], bool] | None = None,
|
||||
) -> None:
|
||||
"""Execute a list of items in parallel when possible.
|
||||
|
||||
Args:
|
||||
items: Items to execute
|
||||
executor_func: Callable that executes a single item
|
||||
item_desc_func: Callable for logging a human-readable description
|
||||
has_blocking_func: Optional callable to decide if an item requires serialization
|
||||
"""
|
||||
blocking_items, parallel_items = self._partition_blocking_items(items, has_blocking_func)
|
||||
|
||||
if parallel_items:
|
||||
self._execute_parallel_batch(parallel_items, executor_func, item_desc_func)
|
||||
|
||||
if blocking_items:
|
||||
self._execute_sequential_batch(blocking_items, executor_func, item_desc_func)
|
||||
|
||||
def execute_nodes_parallel(
|
||||
self,
|
||||
node_ids: List[str],
|
||||
executor_func: Callable[[str], None]
|
||||
) -> None:
|
||||
"""Execute a list of nodes in parallel.
|
||||
|
||||
Convenience wrapper around ``execute_items_parallel`` specialized for nodes.
|
||||
|
||||
Args:
|
||||
node_ids: List of node identifiers
|
||||
executor_func: Callable that executes a single node
|
||||
"""
|
||||
def item_desc_func(node_id: str) -> str:
|
||||
return f"node {node_id}"
|
||||
|
||||
def has_blocking_func(node_id: str) -> bool:
|
||||
return False
|
||||
|
||||
self.execute_items_parallel(
|
||||
node_ids,
|
||||
executor_func,
|
||||
item_desc_func,
|
||||
has_blocking_func
|
||||
)
|
||||
|
||||
def _partition_blocking_items(
|
||||
self,
|
||||
items: List[Any],
|
||||
has_blocking_func: Callable[[Any], bool] | None
|
||||
) -> Tuple[List[Any], List[Any]]:
|
||||
"""Split items into blocking and parallelizable lists."""
|
||||
blocking_items = []
|
||||
parallel_items = []
|
||||
|
||||
for item in items:
|
||||
if has_blocking_func and has_blocking_func(item):
|
||||
blocking_items.append(item)
|
||||
else:
|
||||
parallel_items.append(item)
|
||||
|
||||
return blocking_items, parallel_items
|
||||
|
||||
def _execute_parallel_batch(
|
||||
self,
|
||||
items: List[Any],
|
||||
executor_func: Callable,
|
||||
item_desc_func: Callable[[Any], str]
|
||||
) -> None:
|
||||
"""Execute a batch of items in parallel.
|
||||
|
||||
Args:
|
||||
items: Items to execute
|
||||
executor_func: Callable per item
|
||||
item_desc_func: Callable returning a readable description
|
||||
"""
|
||||
self.log_manager.debug(f"Executing {len(items)} items in parallel")
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(items)) as executor:
|
||||
futures = []
|
||||
for item in items:
|
||||
future = executor.submit(executor_func, item)
|
||||
futures.append((item, future))
|
||||
|
||||
# Wait for every future to finish
|
||||
for item, future in futures:
|
||||
try:
|
||||
future.result()
|
||||
self.log_manager.debug(f"{item_desc_func(item)} completed successfully")
|
||||
except Exception as e:
|
||||
self.log_manager.error(f"{item_desc_func(item)} failed: {str(e)}")
|
||||
raise
|
||||
|
||||
def _execute_sequential_batch(
|
||||
self,
|
||||
items: List[Any],
|
||||
executor_func: Callable,
|
||||
item_desc_func: Callable[[Any], str]
|
||||
) -> None:
|
||||
"""Execute a batch of items sequentially.
|
||||
|
||||
Args:
|
||||
items: Items to execute
|
||||
executor_func: Callable per item
|
||||
item_desc_func: Callable returning a readable description
|
||||
"""
|
||||
for item in items:
|
||||
self.log_manager.debug(f"Executing {item_desc_func(item)} (sequential)")
|
||||
try:
|
||||
executor_func(item)
|
||||
self.log_manager.debug(f"{item_desc_func(item)} completed successfully")
|
||||
except Exception as e:
|
||||
self.log_manager.error(f"{item_desc_func(item)} failed: {str(e)}")
|
||||
raise
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
"""Resource coordination helpers for workflow node execution."""
|
||||
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Iterable, List, Tuple
|
||||
|
||||
from entity.configs import Node
|
||||
from runtime.node.registry import get_node_registration
|
||||
from utils.log_manager import LogManager
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ResourceRequest:
|
||||
"""Represents a single resource requirement."""
|
||||
|
||||
key: str
|
||||
limit: int
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _ResourceSlot:
|
||||
semaphore: threading.Semaphore
|
||||
limit: int
|
||||
|
||||
|
||||
class ResourceManager:
|
||||
"""Coordinates shared resource usage across nodes."""
|
||||
|
||||
def __init__(self, log_manager: LogManager | None = None):
|
||||
self.log_manager = log_manager
|
||||
self._lock = threading.Lock()
|
||||
self._resources: Dict[str, _ResourceSlot] = {}
|
||||
|
||||
@contextmanager
|
||||
def guard_node(self, node: Node):
|
||||
"""Acquire all resources required by the given node."""
|
||||
requests = self._resolve_node_requests(node)
|
||||
with self._acquire_resources(requests):
|
||||
yield
|
||||
|
||||
def _resolve_node_requests(self, node: Node) -> List[ResourceRequest]:
|
||||
registration = get_node_registration(node.node_type)
|
||||
caps = registration.capabilities
|
||||
requests: List[ResourceRequest] = []
|
||||
key = caps.resource_key
|
||||
limit = caps.resource_limit
|
||||
if key and limit and limit > 0:
|
||||
requests.append(ResourceRequest(key=key, limit=limit))
|
||||
return requests
|
||||
|
||||
@contextmanager
|
||||
def _acquire_resources(self, requests: Iterable[ResourceRequest]):
|
||||
acquired: List[Tuple[str, threading.Semaphore]] = []
|
||||
try:
|
||||
for request in sorted(requests, key=lambda item: item.key):
|
||||
semaphore = self._get_or_create_resource(request)
|
||||
self._log_debug(f"Acquiring resource {request.key}")
|
||||
semaphore.acquire()
|
||||
acquired.append((request.key, semaphore))
|
||||
yield
|
||||
finally:
|
||||
for key, semaphore in reversed(acquired):
|
||||
semaphore.release()
|
||||
self._log_debug(f"Released resource {key}")
|
||||
|
||||
def _get_or_create_resource(self, request: ResourceRequest) -> threading.Semaphore:
|
||||
with self._lock:
|
||||
slot = self._resources.get(request.key)
|
||||
if slot and slot.limit != request.limit:
|
||||
slot = None
|
||||
if not slot:
|
||||
slot = _ResourceSlot(
|
||||
semaphore=threading.Semaphore(request.limit),
|
||||
limit=request.limit,
|
||||
)
|
||||
self._resources[request.key] = slot
|
||||
return slot.semaphore
|
||||
|
||||
def _log_debug(self, message: str) -> None:
|
||||
if self.log_manager:
|
||||
self.log_manager.debug(message)
|
||||
Executable
+860
@@ -0,0 +1,860 @@
|
||||
"""Graph orchestration adapted to ChatDev design_0.4.0 workflows."""
|
||||
|
||||
import threading
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from runtime.node.agent.memory import MemoryBase, MemoryFactory, MemoryManager
|
||||
from runtime.node.agent.thinking import ThinkingManagerBase, ThinkingManagerFactory
|
||||
from entity.configs import Node, EdgeLink, AgentConfig, ConfigError
|
||||
from entity.configs.edge import EdgeConditionConfig
|
||||
from entity.configs.node.memory import SimpleMemoryConfig
|
||||
from entity.messages import Message, MessageRole
|
||||
from runtime.node.executor.base import ExecutionContext
|
||||
from runtime.node.executor.factory import NodeExecutorFactory
|
||||
from utils.logger import WorkflowLogger
|
||||
from utils.exceptions import ValidationError, WorkflowExecutionError, WorkflowCancelledError
|
||||
from utils.structured_logger import get_server_logger
|
||||
from utils.human_prompt import (
|
||||
CliPromptChannel,
|
||||
HumanPromptService,
|
||||
resolve_prompt_channel,
|
||||
)
|
||||
from workflow.cycle_manager import CycleManager
|
||||
from workflow.graph_context import GraphContext
|
||||
from workflow.graph_manager import GraphManager
|
||||
from workflow.executor.resource_manager import ResourceManager
|
||||
from workflow.runtime import (
|
||||
RuntimeBuilder,
|
||||
ResultArchiver,
|
||||
DagExecutionStrategy,
|
||||
CycleExecutionStrategy,
|
||||
MajorityVoteStrategy,
|
||||
)
|
||||
from workflow.runtime.runtime_context import RuntimeContext
|
||||
from runtime.edge.conditions import (
|
||||
ConditionFactoryContext,
|
||||
build_edge_condition_manager,
|
||||
)
|
||||
from runtime.edge.processors import (
|
||||
ProcessorFactoryContext as PayloadProcessorFactoryContext,
|
||||
build_edge_processor as build_edge_payload_processor,
|
||||
)
|
||||
from workflow.executor.dynamic_edge_executor import DynamicEdgeExecutor
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Executor class (includes all Memory and Thinking logic)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
class ExecutionError(RuntimeError):
|
||||
"""Raised when the workflow graph cannot be executed."""
|
||||
|
||||
|
||||
class GraphExecutor:
|
||||
"""Executes ChatDev_new graph workflows with integrated memory and thinking management."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: GraphContext,
|
||||
*,
|
||||
session_id: Optional[str] = None,
|
||||
workspace_hook_factory: Optional[Callable[[RuntimeContext], Any]] = None,
|
||||
cancel_event: Optional[threading.Event] = None,
|
||||
) -> None:
|
||||
"""Initialize executor with graph context instance."""
|
||||
self.majority_result = None
|
||||
self.graph: GraphContext = graph
|
||||
self.outputs = {}
|
||||
self.logger = self._create_logger()
|
||||
self._cancel_event = cancel_event or threading.Event()
|
||||
self._cancel_reason: Optional[str] = None
|
||||
runtime = RuntimeBuilder(graph).build(logger=self.logger, session_id=session_id)
|
||||
if workspace_hook_factory:
|
||||
runtime.workspace_hook = workspace_hook_factory(runtime)
|
||||
self.runtime_context = runtime
|
||||
self.tool_manager = runtime.tool_manager
|
||||
self.function_manager = runtime.function_manager
|
||||
self.edge_processor_function_manager = runtime.edge_processor_function_manager
|
||||
self.log_manager = runtime.log_manager
|
||||
self.resource_manager = ResourceManager(self.log_manager)
|
||||
|
||||
# Memory and Thinking management (moved from Graph)
|
||||
self.thinking_managers: Dict[str, ThinkingManagerBase] = {}
|
||||
self.global_memories: Dict[str, MemoryBase] = {}
|
||||
self.agent_memory_managers: Dict[str, MemoryManager] = {}
|
||||
|
||||
# Token tracking
|
||||
self.token_tracker = runtime.token_tracker
|
||||
|
||||
# Workspace roots
|
||||
self.code_workspace = runtime.code_workspace
|
||||
self.attachment_store = runtime.attachment_store
|
||||
|
||||
# Cycle management
|
||||
self.cycle_manager: Optional[CycleManager] = None
|
||||
|
||||
# Node executors (new strategy pattern implementation)
|
||||
self.__execution_context: Optional[ExecutionContext] = None
|
||||
self.node_executors: Dict[str, Any] = {}
|
||||
self._human_prompt_service: Optional[HumanPromptService] = None
|
||||
|
||||
# for majority voting mode
|
||||
self.initial_task_messages: List[Message] = []
|
||||
|
||||
def request_cancel(self, reason: Optional[str] = None) -> None:
|
||||
"""Signal the executor to stop as soon as possible."""
|
||||
if reason:
|
||||
self._cancel_reason = reason
|
||||
elif not self._cancel_reason:
|
||||
self._cancel_reason = "Workflow execution cancelled"
|
||||
self._cancel_event.set()
|
||||
self.logger.info(f"Cancellation requested for workflow {self.graph.name}")
|
||||
|
||||
def is_cancelled(self) -> bool:
|
||||
return self._cancel_event.is_set()
|
||||
|
||||
def _raise_if_cancelled(self) -> None:
|
||||
if self.is_cancelled():
|
||||
message = self._cancel_reason or "Workflow execution cancelled"
|
||||
raise WorkflowCancelledError(message, workflow_id=self.graph.name)
|
||||
|
||||
def _create_logger(self) -> WorkflowLogger:
|
||||
"""Create and return a logger instance."""
|
||||
return WorkflowLogger(self.graph.name, self.graph.log_level)
|
||||
|
||||
@classmethod
|
||||
def execute_graph(
|
||||
cls,
|
||||
graph: GraphContext,
|
||||
task_prompt: Any,
|
||||
*,
|
||||
cancel_event: Optional[threading.Event] = None,
|
||||
) -> "GraphExecutor":
|
||||
"""Convenience method to execute a graph with a task prompt."""
|
||||
executor = cls(graph, cancel_event=cancel_event)
|
||||
executor._execute(task_prompt)
|
||||
return executor
|
||||
|
||||
def _execute(self, task_prompt: Any):
|
||||
self._raise_if_cancelled()
|
||||
results = self.run(task_prompt)
|
||||
self.graph.record(results)
|
||||
|
||||
def _build_memories_and_thinking(self) -> None:
|
||||
"""Initialize all memory and thinking managers before execution."""
|
||||
self._build_global_memories()
|
||||
self._build_thinking_managers()
|
||||
self._build_agent_memories()
|
||||
self._build_node_executors()
|
||||
|
||||
def _build_global_memories(self) -> None:
|
||||
"""Build global memories from config."""
|
||||
memory_config = self.graph.config.get_memory_config()
|
||||
if not memory_config:
|
||||
return
|
||||
|
||||
for store in memory_config:
|
||||
if store.name in self.global_memories:
|
||||
error_msg = f"Duplicated memory name detected: {store.name}"
|
||||
self.log_manager.error(error_msg)
|
||||
raise ValidationError(error_msg, details={"memory_name": store.name})
|
||||
|
||||
simple_cfg = store.as_config(SimpleMemoryConfig)
|
||||
if simple_cfg and (not simple_cfg.memory_path or simple_cfg.memory_path == "auto"):
|
||||
path = self.graph.directory / f"memory_{store.name}.json"
|
||||
simple_cfg.memory_path = str(path)
|
||||
|
||||
try:
|
||||
memory_instance = MemoryFactory.create_memory(store)
|
||||
self.global_memories[store.name] = memory_instance
|
||||
memory_instance.load()
|
||||
self.log_manager.info(
|
||||
f"Global memory '{store.name}' built successfully",
|
||||
details={"memory_name": store.name},
|
||||
)
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to create memory '{store.name}': {str(e)}"
|
||||
self.log_manager.error(error_msg, details={"memory_name": store.name})
|
||||
logger = get_server_logger()
|
||||
logger.log_exception(e, error_msg, memory_name=store.name)
|
||||
raise WorkflowExecutionError(error_msg, details={"memory_name": store.name})
|
||||
|
||||
def _build_thinking_managers(self) -> None:
|
||||
"""Build thinking managers for nodes that require them."""
|
||||
for node_id, node in self.graph.nodes.items():
|
||||
agent_config = node.as_config(AgentConfig)
|
||||
if agent_config and agent_config.thinking:
|
||||
self.thinking_managers[node_id] = ThinkingManagerFactory.get_thinking_manager(
|
||||
agent_config.thinking
|
||||
)
|
||||
|
||||
def _build_agent_memories(self) -> None:
|
||||
"""Build memory managers for agent nodes referencing global stores."""
|
||||
for node_id, node in self.graph.nodes.items():
|
||||
agent_config = node.as_config(AgentConfig)
|
||||
if not (agent_config and agent_config.memories):
|
||||
continue
|
||||
try:
|
||||
self.agent_memory_managers[node_id] = MemoryManager(agent_config.memories, self.global_memories)
|
||||
self.log_manager.info(
|
||||
f"Memory manager built for node {node_id}",
|
||||
node_id=node_id,
|
||||
details={"memory_refs": [mem.name for mem in agent_config.memories]},
|
||||
)
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to create memory manager for node {node_id}: {str(e)}"
|
||||
self.log_manager.error(error_msg, node_id=node_id)
|
||||
logger = get_server_logger()
|
||||
logger.log_exception(e, error_msg, node_id=node_id)
|
||||
raise WorkflowExecutionError(error_msg, node_id=node_id)
|
||||
|
||||
def _get_execution_context(self) -> ExecutionContext:
|
||||
if self.__execution_context is None:
|
||||
global_state = dict(self.runtime_context.global_state)
|
||||
global_state.setdefault("attachment_store", self.attachment_store)
|
||||
prompt_service = self._ensure_human_prompt_service()
|
||||
global_state.setdefault("human_prompt", prompt_service)
|
||||
self.__execution_context = ExecutionContext(
|
||||
tool_manager=self.tool_manager,
|
||||
function_manager=self.function_manager,
|
||||
log_manager=self.log_manager,
|
||||
memory_managers=self.agent_memory_managers,
|
||||
thinking_managers=self.thinking_managers,
|
||||
token_tracker=self.token_tracker,
|
||||
global_state=global_state,
|
||||
workspace_hook=self.runtime_context.workspace_hook,
|
||||
human_prompt_service=prompt_service,
|
||||
cancel_event=self._cancel_event,
|
||||
)
|
||||
return self.__execution_context
|
||||
|
||||
def _build_node_executors(self) -> None:
|
||||
"""Build node executors using strategy pattern."""
|
||||
|
||||
# Create node executors
|
||||
self.node_executors = NodeExecutorFactory.create_executors(
|
||||
self._get_execution_context(),
|
||||
self.graph.subgraphs
|
||||
)
|
||||
|
||||
def _ensure_human_prompt_service(self) -> HumanPromptService:
|
||||
if self._human_prompt_service:
|
||||
return self._human_prompt_service
|
||||
|
||||
channel = resolve_prompt_channel(self.runtime_context.workspace_hook)
|
||||
if channel is None:
|
||||
channel = CliPromptChannel()
|
||||
|
||||
self._human_prompt_service = HumanPromptService(
|
||||
log_manager=self.log_manager,
|
||||
channel=channel,
|
||||
session_id=self.runtime_context.session_id,
|
||||
)
|
||||
return self._human_prompt_service
|
||||
|
||||
def _save_memories(self) -> None:
|
||||
"""Save all memories after execution."""
|
||||
for memory in self.global_memories.values():
|
||||
memory.save()
|
||||
|
||||
def run(self, task_prompt: Any) -> Dict[str, Any]:
|
||||
"""Execute the graph based on topological layers structure or cycle-aware execution."""
|
||||
self._raise_if_cancelled()
|
||||
graph_manager = GraphManager(self.graph)
|
||||
try:
|
||||
graph_manager.build_graph()
|
||||
except ConfigError as err:
|
||||
error_msg = f"Graph configuration error: {str(err)}"
|
||||
self.log_manager.logger.error(error_msg)
|
||||
raise err
|
||||
|
||||
self._prepare_edge_conditions()
|
||||
|
||||
if not self.graph.layers:
|
||||
raise ExecutionError("Graph not built. Call GraphManager.build_graph() first.")
|
||||
|
||||
# Record workflow start
|
||||
self.log_manager.record_workflow_start(self.graph.metadata)
|
||||
|
||||
# Initialize memory and thinking before execution
|
||||
self._build_memories_and_thinking()
|
||||
|
||||
# Initialize cycle manager if graph has cycles
|
||||
if self.graph.has_cycles:
|
||||
self.cycle_manager = graph_manager.get_cycle_manager()
|
||||
|
||||
self.initial_task_messages = [msg.clone() for msg in self._normalize_task_input(task_prompt)]
|
||||
|
||||
start_node_ids = set(self.graph.start_nodes)
|
||||
|
||||
# Reset all trigger states and initialize configured start nodes
|
||||
for node_id, node in self.graph.nodes.items():
|
||||
self._raise_if_cancelled()
|
||||
node.reset_triggers()
|
||||
if node_id in start_node_ids:
|
||||
node.start_triggered = True
|
||||
node.clear_input()
|
||||
for message in self.initial_task_messages:
|
||||
node.append_input(message.clone())
|
||||
|
||||
# Execute based on graph type (using strategy objects)
|
||||
if self.graph.is_majority_voting:
|
||||
strategy = MajorityVoteStrategy(
|
||||
log_manager=self.log_manager,
|
||||
nodes=self.graph.nodes,
|
||||
initial_messages=self.initial_task_messages,
|
||||
execute_node_func=self._execute_node,
|
||||
payload_to_text_func=self._payload_to_text,
|
||||
)
|
||||
self.majority_result = strategy.run()
|
||||
elif self.graph.has_cycles:
|
||||
strategy = CycleExecutionStrategy(
|
||||
log_manager=self.log_manager,
|
||||
nodes=self.graph.nodes,
|
||||
cycle_execution_order=self.graph.cycle_execution_order,
|
||||
cycle_manager=self.cycle_manager,
|
||||
execute_node_func=self._execute_node,
|
||||
)
|
||||
strategy.run()
|
||||
else:
|
||||
strategy = DagExecutionStrategy(
|
||||
log_manager=self.log_manager,
|
||||
nodes=self.graph.nodes,
|
||||
layers=self.graph.layers,
|
||||
execute_node_func=self._execute_node,
|
||||
)
|
||||
strategy.run()
|
||||
|
||||
self._raise_if_cancelled()
|
||||
|
||||
# Collect final outputs and save memories
|
||||
self._collect_all_outputs()
|
||||
|
||||
# Get the final result according to the new logic
|
||||
final_result = self.get_final_output()
|
||||
|
||||
self._save_memories()
|
||||
|
||||
# Export runtime artifacts
|
||||
archiver = ResultArchiver(self.graph, self.log_manager, self.token_tracker)
|
||||
archiver.export(final_result)
|
||||
|
||||
return self.outputs
|
||||
|
||||
def _prepare_edge_conditions(self) -> None:
|
||||
"""Compile registered edge condition types into callable evaluators."""
|
||||
context = ConditionFactoryContext(function_manager=self.function_manager, log_manager=self.log_manager)
|
||||
processor_context = PayloadProcessorFactoryContext(
|
||||
function_manager=self.edge_processor_function_manager,
|
||||
log_manager=self.log_manager,
|
||||
)
|
||||
for node in self.graph.nodes.values():
|
||||
for edge_link in node.iter_outgoing_edges():
|
||||
condition_config = edge_link.condition_config
|
||||
if not isinstance(condition_config, EdgeConditionConfig):
|
||||
raw_value = edge_link.config.get("condition", "true")
|
||||
condition_config = EdgeConditionConfig.from_dict(raw_value, path=f"{node.path}.edges")
|
||||
edge_link.condition_config = condition_config
|
||||
try:
|
||||
manager = build_edge_condition_manager(condition_config, context, self._get_execution_context())
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
error_msg = f"Failed to prepare condition '{condition_config.display_label()}': {exc}"
|
||||
self.log_manager.error(error_msg)
|
||||
logger = get_server_logger()
|
||||
logger.log_exception(exc, error_msg, condition_type=condition_config.type)
|
||||
raise WorkflowExecutionError(error_msg) from exc
|
||||
edge_link.condition_manager = manager
|
||||
label = getattr(manager, "label", None) or condition_config.display_label()
|
||||
metadata = getattr(manager, "metadata", {}) or {}
|
||||
edge_link.condition = label
|
||||
edge_link.condition_metadata = metadata
|
||||
edge_link.condition_type = condition_config.type
|
||||
|
||||
process_config = edge_link.process_config
|
||||
if process_config:
|
||||
try:
|
||||
processor = build_edge_payload_processor(process_config, processor_context)
|
||||
except Exception as exc: # pragma: no cover
|
||||
error_msg = (
|
||||
f"Failed to prepare processor '{process_config.display_label()}': {exc}"
|
||||
)
|
||||
self.log_manager.error(error_msg)
|
||||
logger = get_server_logger()
|
||||
logger.log_exception(exc, error_msg, processor_type=process_config.type)
|
||||
raise WorkflowExecutionError(error_msg) from exc
|
||||
edge_link.payload_processor = processor
|
||||
edge_link.process_type = process_config.type
|
||||
edge_link.process_metadata = getattr(processor, "metadata", {}) or {}
|
||||
processor_label = getattr(processor, "label", None)
|
||||
if processor_label:
|
||||
edge_link.config["process_label"] = processor_label
|
||||
else:
|
||||
edge_link.payload_processor = None
|
||||
edge_link.process_metadata = {}
|
||||
edge_link.process_type = None
|
||||
|
||||
def _process_edge_output(
|
||||
self,
|
||||
edge_link: EdgeLink,
|
||||
source_result: Message,
|
||||
from_node: Node
|
||||
) -> None:
|
||||
"""Perform edge instantiation behavior.
|
||||
|
||||
Edges with dynamic configuration still pass messages normally to the target
|
||||
node's input queue. Dynamic execution happens when the target node executes.
|
||||
"""
|
||||
# All edges (including dynamic ones) use standard processing to pass messages
|
||||
# Dynamic execution will happen in _execute_node when the target node runs
|
||||
|
||||
# Standard edge processing (no dynamic config)
|
||||
manager = edge_link.condition_manager
|
||||
if manager is None:
|
||||
raise WorkflowExecutionError(
|
||||
f"Edge {from_node.id}->{edge_link.target.id} is missing a condition manager"
|
||||
)
|
||||
try:
|
||||
manager.process(
|
||||
edge_link,
|
||||
source_result,
|
||||
from_node,
|
||||
self.log_manager,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
error_msg = (
|
||||
f"Edge manager failed for {from_node.id} -> {edge_link.target.id}: {exc}"
|
||||
)
|
||||
self.log_manager.error(
|
||||
error_msg,
|
||||
details={
|
||||
"condition_type": edge_link.condition_type,
|
||||
"condition_metadata": edge_link.condition_metadata,
|
||||
},
|
||||
)
|
||||
logger = get_server_logger()
|
||||
logger.log_exception(
|
||||
exc,
|
||||
error_msg,
|
||||
condition_type=edge_link.condition_type,
|
||||
condition_metadata=edge_link.condition_metadata,
|
||||
)
|
||||
raise WorkflowExecutionError(error_msg) from exc
|
||||
|
||||
|
||||
def _get_dynamic_config_for_node(self, node: Node):
|
||||
"""Get the dynamic configuration for a node from its incoming edges.
|
||||
|
||||
If multiple incoming edges have dynamic config, they must be identical
|
||||
(same type and parameters). Otherwise raises an error.
|
||||
|
||||
Returns the dynamic config if found, or None.
|
||||
"""
|
||||
from entity.configs.edge.dynamic_edge_config import DynamicEdgeConfig
|
||||
|
||||
found_configs = [] # List of (source_node_id, dynamic_config)
|
||||
|
||||
for predecessor in node.predecessors:
|
||||
for edge_link in predecessor.iter_outgoing_edges():
|
||||
if edge_link.target is node and edge_link.dynamic_config is not None:
|
||||
found_configs.append((predecessor.id, edge_link.dynamic_config))
|
||||
|
||||
if not found_configs:
|
||||
return None
|
||||
|
||||
if len(found_configs) == 1:
|
||||
return found_configs[0][1]
|
||||
|
||||
# Multiple dynamic configs found - verify they are consistent
|
||||
first_source, first_config = found_configs[0]
|
||||
for source_id, config in found_configs[1:]:
|
||||
# Check type consistency
|
||||
if config.type != first_config.type:
|
||||
raise WorkflowExecutionError(
|
||||
f"Node '{node.id}' has inconsistent dynamic configurations on incoming edges: "
|
||||
f"edge from '{first_source}' has type '{first_config.type}', "
|
||||
f"but edge from '{source_id}' has type '{config.type}'. "
|
||||
f"All dynamic edges to the same node must use the same configuration."
|
||||
)
|
||||
# Check split config consistency
|
||||
if (config.split.type != first_config.split.type or
|
||||
config.split.pattern != first_config.split.pattern or
|
||||
config.split.json_path != first_config.split.json_path):
|
||||
raise WorkflowExecutionError(
|
||||
f"Node '{node.id}' has inconsistent split configurations on incoming edges: "
|
||||
f"edges from '{first_source}' and '{source_id}' have different split settings. "
|
||||
f"All dynamic edges to the same node must use the same configuration."
|
||||
)
|
||||
# Check mode-specific config consistency
|
||||
if config.max_parallel != first_config.max_parallel:
|
||||
raise WorkflowExecutionError(
|
||||
f"Node '{node.id}' has inconsistent max_parallel on incoming edges: "
|
||||
f"edge from '{first_source}' has max_parallel={first_config.max_parallel}, "
|
||||
f"but edge from '{source_id}' has max_parallel={config.max_parallel}."
|
||||
)
|
||||
if config.type == "tree" and config.group_size != first_config.group_size:
|
||||
raise WorkflowExecutionError(
|
||||
f"Node '{node.id}' has inconsistent group_size on incoming edges: "
|
||||
f"edge from '{first_source}' has group_size={first_config.group_size}, "
|
||||
f"but edge from '{source_id}' has group_size={config.group_size}."
|
||||
)
|
||||
|
||||
return first_config
|
||||
|
||||
def _execute_with_dynamic_config(
|
||||
self,
|
||||
node: Node,
|
||||
inputs: List[Message],
|
||||
dynamic_config,
|
||||
) -> List[Message]:
|
||||
"""Execute a node with dynamic configuration from incoming edges.
|
||||
|
||||
Args:
|
||||
node: Target node to execute
|
||||
inputs: All input messages collected for this node
|
||||
dynamic_config: Dynamic configuration from the incoming edge
|
||||
|
||||
Returns:
|
||||
Output messages from dynamic execution
|
||||
"""
|
||||
# Separate inputs: dynamic edge inputs vs static (non-dynamic) edge inputs
|
||||
# Dynamic edge inputs will be split, static inputs will be replicated to all units
|
||||
dynamic_inputs: List[Message] = []
|
||||
static_inputs: List[Message] = []
|
||||
|
||||
for msg in inputs:
|
||||
if msg.metadata.get("_from_dynamic_edge"):
|
||||
dynamic_inputs.append(msg)
|
||||
else:
|
||||
static_inputs.append(msg)
|
||||
|
||||
self.log_manager.info(
|
||||
f"Executing node {node.id} with edge dynamic config ({dynamic_config.type} mode): "
|
||||
f"{len(dynamic_inputs)} dynamic inputs, {len(static_inputs)} static inputs"
|
||||
)
|
||||
|
||||
# Create node executor function
|
||||
def node_executor_func(n: Node, inp: List[Message]) -> List[Message]:
|
||||
return self._process_result(n, inp)
|
||||
|
||||
# Execute with dynamic edge executor
|
||||
dynamic_executor = DynamicEdgeExecutor(self.log_manager, node_executor_func)
|
||||
|
||||
# Pass dynamic inputs for splitting, static inputs for replication
|
||||
return dynamic_executor.execute_from_inputs(
|
||||
node, dynamic_inputs, dynamic_config, static_inputs=static_inputs
|
||||
)
|
||||
|
||||
def _execute_node(self, node: Node) -> None:
|
||||
"""Execute a single node."""
|
||||
self._raise_if_cancelled()
|
||||
with self.resource_manager.guard_node(node):
|
||||
input_results = node.input
|
||||
|
||||
# Clear incoming triggers so future iterations wait for fresh signals
|
||||
node.reset_triggers()
|
||||
|
||||
serialized_inputs = [message.to_dict(include_data=False) for message in input_results]
|
||||
|
||||
# Record node start
|
||||
self.log_manager.record_node_start(node.id, serialized_inputs, node.node_type, {
|
||||
"input_count": len(input_results),
|
||||
"predecessors": [p.id for p in node.predecessors],
|
||||
"successors": [s.id for s in node.successors]
|
||||
})
|
||||
|
||||
self.log_manager.debug(f"Processing {len(input_results)} inputs together for node {node.id}")
|
||||
|
||||
# Check if any incoming edge has dynamic configuration
|
||||
dynamic_config = self._get_dynamic_config_for_node(node)
|
||||
|
||||
# Process all inputs together in a single executor call
|
||||
with self.log_manager.node_timer(node.id):
|
||||
if dynamic_config is not None:
|
||||
raw_outputs = self._execute_with_dynamic_config(node, input_results, dynamic_config)
|
||||
else:
|
||||
raw_outputs = self._process_result(node, input_results)
|
||||
|
||||
# Process all output messages
|
||||
output_messages: List[Message] = []
|
||||
for raw_output in raw_outputs:
|
||||
msg = self._ensure_source_output(raw_output, node.id)
|
||||
node.append_output(msg)
|
||||
output_messages.append(msg)
|
||||
|
||||
# Use first output for context trace handling (backward compat)
|
||||
unified_output = output_messages[0] if output_messages else None
|
||||
|
||||
context_trace_payload = None
|
||||
context_restored = False
|
||||
if unified_output is not None and isinstance(unified_output.metadata, dict):
|
||||
context_trace_payload = unified_output.metadata.get("context_trace")
|
||||
if node.context_window != 0 and context_trace_payload:
|
||||
context_restored = self._restore_context_trace(node, context_trace_payload)
|
||||
|
||||
if node.context_window != -1:
|
||||
preserved_inputs = node.clear_input(preserve_kept=True, context_window=node.context_window)
|
||||
if preserved_inputs:
|
||||
self.log_manager.debug(
|
||||
f"Node {node.id} cleaned up its input context after execution (preserved {preserved_inputs} keep-marked inputs)"
|
||||
)
|
||||
else:
|
||||
self.log_manager.debug(
|
||||
f"Node {node.id} cleaned up its input context after execution"
|
||||
)
|
||||
|
||||
if output_messages:
|
||||
self.log_manager.debug(
|
||||
f"Node {node.id} processed {len(input_results)} inputs into {len(output_messages)} output(s)"
|
||||
)
|
||||
else:
|
||||
self.log_manager.debug(
|
||||
f"Node {node.id} produced no output; downstream edges suppressed"
|
||||
)
|
||||
|
||||
# Record node end
|
||||
output_text = ""
|
||||
if output_messages:
|
||||
if len(output_messages) == 1:
|
||||
output_text = unified_output.text_content()
|
||||
else:
|
||||
for idx, msg in enumerate(output_messages):
|
||||
output_text += f"===== OUTPUT {idx} =====\n\n" + msg.text_content() + "\n\n"
|
||||
output_role = unified_output.role.value
|
||||
output_source = unified_output.metadata.get("source")
|
||||
else:
|
||||
output_text = ""
|
||||
output_role = "none"
|
||||
output_source = None
|
||||
|
||||
self.log_manager.record_node_end(node.id, output_text if node.log_output else "", {
|
||||
"output_size": len(output_text),
|
||||
"output_count": len(output_messages),
|
||||
"output_role": output_role,
|
||||
"output_source": output_source
|
||||
})
|
||||
|
||||
# Pass results to successor nodes via edges
|
||||
# For each output message, process all edges
|
||||
for output_msg in output_messages:
|
||||
for edge_link in node.iter_outgoing_edges():
|
||||
self._process_edge_output(edge_link, output_msg, node)
|
||||
|
||||
if output_messages and node.context_window != 0 and not context_restored:
|
||||
# Use first output for pseudo edge
|
||||
pseudo_condition = EdgeConditionConfig.from_dict("true", path=f"{node.path}.pseudo_edge")
|
||||
pseudo_link = EdgeLink(target=node, trigger=False)
|
||||
pseudo_link.condition_config = pseudo_condition
|
||||
pseudo_context = ConditionFactoryContext(
|
||||
function_manager=self.function_manager,
|
||||
log_manager=self.log_manager,
|
||||
)
|
||||
pseudo_link.condition_manager = build_edge_condition_manager(pseudo_condition, pseudo_context, self._get_execution_context())
|
||||
pseudo_link.condition = pseudo_condition.display_label()
|
||||
pseudo_link.condition_type = pseudo_condition.type
|
||||
for output_msg in output_messages:
|
||||
self._process_edge_output(pseudo_link, output_msg, node)
|
||||
|
||||
def _process_result(self, node: Node, input_payload: List[Message]) -> List[Message]:
|
||||
"""Process a single input result using strategy pattern executors.
|
||||
|
||||
This method delegates to specific node executors based on node type.
|
||||
Returns a list of messages (maybe empty if node suppresses output).
|
||||
"""
|
||||
if not self.node_executors:
|
||||
raise RuntimeError("Node executors not initialized. Call _build_memories_and_thinking() first.")
|
||||
|
||||
if node.type not in self.node_executors:
|
||||
raise ValueError(f"Unsupported node type: {node.type}")
|
||||
|
||||
executor = self.node_executors[node.type]
|
||||
hook = self.runtime_context.workspace_hook
|
||||
workspace = self.runtime_context.code_workspace
|
||||
if hook:
|
||||
try:
|
||||
hook.before_node(node, workspace)
|
||||
except Exception:
|
||||
self.log_manager.warning("workspace hook before_node failed for %s", node.id)
|
||||
success = False
|
||||
try:
|
||||
result = executor.execute(node, input_payload)
|
||||
success = True
|
||||
return result
|
||||
finally:
|
||||
if hook:
|
||||
try:
|
||||
hook.after_node(node, workspace, success=success)
|
||||
except Exception:
|
||||
self.log_manager.warning("workspace hook after_node failed for %s", node.id)
|
||||
|
||||
|
||||
def _collect_all_outputs(self) -> None:
|
||||
"""Collect final outputs from all nodes, especially sink nodes."""
|
||||
all_outputs = {}
|
||||
|
||||
# For majority voting, we might want to collect differently
|
||||
if self.graph.is_majority_voting:
|
||||
# In majority voting mode, collect all outputs and the final majority result
|
||||
for node_id, node in self.graph.nodes.items():
|
||||
if node.output:
|
||||
node_output = {
|
||||
"node_id": node_id,
|
||||
"node_type": node.node_type,
|
||||
"predecessors_num": len(node.predecessors),
|
||||
"successors_num": len(node.successors),
|
||||
"results": [self._serialize_output_payload(item) for item in node.output]
|
||||
}
|
||||
all_outputs[f"node_{node_id}"] = node_output
|
||||
|
||||
# Add the majority result
|
||||
if hasattr(self, 'majority_result'):
|
||||
all_outputs["majority_result"] = self.majority_result
|
||||
else:
|
||||
# Collect outputs from all nodes normally
|
||||
for node_id, node in self.graph.nodes.items():
|
||||
if node.output:
|
||||
node_output = {
|
||||
"node_id": node_id,
|
||||
"node_type": node.node_type,
|
||||
"predecessors_num": len(node.predecessors),
|
||||
"successors_num": len(node.successors),
|
||||
"results": [self._serialize_output_payload(item) for item in node.output]
|
||||
}
|
||||
all_outputs[f"node_{node_id}"] = node_output
|
||||
|
||||
# Add graph summary
|
||||
all_outputs["graph_summary"] = {
|
||||
"total_nodes": len(self.graph.nodes),
|
||||
"total_edges": len(self.graph.edges),
|
||||
"total_transmissions": len([k for k in self.outputs.keys() if "->" in k]),
|
||||
"layers": len(self.graph.layers),
|
||||
"execution_completed": True,
|
||||
"is_majority_voting": self.graph.is_majority_voting
|
||||
}
|
||||
|
||||
self.outputs.update(all_outputs)
|
||||
|
||||
def get_final_output(self) -> str:
|
||||
final_message = self.get_final_output_message()
|
||||
return final_message.text_content() if final_message else ""
|
||||
|
||||
def get_final_output_message(self) -> Message | None:
|
||||
if self.graph.is_majority_voting:
|
||||
if self.majority_result is None:
|
||||
return None
|
||||
if isinstance(self.majority_result, Message):
|
||||
return self.majority_result.clone()
|
||||
return self._create_message(MessageRole.ASSISTANT, str(self.majority_result), "MAJORITY_VOTE")
|
||||
|
||||
final_node = self._get_final_node()
|
||||
if not final_node:
|
||||
return None
|
||||
if final_node.output:
|
||||
value = final_node.output[-1]
|
||||
if isinstance(value, Message):
|
||||
return value.clone()
|
||||
return self._create_message(MessageRole.ASSISTANT, str(value), final_node.id)
|
||||
return None
|
||||
|
||||
def get_final_output_messages(self) -> List[Message]:
|
||||
"""Return all messages from the final node."""
|
||||
if self.graph.is_majority_voting:
|
||||
msg = self.get_final_output_message()
|
||||
return [msg] if msg else []
|
||||
|
||||
final_node = self._get_final_node()
|
||||
if not final_node:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for value in final_node.output:
|
||||
if isinstance(value, Message):
|
||||
results.append(value.clone())
|
||||
else:
|
||||
results.append(self._create_message(MessageRole.ASSISTANT, str(value), final_node.id))
|
||||
return results
|
||||
|
||||
def _get_final_node(self) -> Node:
|
||||
"""Return the explicitly configured end node, or sink node as fallback."""
|
||||
end_node_ids = self.graph.config.definition.end_nodes
|
||||
|
||||
if end_node_ids:
|
||||
for end_node_id in end_node_ids:
|
||||
if end_node_id in self.graph.nodes:
|
||||
node = self.graph.nodes[end_node_id]
|
||||
# Check if node has output
|
||||
if node.output:
|
||||
return node
|
||||
|
||||
# Fallback to default behavior - return sink node
|
||||
sink_node = [node for node in self.graph.nodes.values() if not node.successors]
|
||||
return sink_node[0] if sink_node else None
|
||||
|
||||
def _restore_context_trace(self, node: Node, trace_payload: Any) -> bool:
|
||||
if not isinstance(trace_payload, list):
|
||||
return False
|
||||
|
||||
restored = 0
|
||||
for entry in trace_payload:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
try:
|
||||
message = Message.from_dict(entry)
|
||||
if message.role not in [MessageRole.USER, MessageRole.ASSISTANT]:
|
||||
continue
|
||||
except Exception as exc:
|
||||
self.log_manager.warning(
|
||||
f"Failed to deserialize context trace for node {node.id}: {exc}"
|
||||
)
|
||||
continue
|
||||
node.append_input(self._ensure_source(message, node.id))
|
||||
restored += 1
|
||||
|
||||
if restored:
|
||||
self.log_manager.debug(
|
||||
f"Node {node.id} preserved {restored} messages from its tool execution trace"
|
||||
)
|
||||
return restored > 0
|
||||
|
||||
def _payload_to_text(self, payload: Any) -> str:
|
||||
if isinstance(payload, Message):
|
||||
return payload.text_content()
|
||||
if payload is None:
|
||||
return ""
|
||||
return str(payload)
|
||||
|
||||
def _serialize_output_payload(self, payload: Any) -> Any:
|
||||
if isinstance(payload, Message):
|
||||
return {"type": "message", "payload": payload.to_dict(include_data=False)}
|
||||
return {"type": "text", "payload": str(payload)}
|
||||
|
||||
def _normalize_task_input(self, raw_input: Any) -> List[Message]:
|
||||
if isinstance(raw_input, list):
|
||||
messages: List[Message] = []
|
||||
for item in raw_input:
|
||||
if isinstance(item, Message):
|
||||
messages.append(self._ensure_source(item, "TASK"))
|
||||
elif isinstance(item, str):
|
||||
messages.append(self._create_message(MessageRole.USER, item, "TASK"))
|
||||
return messages or [self._create_message(MessageRole.USER, "", "TASK")]
|
||||
if isinstance(raw_input, Message):
|
||||
return [self._ensure_source(raw_input, "TASK")]
|
||||
return [self._create_message(MessageRole.USER, str(raw_input), "TASK")]
|
||||
|
||||
def _ensure_source(self, message: Message, default_source: str) -> Message:
|
||||
cloned = message.clone()
|
||||
metadata = dict(cloned.metadata)
|
||||
metadata.setdefault("source", default_source)
|
||||
cloned.metadata = metadata
|
||||
return cloned
|
||||
|
||||
def _create_message(self, role: MessageRole, content: str, source: str) -> Message:
|
||||
return Message(role=role, content=content, metadata={"source": source})
|
||||
|
||||
def _ensure_source_output(self, message: Any, node_id: str) -> Message:
|
||||
if not isinstance(message, Message):
|
||||
return self._create_message(MessageRole.ASSISTANT, str(message), node_id)
|
||||
cloned = message.clone()
|
||||
metadata = dict(message.metadata)
|
||||
metadata.setdefault("source", node_id)
|
||||
cloned.metadata = metadata
|
||||
return cloned
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
"""Runtime context for workflow graphs.
|
||||
|
||||
This module stores execution-time state and business logic for graphs.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import yaml
|
||||
|
||||
from entity.configs import Node
|
||||
from entity.graph_config import GraphConfig
|
||||
|
||||
|
||||
class GraphContext:
|
||||
"""Runtime context for a workflow graph (state + business logic).
|
||||
|
||||
Differences from ``GraphConfig``:
|
||||
- ``GraphConfig`` is immutable configuration data.
|
||||
- ``GraphContext`` is mutable runtime state with dynamic execution data.
|
||||
|
||||
Attributes:
|
||||
config: Graph configuration
|
||||
nodes: Mapping of ``node_id`` to ``Node``
|
||||
edges: List of edges
|
||||
layers: Topological layer layout
|
||||
outputs: Node outputs captured during execution
|
||||
topology: Topological ordering list
|
||||
subgraphs: Mapping of ``node_id`` to nested ``GraphContext``
|
||||
has_cycles: Whether the graph contains cycles
|
||||
cycle_execution_order: Execution order for cycles
|
||||
directory: Output directory for artifacts
|
||||
depth: Graph depth
|
||||
"""
|
||||
|
||||
def __init__(self, config: GraphConfig) -> None:
|
||||
"""Initialize the graph context.
|
||||
|
||||
Args:
|
||||
config: Graph configuration
|
||||
"""
|
||||
self.config = config
|
||||
self.vars: Dict[str, Any] = dict(config.vars)
|
||||
|
||||
# Graph structure
|
||||
self.nodes: Dict[str, Node] = {}
|
||||
self.edges: List[Dict[str, Any]] = []
|
||||
self.layers: List[List[str]] = []
|
||||
self.topology: List[str] = []
|
||||
self.depth: int = 0
|
||||
self.start_nodes: List[str] = []
|
||||
self.explicit_start_nodes: List[str] = []
|
||||
|
||||
# Runtime state
|
||||
self.outputs: Dict[str, str] = {}
|
||||
self.subgraphs: Dict[str, "GraphContext"] = {}
|
||||
|
||||
# Cycle support
|
||||
self.has_cycles: bool = False
|
||||
self.cycle_execution_order: List[Dict[str, Any]] = []
|
||||
|
||||
# Output directory
|
||||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
fixed_output_dir = bool(config.metadata.get("fixed_output_dir"))
|
||||
if fixed_output_dir or "session_" in config.name:
|
||||
self.directory = config.output_root / config.name
|
||||
else:
|
||||
self.directory = config.output_root / f"{config.name}_{timestamp}"
|
||||
self.directory.mkdir(parents=True, exist_ok=True)
|
||||
# Voting mode flag
|
||||
self.is_majority_voting: bool = config.is_majority_voting
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""Return the project name."""
|
||||
return self.config.name
|
||||
|
||||
@property
|
||||
def log_level(self):
|
||||
"""Return the configured log level."""
|
||||
return self.config.log_level
|
||||
|
||||
@property
|
||||
def metadata(self) -> Dict[str, Any]:
|
||||
"""Return graph metadata."""
|
||||
return self.config.metadata
|
||||
|
||||
@metadata.setter
|
||||
def metadata(self, value: Dict[str, Any]) -> None:
|
||||
"""Set graph metadata."""
|
||||
self.config.metadata = value
|
||||
|
||||
def record(self, outputs: Dict[str, Any]) -> None:
|
||||
"""Persist execution results to disk.
|
||||
|
||||
Args:
|
||||
outputs: Mapping of node outputs
|
||||
"""
|
||||
self.outputs = outputs
|
||||
# self.directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Persist node outputs
|
||||
outputs_path = self.directory / "node_outputs.yaml"
|
||||
if self.outputs:
|
||||
with outputs_path.open("w", encoding="utf-8") as handle:
|
||||
yaml.dump(self.outputs, handle, allow_unicode=True, sort_keys=False)
|
||||
|
||||
# Persist workflow summary
|
||||
summary = {
|
||||
"project": self.config.name,
|
||||
"organization": self.config.get_organization(),
|
||||
"design_path": self.config.get_source_path(),
|
||||
"metadata": self.config.metadata,
|
||||
}
|
||||
summary_path = self.directory / "workflow_summary.yaml"
|
||||
with summary_path.open("w", encoding="utf-8") as handle:
|
||||
yaml.dump(summary, handle, allow_unicode=True, sort_keys=False)
|
||||
|
||||
def final_message(self) -> str:
|
||||
"""Build the final completion string.
|
||||
|
||||
Returns:
|
||||
Completion message text
|
||||
"""
|
||||
if not self.outputs:
|
||||
return "Workflow finished with no outputs."
|
||||
|
||||
sink_nodes = [node_id for node_id, node in self.nodes.items() if not node.successors]
|
||||
return (
|
||||
f"Workflow finished with {len(self.outputs)} node outputs"
|
||||
f" ({len(sink_nodes)} terminal nodes)."
|
||||
)
|
||||
|
||||
def get_sink_nodes(self) -> List[Node]:
|
||||
"""Return all leaf nodes (nodes without successors)."""
|
||||
return [node for node in self.nodes.values() if not node.successors]
|
||||
|
||||
def get_source_nodes(self) -> List[Node]:
|
||||
"""Return all source nodes (nodes without predecessors)."""
|
||||
return [node for node in self.nodes.values() if not node.predecessors]
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert the graph context to a dictionary."""
|
||||
return {
|
||||
"config": self.config.to_dict(),
|
||||
"nodes": {node_id: node.to_dict() for node_id, node in self.nodes.items()},
|
||||
"edges": list(self.edges),
|
||||
"layers": list(self.layers),
|
||||
"topology": list(self.topology),
|
||||
"depth": self.depth,
|
||||
"has_cycles": self.has_cycles,
|
||||
"start_nodes": list(self.start_nodes),
|
||||
"explicit_start_nodes": list(self.explicit_start_nodes),
|
||||
"outputs": dict(self.outputs),
|
||||
}
|
||||
Executable
+350
@@ -0,0 +1,350 @@
|
||||
"""Graph management and construction utilities for workflow graphs."""
|
||||
|
||||
from typing import Dict, List, Set, Any
|
||||
import copy
|
||||
|
||||
from entity.configs import ConfigError, SubgraphConfig
|
||||
from entity.configs.edge.edge_condition import EdgeConditionConfig
|
||||
from entity.configs.base import extend_path
|
||||
from entity.configs.node.subgraph import SubgraphFileConfig, SubgraphInlineConfig
|
||||
from workflow.cycle_manager import CycleManager
|
||||
from workflow.subgraph_loader import load_subgraph_config
|
||||
from workflow.topology_builder import GraphTopologyBuilder
|
||||
from utils.env_loader import build_env_var_map
|
||||
from utils.vars_resolver import resolve_mapping_with_vars
|
||||
from workflow.graph_context import GraphContext
|
||||
|
||||
|
||||
class GraphManager:
|
||||
"""Manages graph construction, cycle detection, and execution order determination."""
|
||||
|
||||
def __init__(self, graph: "GraphContext") -> None:
|
||||
"""Initialize GraphManager with a GraphContext instance."""
|
||||
self.graph = graph
|
||||
self.cycle_manager = CycleManager()
|
||||
|
||||
def build_graph_structure(self) -> None:
|
||||
"""Build the complete graph structure including nodes, edges, and layers."""
|
||||
self._instantiate_nodes()
|
||||
self._initiate_edges()
|
||||
self._determine_start_nodes()
|
||||
self._warn_on_untriggerable_nodes()
|
||||
self._build_topology_and_metadata()
|
||||
|
||||
def _instantiate_nodes(self) -> None:
|
||||
"""Instantiate all nodes from configuration."""
|
||||
self.graph.nodes.clear()
|
||||
for node_def in self.graph.config.get_node_definitions():
|
||||
node_id = node_def.id
|
||||
if node_id in self.graph.nodes:
|
||||
print(f"Duplicated node id detected: {node_id}")
|
||||
continue
|
||||
node_instance = copy.deepcopy(node_def)
|
||||
node_instance.predecessors = []
|
||||
node_instance.successors = []
|
||||
node_instance._outgoing_edges = []
|
||||
node_instance.vars = dict(self.graph.vars)
|
||||
self.graph.nodes[node_id] = node_instance
|
||||
|
||||
if node_instance.node_type == "subgraph":
|
||||
self._build_subgraph(node_id)
|
||||
|
||||
def _build_subgraph(self, node_id: str) -> None:
|
||||
"""Build a subgraph for the given node ID."""
|
||||
from entity.graph_config import GraphConfig
|
||||
from workflow.graph_context import GraphContext
|
||||
|
||||
subgraph_config_data = self.graph.nodes[node_id].as_config(SubgraphConfig)
|
||||
if not subgraph_config_data:
|
||||
return
|
||||
|
||||
parent_source = self.graph.config.get_source_path()
|
||||
subgraph_vars: Dict[str, Any] = {}
|
||||
|
||||
if subgraph_config_data.type == "config":
|
||||
inline_cfg = subgraph_config_data.as_config(SubgraphInlineConfig)
|
||||
if not inline_cfg:
|
||||
raise ConfigError(
|
||||
f"Inline subgraph configuration missing for node '{node_id}'",
|
||||
subgraph_config_data.path,
|
||||
)
|
||||
config_payload = copy.deepcopy(inline_cfg.graph)
|
||||
source_path = parent_source
|
||||
elif subgraph_config_data.type == "file":
|
||||
file_cfg = subgraph_config_data.as_config(SubgraphFileConfig)
|
||||
if not file_cfg:
|
||||
raise ConfigError(
|
||||
f"File subgraph configuration missing for node '{node_id}'",
|
||||
subgraph_config_data.path,
|
||||
)
|
||||
config_payload, subgraph_vars, source_path = load_subgraph_config(
|
||||
file_cfg.file_path,
|
||||
parent_source=parent_source,
|
||||
)
|
||||
else:
|
||||
raise ConfigError(
|
||||
f"Unsupported subgraph configuration on node '{node_id}'",
|
||||
subgraph_config_data.path,
|
||||
)
|
||||
|
||||
combined_vars = dict(self.graph.config.vars)
|
||||
combined_vars.update(subgraph_vars)
|
||||
|
||||
resolve_mapping_with_vars(
|
||||
config_payload,
|
||||
env_lookup=build_env_var_map(),
|
||||
vars_map=combined_vars,
|
||||
path=f"subgraph[{node_id}]",
|
||||
)
|
||||
|
||||
if config_payload.get("log_level", None) is None:
|
||||
config_payload["log_level"] = self.graph.log_level.value
|
||||
|
||||
subgraph_config = GraphConfig.from_dict(
|
||||
config=config_payload,
|
||||
name=f"{self.graph.name}_{node_id}_subgraph",
|
||||
output_root=self.graph.config.output_root,
|
||||
source_path=source_path,
|
||||
vars=combined_vars,
|
||||
)
|
||||
|
||||
subgraph = GraphContext(config=subgraph_config)
|
||||
|
||||
subgraph_manager = GraphManager(subgraph)
|
||||
subgraph_manager.build_graph_structure()
|
||||
|
||||
self.graph.subgraphs[node_id] = subgraph
|
||||
|
||||
def _initiate_edges(self) -> None:
|
||||
"""Initialize edges and determine layers or cycle execution order."""
|
||||
# For majority voting mode, there are no edges by design
|
||||
if self.graph.is_majority_voting:
|
||||
print("Majority voting mode detected - skipping edge initialization")
|
||||
self.graph.edges = []
|
||||
|
||||
# For majority voting, all nodes are independent and can be executed in parallel
|
||||
# Create a single layer with all nodes
|
||||
all_node_ids = list(self.graph.nodes.keys())
|
||||
self.graph.layers = [all_node_ids]
|
||||
return
|
||||
|
||||
self.graph.edges = []
|
||||
for edge_config in self.graph.config.get_edge_definitions():
|
||||
src = edge_config.source
|
||||
dst = edge_config.target
|
||||
if src not in self.graph.nodes or dst not in self.graph.nodes:
|
||||
print(f"Edge references unknown node: {src}->{dst}")
|
||||
continue
|
||||
|
||||
condition_config = edge_config.condition
|
||||
if condition_config is None:
|
||||
condition_config = EdgeConditionConfig.from_dict("true", path=extend_path(edge_config.path, "condition"))
|
||||
condition_value = condition_config.to_external_value()
|
||||
process_config = edge_config.process
|
||||
process_value = process_config.to_external_value() if process_config else None
|
||||
dynamic_config = edge_config.dynamic
|
||||
payload = {
|
||||
"trigger": edge_config.trigger,
|
||||
"condition": condition_value,
|
||||
"condition_config": condition_config,
|
||||
"condition_label": condition_config.display_label(),
|
||||
"condition_type": condition_config.type,
|
||||
"carry_data": edge_config.carry_data,
|
||||
"keep_message": edge_config.keep_message,
|
||||
"clear_context": edge_config.clear_context,
|
||||
"clear_kept_context": edge_config.clear_kept_context,
|
||||
"process_config": process_config,
|
||||
"process": process_value,
|
||||
"process_type": process_config.type if process_config else None,
|
||||
"dynamic_config": dynamic_config,
|
||||
}
|
||||
self.graph.nodes[src].add_successor(self.graph.nodes[dst], payload)
|
||||
self.graph.nodes[dst].add_predecessor(self.graph.nodes[src])
|
||||
self.graph.edges.append({
|
||||
"from": src,
|
||||
"to": dst,
|
||||
"trigger": edge_config.trigger,
|
||||
"condition": condition_value,
|
||||
"condition_type": condition_config.type,
|
||||
"carry_data": edge_config.carry_data,
|
||||
"keep_message": edge_config.keep_message,
|
||||
"clear_context": edge_config.clear_context,
|
||||
"clear_kept_context": edge_config.clear_kept_context,
|
||||
"process": process_value,
|
||||
"process_type": process_config.type if process_config else None,
|
||||
"dynamic": dynamic_config is not None,
|
||||
})
|
||||
|
||||
# Check for cycles and build appropriate execution structure
|
||||
cycles = self._detect_cycles()
|
||||
self.graph.has_cycles = len(cycles) > 0
|
||||
|
||||
if self.graph.has_cycles:
|
||||
print(f"Detected {len(cycles)} cycle(s) in the workflow graph.")
|
||||
self.graph.layers = self._build_cycle_execution_order(cycles)
|
||||
else:
|
||||
self.graph.layers = self._build_dag_layers()
|
||||
|
||||
def _detect_cycles(self) -> List[Set[str]]:
|
||||
"""Detect cycles in the graph using GraphTopologyBuilder."""
|
||||
return GraphTopologyBuilder.detect_cycles(self.graph.nodes)
|
||||
|
||||
def _build_dag_layers(self) -> List[List[str]]:
|
||||
"""Build layers for DAG (Directed Acyclic Graph) using GraphTopologyBuilder."""
|
||||
layers_with_items = GraphTopologyBuilder.build_dag_layers(self.graph.nodes)
|
||||
|
||||
# Convert format to be compatible with existing code
|
||||
layers = [
|
||||
[item["node_id"] for item in layer]
|
||||
for layer in layers_with_items
|
||||
]
|
||||
|
||||
print(f"layers: {layers}")
|
||||
|
||||
if len(set(node_id for layer in layers for node_id in layer)) != len(self.graph.nodes):
|
||||
print("Detected a cycle in the workflow graph; a DAG is required.")
|
||||
|
||||
return layers
|
||||
|
||||
def _build_cycle_execution_order(self, cycles: List[Set[str]]) -> List[List[str]]:
|
||||
"""Build execution order for graphs with cycles using super-node abstraction and GraphTopologyBuilder."""
|
||||
# Initialize cycle manager
|
||||
self.cycle_manager.initialize_cycles(cycles, self.graph.nodes)
|
||||
|
||||
# Use GraphTopologyBuilder to create super-node graph
|
||||
super_node_graph = GraphTopologyBuilder.create_super_node_graph(
|
||||
self.graph.nodes,
|
||||
self.graph.edges,
|
||||
cycles
|
||||
)
|
||||
|
||||
# Use GraphTopologyBuilder for topological sorting
|
||||
execution_order = GraphTopologyBuilder.topological_sort_super_nodes(
|
||||
super_node_graph,
|
||||
cycles
|
||||
)
|
||||
|
||||
# Enrich execution_order with entry_nodes and exit_edges from cycle_manager
|
||||
for layer in execution_order:
|
||||
for item in layer:
|
||||
if item["type"] == "cycle":
|
||||
cycle_id = item["cycle_id"]
|
||||
cycle_info = self.cycle_manager.cycles[cycle_id]
|
||||
item["entry_nodes"] = list(cycle_info.entry_nodes)
|
||||
item["exit_edges"] = cycle_info.exit_edges
|
||||
|
||||
self.graph.cycle_execution_order = execution_order
|
||||
|
||||
# Return a simplified layer structure for compatibility
|
||||
return [["__CYCLE_AWARE__"]] # Special marker for cycle-aware execution
|
||||
|
||||
def _build_topology_and_metadata(self) -> None:
|
||||
"""Build topology and metadata for the graph."""
|
||||
self.graph.topology = [node_id for layer in self.graph.layers for node_id in layer]
|
||||
self.graph.depth = len(self.graph.layers) - 1 if self.graph.layers else 0
|
||||
self.graph.metadata = self._build_metadata()
|
||||
|
||||
def _build_metadata(self) -> Dict[str, Any]:
|
||||
"""Build metadata for the graph."""
|
||||
graph_def = self.graph.config.definition
|
||||
catalog: Dict[str, Any] = {}
|
||||
for node_id, node in self.graph.nodes.items():
|
||||
catalog[node_id] = {
|
||||
"type": node.node_type,
|
||||
"description": node.description,
|
||||
"model_name": node.model_name,
|
||||
"role": node.role,
|
||||
"tools": node.tools,
|
||||
"memories": node.memories,
|
||||
"params": node.params,
|
||||
}
|
||||
|
||||
return {
|
||||
"design_id": graph_def.id,
|
||||
"node_count": len(self.graph.nodes),
|
||||
"edge_count": len(self.graph.edges),
|
||||
"start": list(self.graph.start_nodes),
|
||||
"end": graph_def.end_nodes,
|
||||
"catalog": catalog,
|
||||
"topology": self.graph.topology,
|
||||
"layers": self.graph.layers,
|
||||
}
|
||||
|
||||
def _determine_start_nodes(self) -> None:
|
||||
"""Determine the effective set of start nodes (explicit only)."""
|
||||
definition = self.graph.config.definition
|
||||
explicit_ordered = list(definition.start_nodes)
|
||||
explicit_set = set(explicit_ordered)
|
||||
|
||||
# if explicit_ordered and not self.graph.has_cycles:
|
||||
# raise ConfigError(
|
||||
# "start nodes can only be specified for graphs that contain cycles",
|
||||
# extend_path(definition.path, "start"),
|
||||
# )
|
||||
|
||||
if explicit_set:
|
||||
cycle_path = extend_path(definition.path, "start")
|
||||
for node_id in explicit_ordered:
|
||||
if node_id not in self.graph.nodes:
|
||||
raise ConfigError(
|
||||
f"start node '{node_id}' not defined in nodes",
|
||||
cycle_path,
|
||||
)
|
||||
|
||||
cycle_id = self.cycle_manager.node_to_cycle.get(node_id)
|
||||
if cycle_id is None:
|
||||
continue
|
||||
cycle_info = self.cycle_manager.cycles.get(cycle_id)
|
||||
if cycle_info is None:
|
||||
raise ConfigError(
|
||||
f"cycle data missing for start node '{node_id}'",
|
||||
cycle_path,
|
||||
)
|
||||
|
||||
if cycle_info.configured_entry_node and cycle_info.configured_entry_node != node_id:
|
||||
raise ConfigError(
|
||||
f"cycle '{cycle_id}' already has start node '{cycle_info.configured_entry_node}'",
|
||||
cycle_path,
|
||||
)
|
||||
|
||||
cycle_info.configured_entry_node = node_id
|
||||
|
||||
if not explicit_ordered:
|
||||
raise ConfigError(
|
||||
"Unable to determine a start node for this graph. Configure at least one Start Node via Configure Graph > Advanced Settings > Start Node > input node ID.",
|
||||
extend_path(definition.path, "start"),
|
||||
)
|
||||
|
||||
self.graph.start_nodes = explicit_ordered
|
||||
self.graph.explicit_start_nodes = explicit_ordered
|
||||
|
||||
def _warn_on_untriggerable_nodes(self) -> None:
|
||||
"""Emit warnings for nodes that cannot be triggered by any predecessor."""
|
||||
start_nodes = set(self.graph.start_nodes or [])
|
||||
for node_id, node in self.graph.nodes.items():
|
||||
if not node.predecessors:
|
||||
continue
|
||||
if node_id in start_nodes:
|
||||
continue
|
||||
|
||||
has_triggerable_edge = False
|
||||
for predecessor in node.predecessors:
|
||||
for edge_link in predecessor.iter_outgoing_edges():
|
||||
if edge_link.target is node and edge_link.trigger:
|
||||
has_triggerable_edge = True
|
||||
break
|
||||
if has_triggerable_edge:
|
||||
break
|
||||
|
||||
if not has_triggerable_edge:
|
||||
print(
|
||||
f"Warning: node '{node_id}' has no triggerable incoming edges and will never execute."
|
||||
)
|
||||
|
||||
def get_cycle_manager(self) -> CycleManager:
|
||||
"""Get the cycle manager instance."""
|
||||
return self.cycle_manager
|
||||
|
||||
def build_graph(self) -> None:
|
||||
"""Build graph structure only (no memory/thinking initialization)."""
|
||||
self.build_graph_structure()
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
"""Hook implementations for workflow runtime."""
|
||||
|
||||
from .workspace_artifact import WorkspaceArtifactHook, WorkspaceArtifact
|
||||
|
||||
__all__ = ["WorkspaceArtifactHook", "WorkspaceArtifact"]
|
||||
Executable
+274
@@ -0,0 +1,274 @@
|
||||
"""Hook that scans a node workspace for newly created files."""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple
|
||||
|
||||
from entity.configs import Node
|
||||
from entity.messages import MessageBlockType
|
||||
from utils.attachments import AttachmentRecord, AttachmentStore
|
||||
from utils.human_prompt import PromptChannel
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkspaceArtifact:
|
||||
"""Represents a file artifact detected by the workspace hook."""
|
||||
|
||||
node_id: str
|
||||
attachment_id: str
|
||||
file_name: str
|
||||
relative_path: str
|
||||
absolute_path: str
|
||||
mime_type: Optional[str]
|
||||
size: Optional[int]
|
||||
sha256: Optional[str]
|
||||
data_uri: Optional[str]
|
||||
created_at: float
|
||||
change_type: str
|
||||
extra: Dict[str, object]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FileSignature:
|
||||
sha256: str
|
||||
size: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class _TrackedEntry:
|
||||
sha256: str
|
||||
attachment_id: str
|
||||
absolute_path: str
|
||||
mime_type: Optional[str]
|
||||
size: Optional[int]
|
||||
data_uri: Optional[str]
|
||||
|
||||
|
||||
class WorkspaceArtifactHook:
|
||||
"""Detects workspace file changes for selected node types."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
attachment_store: AttachmentStore,
|
||||
emit_callback: Callable[[List[WorkspaceArtifact]], None],
|
||||
node_types: Optional[Sequence[str]] = None,
|
||||
exclude_dirs: Optional[Sequence[str]] = None,
|
||||
max_files_scanned: int = 500,
|
||||
max_bytes_scanned: int = 500 * 1024 * 1024,
|
||||
prompt_channel: Optional[PromptChannel] = None,
|
||||
) -> None:
|
||||
self.attachment_store = attachment_store
|
||||
self.emit_callback = emit_callback
|
||||
self.node_types: Set[str] = set(node_types or {"python", "agent"})
|
||||
self.exclude_dirs = set(exclude_dirs or {"attachments", "__pycache__"})
|
||||
self.max_files_scanned = max_files_scanned
|
||||
self.max_bytes_scanned = max_bytes_scanned
|
||||
self.logger = logging.getLogger(__name__)
|
||||
self._snapshots: Dict[str, Dict[str, _FileSignature]] = {}
|
||||
self._last_emitted: Dict[str, _TrackedEntry] = {}
|
||||
self.prompt_channel = prompt_channel
|
||||
|
||||
def can_handle(self, node: Node) -> bool:
|
||||
return node.node_type in self.node_types
|
||||
|
||||
def get_prompt_channel(self) -> Optional[PromptChannel]:
|
||||
return self.prompt_channel
|
||||
|
||||
def before_node(self, node: Node, workspace: Path) -> None:
|
||||
if not self.can_handle(node):
|
||||
return
|
||||
snapshot, _ = self._snapshot(workspace)
|
||||
self._snapshots[node.id] = snapshot
|
||||
|
||||
def after_node(
|
||||
self,
|
||||
node: Node,
|
||||
workspace: Path,
|
||||
*,
|
||||
success: bool,
|
||||
) -> None:
|
||||
if not success or not self.can_handle(node):
|
||||
self._snapshots.pop(node.id, None)
|
||||
return
|
||||
|
||||
before = self._snapshots.pop(node.id, {})
|
||||
after, truncated = self._snapshot(workspace)
|
||||
if not after and not self._last_emitted:
|
||||
return
|
||||
|
||||
changed_paths = [
|
||||
Path(path_str)
|
||||
for path_str, signature in after.items()
|
||||
if path_str not in before or before[path_str].sha256 != signature.sha256
|
||||
]
|
||||
|
||||
artifacts: List[WorkspaceArtifact] = []
|
||||
for relative_path in changed_paths:
|
||||
signature = after[str(relative_path)]
|
||||
full_path = workspace / relative_path
|
||||
if not full_path.exists() or not full_path.is_file():
|
||||
continue
|
||||
try:
|
||||
tracked = self._last_emitted.get(str(relative_path))
|
||||
change_type = "created" if tracked is None else "updated"
|
||||
record = self._register_artifact(
|
||||
full_path,
|
||||
relative_path,
|
||||
node,
|
||||
attachment_id=tracked.attachment_id if tracked else None,
|
||||
)
|
||||
except Exception as exc:
|
||||
self.logger.warning(
|
||||
"Failed to register artifact %s for node %s: %s",
|
||||
relative_path,
|
||||
node.id,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
artifacts.append(
|
||||
self._to_artifact(
|
||||
record,
|
||||
node,
|
||||
relative_path,
|
||||
full_path,
|
||||
change_type=change_type,
|
||||
)
|
||||
)
|
||||
self._last_emitted[str(relative_path)] = _TrackedEntry(
|
||||
sha256=signature.sha256,
|
||||
attachment_id=record.ref.attachment_id or "",
|
||||
absolute_path=str(full_path),
|
||||
mime_type=record.ref.mime_type,
|
||||
size=record.ref.size,
|
||||
data_uri=record.ref.data_uri,
|
||||
)
|
||||
|
||||
if not truncated:
|
||||
deleted_paths = [
|
||||
relative_path
|
||||
for relative_path in list(self._last_emitted.keys())
|
||||
if relative_path not in after
|
||||
]
|
||||
for relative_path in deleted_paths:
|
||||
tracked = self._last_emitted.pop(relative_path, None)
|
||||
if not tracked:
|
||||
continue
|
||||
artifacts.append(
|
||||
WorkspaceArtifact(
|
||||
node_id=node.id,
|
||||
attachment_id=tracked.attachment_id,
|
||||
file_name=Path(relative_path).name,
|
||||
relative_path=relative_path,
|
||||
absolute_path=tracked.absolute_path,
|
||||
mime_type=tracked.mime_type,
|
||||
size=tracked.size,
|
||||
sha256=tracked.sha256,
|
||||
data_uri=tracked.data_uri,
|
||||
created_at=time.time(),
|
||||
change_type="deleted",
|
||||
extra={
|
||||
"hook": "workspace_scan",
|
||||
"relative_path": relative_path,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
if artifacts:
|
||||
self.emit_callback(artifacts)
|
||||
|
||||
def _snapshot(self, workspace: Path) -> Tuple[Dict[str, _FileSignature], bool]:
|
||||
entries: Dict[str, _FileSignature] = {}
|
||||
total_bytes = 0
|
||||
file_count = 0
|
||||
for root, dirs, files in os.walk(workspace):
|
||||
rel_root = Path(root).relative_to(workspace)
|
||||
dirs[:] = [d for d in dirs if not self._is_excluded(rel_root / d)]
|
||||
for filename in files:
|
||||
rel_path = rel_root / filename
|
||||
if self._is_excluded(rel_path):
|
||||
continue
|
||||
full_path = Path(root) / filename
|
||||
try:
|
||||
stat = full_path.stat()
|
||||
sha256 = self._hash_file(full_path)
|
||||
except OSError:
|
||||
continue
|
||||
file_count += 1
|
||||
total_bytes += stat.st_size
|
||||
entries[str(rel_path)] = _FileSignature(sha256=sha256, size=stat.st_size)
|
||||
if file_count >= self.max_files_scanned or total_bytes >= self.max_bytes_scanned:
|
||||
self.logger.warning(
|
||||
"Workspace scan truncated (files=%s total_bytes=%s) for session %s",
|
||||
file_count,
|
||||
total_bytes,
|
||||
)
|
||||
return entries, True
|
||||
return entries, False
|
||||
|
||||
def _is_excluded(self, rel_path: Path) -> bool:
|
||||
if not rel_path.parts:
|
||||
return False
|
||||
return rel_path.parts[0] in self.exclude_dirs
|
||||
|
||||
def _register_artifact(
|
||||
self,
|
||||
full_path: Path,
|
||||
relative_path: Path,
|
||||
node: Node,
|
||||
*,
|
||||
attachment_id: Optional[str] = None,
|
||||
) -> AttachmentRecord:
|
||||
mime_type = mimetypes.guess_type(relative_path.name)[0] or "application/octet-stream"
|
||||
return self.attachment_store.register_file(
|
||||
full_path,
|
||||
kind=MessageBlockType.from_mime_type(mime_type),
|
||||
mime_type=mime_type,
|
||||
display_name=full_path.name,
|
||||
copy_file=False,
|
||||
persist=True,
|
||||
deduplicate=False,
|
||||
attachment_id=attachment_id,
|
||||
extra={
|
||||
"node_id": node.id,
|
||||
"relative_path": str(relative_path),
|
||||
"hook": "workspace_scan",
|
||||
},
|
||||
)
|
||||
|
||||
def _to_artifact(
|
||||
self,
|
||||
record: AttachmentRecord,
|
||||
node: Node,
|
||||
relative_path: Path,
|
||||
full_path: Path,
|
||||
*,
|
||||
change_type: str,
|
||||
) -> WorkspaceArtifact:
|
||||
ref = record.ref
|
||||
return WorkspaceArtifact(
|
||||
node_id=node.id,
|
||||
attachment_id=ref.attachment_id or "",
|
||||
file_name=ref.name or full_path.name,
|
||||
relative_path=str(relative_path),
|
||||
absolute_path=str(full_path),
|
||||
mime_type=ref.mime_type,
|
||||
size=ref.size,
|
||||
sha256=ref.sha256,
|
||||
data_uri=ref.data_uri,
|
||||
created_at=time.time(),
|
||||
change_type=change_type,
|
||||
extra=dict(record.extra),
|
||||
)
|
||||
|
||||
def _hash_file(self, path: Path) -> str:
|
||||
hasher = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
"""Runtime utilities for workflow execution."""
|
||||
|
||||
from .runtime_context import RuntimeContext
|
||||
from .runtime_builder import RuntimeBuilder
|
||||
from .execution_strategy import (
|
||||
DagExecutionStrategy,
|
||||
CycleExecutionStrategy,
|
||||
MajorityVoteStrategy,
|
||||
)
|
||||
from .result_archiver import ResultArchiver
|
||||
|
||||
__all__ = [
|
||||
"RuntimeContext",
|
||||
"RuntimeBuilder",
|
||||
"DagExecutionStrategy",
|
||||
"CycleExecutionStrategy",
|
||||
"MajorityVoteStrategy",
|
||||
"ResultArchiver",
|
||||
]
|
||||
Executable
+149
@@ -0,0 +1,149 @@
|
||||
"""Execution strategies for different graph topologies."""
|
||||
|
||||
from collections import Counter
|
||||
from typing import Callable, Dict, List, Sequence
|
||||
|
||||
from entity.configs import Node
|
||||
from entity.messages import Message
|
||||
from utils.log_manager import LogManager
|
||||
from workflow.executor.dag_executor import DAGExecutor
|
||||
from workflow.executor.cycle_executor import CycleExecutor
|
||||
from workflow.executor.parallel_executor import ParallelExecutor
|
||||
|
||||
|
||||
class DagExecutionStrategy:
|
||||
"""Executes acyclic graphs using the DAGExecutor."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
log_manager: LogManager,
|
||||
nodes: Dict[str, Node],
|
||||
layers: List[List[str]],
|
||||
execute_node_func: Callable[[Node], None],
|
||||
) -> None:
|
||||
self.log_manager = log_manager
|
||||
self.nodes = nodes
|
||||
self.layers = layers
|
||||
self.execute_node_func = execute_node_func
|
||||
|
||||
def run(self) -> None:
|
||||
dag_executor = DAGExecutor(
|
||||
log_manager=self.log_manager,
|
||||
nodes=self.nodes,
|
||||
layers=self.layers,
|
||||
execute_node_func=self.execute_node_func,
|
||||
)
|
||||
dag_executor.execute()
|
||||
|
||||
|
||||
class CycleExecutionStrategy:
|
||||
"""Executes graphs containing cycles via CycleExecutor."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
log_manager: LogManager,
|
||||
nodes: Dict[str, Node],
|
||||
cycle_execution_order: List[Dict[str, str]],
|
||||
cycle_manager,
|
||||
execute_node_func: Callable[[Node], None],
|
||||
) -> None:
|
||||
self.log_manager = log_manager
|
||||
self.nodes = nodes
|
||||
self.cycle_execution_order = cycle_execution_order
|
||||
self.cycle_manager = cycle_manager
|
||||
self.execute_node_func = execute_node_func
|
||||
|
||||
def run(self) -> None:
|
||||
cycle_executor = CycleExecutor(
|
||||
log_manager=self.log_manager,
|
||||
nodes=self.nodes,
|
||||
cycle_execution_order=self.cycle_execution_order,
|
||||
cycle_manager=self.cycle_manager,
|
||||
execute_node_func=self.execute_node_func,
|
||||
)
|
||||
cycle_executor.execute()
|
||||
|
||||
|
||||
class MajorityVoteStrategy:
|
||||
"""Executes graphs configured for majority voting (no edges)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
log_manager: LogManager,
|
||||
nodes: Dict[str, Node],
|
||||
initial_messages: Sequence[Message],
|
||||
execute_node_func: Callable[[Node], None],
|
||||
payload_to_text_func: Callable[[object], str],
|
||||
) -> None:
|
||||
self.log_manager = log_manager
|
||||
self.nodes = nodes
|
||||
self.initial_messages = initial_messages
|
||||
self.execute_node_func = execute_node_func
|
||||
self.payload_to_text = payload_to_text_func
|
||||
|
||||
def run(self) -> str:
|
||||
self.log_manager.info("Executing graph with majority voting approach")
|
||||
all_nodes = list(self.nodes.values())
|
||||
if not all_nodes:
|
||||
self.log_manager.error("No nodes to execute in majority voting mode")
|
||||
return ""
|
||||
|
||||
for node in all_nodes:
|
||||
node.clear_input()
|
||||
for message in self.initial_messages:
|
||||
node.append_input(message.clone())
|
||||
|
||||
node_ids = [node.id for node in all_nodes]
|
||||
|
||||
def _execute(node_id: str) -> None:
|
||||
self.execute_node_func(self.nodes[node_id])
|
||||
|
||||
parallel_executor = ParallelExecutor(self.log_manager, self.nodes)
|
||||
parallel_executor.execute_nodes_parallel(node_ids, _execute)
|
||||
|
||||
return self._collect_majority_result()
|
||||
|
||||
def _collect_majority_result(self) -> str:
|
||||
node_outputs: List[Dict[str, str]] = []
|
||||
for node_id, node in self.nodes.items():
|
||||
if node.output:
|
||||
output_text = self.payload_to_text(node.output[-1])
|
||||
else:
|
||||
output_text = ""
|
||||
node_outputs.append(
|
||||
{
|
||||
"node_id": node_id,
|
||||
"node_type": node.node_type,
|
||||
"output": output_text,
|
||||
}
|
||||
)
|
||||
|
||||
output_values = [item["output"] for item in node_outputs]
|
||||
output_counts = Counter(output_values)
|
||||
|
||||
non_empty_outputs = [value for value in output_values if value.strip()]
|
||||
if non_empty_outputs:
|
||||
output_counts = Counter(non_empty_outputs)
|
||||
|
||||
if not output_counts:
|
||||
self.log_manager.warning("No outputs available for majority voting")
|
||||
return ""
|
||||
|
||||
majority_output, count = output_counts.most_common(1)[0]
|
||||
self.log_manager.info(
|
||||
"Majority output determined",
|
||||
details={"result": majority_output, "votes": count},
|
||||
)
|
||||
self.log_manager.info(
|
||||
"All node outputs",
|
||||
details={
|
||||
"outputs": [
|
||||
(
|
||||
item["node_id"],
|
||||
item["output"][:50] + "..." if len(item["output"]) > 50 else item["output"],
|
||||
)
|
||||
for item in node_outputs
|
||||
]
|
||||
},
|
||||
)
|
||||
return majority_output
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
"""Utilities for persisting execution artifacts."""
|
||||
|
||||
from utils.log_manager import LogManager
|
||||
from utils.token_tracker import TokenTracker
|
||||
from workflow.graph_context import GraphContext
|
||||
|
||||
|
||||
class ResultArchiver:
|
||||
"""Handles post-execution persistence (tokens, logs, metadata)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: GraphContext,
|
||||
log_manager: LogManager,
|
||||
token_tracker: TokenTracker,
|
||||
) -> None:
|
||||
self.graph = graph
|
||||
self.log_manager = log_manager
|
||||
self.token_tracker = token_tracker
|
||||
|
||||
def export(self, final_result: str) -> None:
|
||||
token_usage_path = self.graph.directory / f"token_usage_{self.graph.name}.json"
|
||||
self.token_tracker.export_to_file(str(token_usage_path))
|
||||
self.log_manager.record_workflow_end(
|
||||
success=True,
|
||||
details={
|
||||
"token_usage": self.token_tracker.get_token_usage(),
|
||||
"final_result": final_result,
|
||||
},
|
||||
)
|
||||
log_file_path = self.graph.directory / "execution_logs.json"
|
||||
self.log_manager.save_logs(str(log_file_path))
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
"""Builder that assembles the runtime context for workflow execution."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from runtime.node.agent import ToolManager
|
||||
from utils.attachments import AttachmentStore
|
||||
from utils.function_manager import EDGE_FUNCTION_DIR, EDGE_PROCESSOR_FUNCTION_DIR, get_function_manager
|
||||
from utils.log_manager import LogManager
|
||||
from utils.logger import WorkflowLogger
|
||||
from utils.token_tracker import TokenTracker
|
||||
from workflow.graph_context import GraphContext
|
||||
|
||||
from .runtime_context import RuntimeContext
|
||||
|
||||
|
||||
@dataclass
|
||||
class RuntimeBuilder:
|
||||
"""Constructs RuntimeContext instances for GraphExecutor."""
|
||||
|
||||
graph: GraphContext
|
||||
|
||||
def build(self, logger: Optional[WorkflowLogger] = None, *, session_id: Optional[str] = None) -> RuntimeContext:
|
||||
tool_manager = ToolManager()
|
||||
function_manager = get_function_manager(EDGE_FUNCTION_DIR)
|
||||
processor_function_manager = get_function_manager(EDGE_PROCESSOR_FUNCTION_DIR)
|
||||
logger = logger or WorkflowLogger(self.graph.name, self.graph.log_level)
|
||||
log_manager = LogManager(logger)
|
||||
token_tracker = TokenTracker(workflow_id=self.graph.name)
|
||||
|
||||
code_workspace = (self.graph.directory / "code_workspace").resolve()
|
||||
code_workspace.mkdir(parents=True, exist_ok=True)
|
||||
attachments_dir = code_workspace / "attachments"
|
||||
attachments_dir.mkdir(parents=True, exist_ok=True)
|
||||
attachment_store = AttachmentStore(attachments_dir)
|
||||
|
||||
global_state: Dict[str, Any] = {
|
||||
"graph_directory": self.graph.directory,
|
||||
"vars": self.graph.config.vars,
|
||||
"python_workspace_root": code_workspace,
|
||||
"attachment_store": attachment_store,
|
||||
}
|
||||
|
||||
context = RuntimeContext(
|
||||
tool_manager=tool_manager,
|
||||
function_manager=function_manager,
|
||||
edge_processor_function_manager=processor_function_manager,
|
||||
logger=logger,
|
||||
log_manager=log_manager,
|
||||
token_tracker=token_tracker,
|
||||
attachment_store=attachment_store,
|
||||
code_workspace=code_workspace,
|
||||
global_state=global_state,
|
||||
)
|
||||
context.session_id = session_id
|
||||
if session_id:
|
||||
context.global_state.setdefault("session_id", session_id)
|
||||
return context
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
"""Shared runtime context for workflow execution."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from runtime.node.agent import ToolManager
|
||||
from utils.function_manager import FunctionManager
|
||||
from utils.logger import WorkflowLogger
|
||||
from utils.log_manager import LogManager
|
||||
from utils.token_tracker import TokenTracker
|
||||
from utils.attachments import AttachmentStore
|
||||
|
||||
|
||||
@dataclass
|
||||
class RuntimeContext:
|
||||
"""Container for runtime-wide dependencies required by GraphExecutor."""
|
||||
|
||||
tool_manager: ToolManager
|
||||
function_manager: FunctionManager
|
||||
edge_processor_function_manager: FunctionManager
|
||||
logger: WorkflowLogger
|
||||
log_manager: LogManager
|
||||
token_tracker: TokenTracker
|
||||
attachment_store: AttachmentStore
|
||||
code_workspace: Path
|
||||
global_state: Dict[str, Any] = field(default_factory=dict)
|
||||
cycle_manager: Optional[Any] = None # Late-bound by GraphManager
|
||||
session_id: Optional[str] = None
|
||||
workspace_hook: Optional[Any] = None
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
"""Utilities for loading reusable subgraph YAML definitions."""
|
||||
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from entity.configs import ConfigError
|
||||
from utils.io_utils import read_yaml
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
_DEFAULT_SUBGRAPH_ROOT = (_REPO_ROOT / "yaml_instance").resolve()
|
||||
_SUBGRAPH_CACHE: Dict[Path, Dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _resolve_candidate_paths(file_path: str, parent_source: str | None) -> List[Path]:
|
||||
path = Path(file_path)
|
||||
if path.is_absolute():
|
||||
return [path]
|
||||
|
||||
candidates: List[Path] = []
|
||||
default_candidate = (_DEFAULT_SUBGRAPH_ROOT / path).resolve()
|
||||
candidates.append(default_candidate)
|
||||
|
||||
if parent_source:
|
||||
parent = Path(parent_source)
|
||||
parent_dir = parent.parent if parent.is_file() else parent
|
||||
candidates.append((parent_dir / path).resolve())
|
||||
|
||||
# As a last resort, allow relative to repo root / current working dir
|
||||
candidates.append((_REPO_ROOT / path).resolve())
|
||||
return candidates
|
||||
|
||||
|
||||
def _resolve_existing_path(candidates: List[Path]) -> Path:
|
||||
checked: List[str] = []
|
||||
for candidate in candidates:
|
||||
checked.append(str(candidate))
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
raise ConfigError(
|
||||
f"subgraph YAML not found; tried: {', '.join(checked)}",
|
||||
path=checked[-1] if checked else None,
|
||||
)
|
||||
|
||||
|
||||
def _load_graph_dict(path: Path) -> Dict[str, Any]:
|
||||
data = read_yaml(path)
|
||||
if not isinstance(data, dict):
|
||||
raise ConfigError("subgraph YAML root must be a mapping", path=str(path))
|
||||
|
||||
graph_block = data.get("graph")
|
||||
if graph_block is None:
|
||||
graph_block = data
|
||||
|
||||
if not isinstance(graph_block, dict):
|
||||
raise ConfigError("subgraph graph section must be a mapping", path=f"{path}.graph")
|
||||
|
||||
vars_block = data.get("vars") if isinstance(data.get("vars"), dict) else {}
|
||||
|
||||
return {"graph": graph_block, "vars": vars_block}
|
||||
|
||||
|
||||
def load_subgraph_config(file_path: str, *, parent_source: str | None = None) -> Tuple[Dict[str, Any], Dict[str, Any], str]:
|
||||
"""Load a subgraph definition from disk.
|
||||
|
||||
Returns a tuple of (graph_dict, resolved_path).
|
||||
"""
|
||||
|
||||
candidates = _resolve_candidate_paths(file_path, parent_source)
|
||||
resolved_path = _resolve_existing_path(candidates).resolve()
|
||||
|
||||
if resolved_path not in _SUBGRAPH_CACHE:
|
||||
_SUBGRAPH_CACHE[resolved_path] = _load_graph_dict(resolved_path)
|
||||
|
||||
payload = _SUBGRAPH_CACHE[resolved_path]
|
||||
graph_dict = deepcopy(payload["graph"])
|
||||
vars_dict = dict(payload["vars"])
|
||||
return graph_dict, vars_dict, str(resolved_path)
|
||||
|
||||
|
||||
__all__ = ["load_subgraph_config"]
|
||||
Executable
+235
@@ -0,0 +1,235 @@
|
||||
"""Graph topology builder utility for cycle detection and topological sorting.
|
||||
|
||||
This module provides stateless utilities for building execution order of graphs,
|
||||
supporting both global graphs and scoped subgraphs (e.g., within cycles).
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Set, Any
|
||||
from entity.configs import Node
|
||||
from workflow.cycle_manager import CycleDetector
|
||||
|
||||
|
||||
class GraphTopologyBuilder:
|
||||
"""
|
||||
Graph topology structure builder.
|
||||
|
||||
Responsibilities:
|
||||
1. Detect cycles (based on CycleDetector)
|
||||
2. Build super-node graphs
|
||||
3. Perform topological sorting
|
||||
|
||||
Features:
|
||||
- Stateless (pure static methods)
|
||||
- Can be used for both global graphs and local subgraphs
|
||||
- Does not depend on specific GraphContext instances
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def detect_cycles(nodes: Dict[str, Node]) -> List[Set[str]]:
|
||||
"""
|
||||
Detect cycles in the given node set.
|
||||
|
||||
Args:
|
||||
nodes: Dictionary of nodes to analyze
|
||||
|
||||
Returns:
|
||||
List of cycles, where each cycle is a set of node IDs
|
||||
"""
|
||||
detector = CycleDetector()
|
||||
return detector.detect_cycles(nodes)
|
||||
|
||||
@staticmethod
|
||||
def create_super_node_graph(
|
||||
nodes: Dict[str, Node],
|
||||
edges: List[Dict[str, Any]],
|
||||
cycles: List[Set[str]]
|
||||
) -> Dict[str, Set[str]]:
|
||||
"""
|
||||
Create a super-node graph where each cycle is treated as a single node.
|
||||
|
||||
Args:
|
||||
nodes: Node dictionary
|
||||
edges: Edge configuration list (only edges to consider)
|
||||
cycles: List of detected cycles
|
||||
|
||||
Returns:
|
||||
Super-node dependency graph: {super_node_id: set(predecessor_super_node_ids)}
|
||||
"""
|
||||
super_nodes = {}
|
||||
node_to_super = {}
|
||||
|
||||
# Create super-nodes for cycles
|
||||
for i, cycle_nodes in enumerate(cycles):
|
||||
super_node_id = f"super_cycle_{i}"
|
||||
super_nodes[super_node_id] = set()
|
||||
for node_id in cycle_nodes:
|
||||
node_to_super[node_id] = super_node_id
|
||||
|
||||
# Create super-nodes for non-cycle nodes (each non-cycle node is its own super-node)
|
||||
for node_id in nodes.keys():
|
||||
if node_id not in node_to_super:
|
||||
super_node_id = f"node_{node_id}"
|
||||
super_nodes[super_node_id] = set()
|
||||
node_to_super[node_id] = super_node_id
|
||||
|
||||
# Build dependencies between super-nodes
|
||||
for edge_config in edges:
|
||||
from_node = edge_config["from"]
|
||||
to_node = edge_config["to"]
|
||||
|
||||
# Skip edges not in the node set
|
||||
if from_node not in nodes or to_node not in nodes:
|
||||
continue
|
||||
|
||||
from_super = node_to_super[from_node]
|
||||
to_super = node_to_super[to_node]
|
||||
|
||||
# Only add dependency if between different super-nodes
|
||||
if from_super != to_super:
|
||||
super_nodes[to_super].add(from_super)
|
||||
|
||||
return super_nodes
|
||||
|
||||
@staticmethod
|
||||
def topological_sort_super_nodes(
|
||||
super_node_graph: Dict[str, Set[str]],
|
||||
cycles: List[Set[str]]
|
||||
) -> List[List[Dict[str, Any]]]:
|
||||
"""
|
||||
Perform topological sort on super-node graph to determine execution order.
|
||||
|
||||
Args:
|
||||
super_node_graph: Super-node dependency graph
|
||||
cycles: List of cycles for mapping super-nodes to cycle info
|
||||
|
||||
Returns:
|
||||
Execution layers, where each layer contains items that can be executed in parallel.
|
||||
Format: [
|
||||
[{"type": "node", "node_id": "A"}, {"type": "cycle", "cycle_id": "...", "nodes": [...]}],
|
||||
[...]
|
||||
]
|
||||
"""
|
||||
# Calculate in-degrees
|
||||
in_degree = {
|
||||
super_node: len(predecessors)
|
||||
for super_node, predecessors in super_node_graph.items()
|
||||
}
|
||||
|
||||
# Find super-nodes with no dependencies
|
||||
ready = [node for node, degree in in_degree.items() if degree == 0]
|
||||
execution_layers = []
|
||||
|
||||
# Create cycle lookup
|
||||
cycle_lookup = {}
|
||||
for i, cycle_nodes in enumerate(cycles):
|
||||
cycle_id = f"cycle_{i}_{cycle_nodes}"
|
||||
cycle_lookup[f"super_cycle_{i}"] = {
|
||||
"cycle_id": cycle_id,
|
||||
"nodes": cycle_nodes
|
||||
}
|
||||
|
||||
while ready:
|
||||
current_layer = ready[:]
|
||||
ready.clear()
|
||||
|
||||
# Convert to execution items
|
||||
layer_items = []
|
||||
for super_node in current_layer:
|
||||
if super_node.startswith("super_cycle_"):
|
||||
# Cycle super-node
|
||||
cycle_data = cycle_lookup[super_node]
|
||||
layer_items.append({
|
||||
"type": "cycle",
|
||||
"cycle_id": cycle_data["cycle_id"],
|
||||
"nodes": list(cycle_data["nodes"])
|
||||
})
|
||||
elif super_node.startswith("node_"):
|
||||
# Regular node
|
||||
node_id = super_node.replace("node_", "")
|
||||
layer_items.append({
|
||||
"type": "node",
|
||||
"node_id": node_id
|
||||
})
|
||||
|
||||
# Update dependencies
|
||||
for dependent in super_node_graph:
|
||||
if super_node in super_node_graph[dependent]:
|
||||
super_node_graph[dependent].remove(super_node)
|
||||
in_degree[dependent] -= 1
|
||||
if in_degree[dependent] == 0:
|
||||
ready.append(dependent)
|
||||
|
||||
if layer_items:
|
||||
execution_layers.append(layer_items)
|
||||
|
||||
return execution_layers
|
||||
|
||||
@staticmethod
|
||||
def build_execution_order(
|
||||
nodes: Dict[str, Node],
|
||||
edges: List[Dict[str, Any]]
|
||||
) -> List[List[Dict[str, Any]]]:
|
||||
"""
|
||||
One-stop method to build execution order.
|
||||
|
||||
Combines cycle detection, super-node construction, and topological sorting.
|
||||
|
||||
Args:
|
||||
nodes: Node dictionary
|
||||
edges: Edge configuration list
|
||||
|
||||
Returns:
|
||||
Execution layers
|
||||
"""
|
||||
cycles = GraphTopologyBuilder.detect_cycles(nodes)
|
||||
|
||||
if not cycles:
|
||||
# No cycles, return DAG layers directly
|
||||
return GraphTopologyBuilder.build_dag_layers(nodes)
|
||||
|
||||
super_graph = GraphTopologyBuilder.create_super_node_graph(
|
||||
nodes, edges, cycles
|
||||
)
|
||||
|
||||
return GraphTopologyBuilder.topological_sort_super_nodes(
|
||||
super_graph, cycles
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def build_dag_layers(nodes: Dict[str, Node]) -> List[List[Dict[str, Any]]]:
|
||||
"""
|
||||
Build topological layers for DAG (Directed Acyclic Graph).
|
||||
|
||||
Args:
|
||||
nodes: Node dictionary
|
||||
|
||||
Returns:
|
||||
Layers in execution item format
|
||||
"""
|
||||
in_degree = {
|
||||
node_id: len(node.predecessors)
|
||||
for node_id, node in nodes.items()
|
||||
}
|
||||
|
||||
frontier = [
|
||||
node_id for node_id, deg in in_degree.items() if deg == 0
|
||||
]
|
||||
layers = []
|
||||
|
||||
while frontier:
|
||||
# Convert to execution item format
|
||||
layer_items = [
|
||||
{"type": "node", "node_id": node_id}
|
||||
for node_id in frontier
|
||||
]
|
||||
layers.append(layer_items)
|
||||
|
||||
next_frontier = []
|
||||
for node_id in frontier:
|
||||
for successor in nodes[node_id].successors:
|
||||
in_degree[successor.id] -= 1
|
||||
if in_degree[successor.id] == 0:
|
||||
next_frontier.append(successor.id)
|
||||
frontier = next_frontier
|
||||
|
||||
return layers
|
||||
Reference in New Issue
Block a user