Files
2026-07-13 12:49:10 +08:00

3795 lines
106 KiB
Go

/*
* Copyright 2026 CloudWeGo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package adk
import (
"context"
"errors"
"fmt"
"io"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/cloudwego/eino/components/model"
"github.com/cloudwego/eino/components/tool"
"github.com/cloudwego/eino/compose"
"github.com/cloudwego/eino/schema"
)
type cancelTestChatModel struct {
delayNs int64
response *schema.Message
startedChan chan struct{}
doneChan chan struct{}
}
func (m *cancelTestChatModel) getDelay() time.Duration {
return time.Duration(atomic.LoadInt64(&m.delayNs))
}
func (m *cancelTestChatModel) setDelay(d time.Duration) {
atomic.StoreInt64(&m.delayNs, int64(d))
}
func (m *cancelTestChatModel) Generate(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
select {
case m.startedChan <- struct{}{}:
default:
}
select {
case <-time.After(m.getDelay()):
case <-ctx.Done():
return nil, ctx.Err()
}
select {
case m.doneChan <- struct{}{}:
default:
}
return m.response, nil
}
func (m *cancelTestChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
m.startedChan <- struct{}{}
time.Sleep(m.getDelay())
m.doneChan <- struct{}{}
return schema.StreamReaderFromArray([]*schema.Message{m.response}), nil
}
func (m *cancelTestChatModel) BindTools(tools []*schema.ToolInfo) error {
return nil
}
type slowTool struct {
name string
delay time.Duration
result string
callCount int32
startedChan chan struct{}
}
func newSlowTool(name string, delay time.Duration, result string) *slowTool {
return &slowTool{
name: name,
delay: delay,
result: result,
startedChan: make(chan struct{}, 10),
}
}
func (t *slowTool) Info(ctx context.Context) (*schema.ToolInfo, error) {
return &schema.ToolInfo{
Name: t.name,
Desc: "A slow tool for testing",
ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
"input": {Type: "string", Desc: "Input parameter"},
}),
}, nil
}
func (t *slowTool) InvokableRun(ctx context.Context, argumentsInJSON string, opts ...tool.Option) (string, error) {
atomic.AddInt32(&t.callCount, 1)
select {
case t.startedChan <- struct{}{}:
default:
}
select {
case <-time.After(t.delay):
case <-ctx.Done():
return "", ctx.Err()
}
return t.result, nil
}
type cancelTestStore struct {
m map[string][]byte
mu sync.Mutex
}
func newCancelTestStore() *cancelTestStore {
return &cancelTestStore{m: make(map[string][]byte)}
}
func (s *cancelTestStore) Set(_ context.Context, key string, value []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
s.m[key] = value
return nil
}
func (s *cancelTestStore) Get(_ context.Context, key string) ([]byte, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.m[key]
return v, ok, nil
}
func assertHasCancelError(t *testing.T, events []*AgentEvent) {
t.Helper()
for _, e := range events {
var ce *CancelError
if e.Err != nil && errors.As(e.Err, &ce) {
return
}
}
t.Fatal("expected CancelError in events")
}
func drainAndAssertCancelError(t *testing.T, iter *AsyncIterator[*AgentEvent]) {
t.Helper()
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
return
}
}
t.Fatal("expected CancelError in event stream")
}
func drainEventsAndAssertCancelError(t *testing.T, iter *AsyncIterator[*AgentEvent]) []*AgentEvent {
t.Helper()
var events []*AgentEvent
hasCancelError := false
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
hasCancelError = true
}
events = append(events, event)
}
assert.True(t, hasCancelError, "expected CancelError in event stream")
return events
}
func TestCancelContext(t *testing.T) {
t.Run("BasicCancelContext", func(t *testing.T) {
cc := newCancelContext()
assert.False(t, cc.shouldCancel(), "Should not be cancelled initially")
cc.setMode(CancelImmediate)
close(cc.cancelChan)
assert.True(t, cc.shouldCancel(), "Should be cancelled after close(cancelChan)")
assert.Equal(t, CancelImmediate, cc.getMode())
})
}
func TestWithCancel_WithTools(t *testing.T) {
ctx := context.Background()
t.Run("CancelImmediate_DuringModelCall", func(t *testing.T) {
modelStarted := make(chan struct{}, 1)
st := newSlowTool("slow_tool", 100*time.Millisecond, "tool result")
slowModel := &cancelTestChatModel{
delayNs: int64(2 * time.Second),
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
startedChan: modelStarted,
doneChan: make(chan struct{}, 1),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: slowModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: agent,
EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("Use the tool")}, cancelOpt)
assert.NotNil(t, iter)
assert.NotNil(t, cancelFn)
eventsCh := make(chan []*AgentEvent, 1)
go func() {
var events []*AgentEvent
for {
event, ok := iter.Next()
if !ok {
break
}
events = append(events, event)
}
eventsCh <- events
}()
select {
case <-modelStarted:
case <-time.After(5 * time.Second):
t.Fatal("Model did not start within 5 seconds")
}
time.Sleep(100 * time.Millisecond)
handle, _ := cancelFn()
err = handle.Wait()
assert.NoError(t, err)
var events []*AgentEvent
select {
case events = <-eventsCh:
case <-time.After(5 * time.Second):
t.Fatal("Timed out waiting for events")
}
assert.NotEmpty(t, events)
assertHasCancelError(t, events)
})
t.Run("CancelAfterChatModel_DuringToolCall", func(t *testing.T) {
toolStarted := make(chan struct{}, 1)
st := &slowToolWithSignal{
name: "slow_tool",
delay: 2 * time.Second,
result: "tool result",
startedChan: toolStarted,
}
modelWithToolCall := &simpleChatModel{
delay: 1 * time.Second,
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: modelWithToolCall,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
cancelOpt, cancelFn := WithCancel()
iter := agent.Run(ctx, &AgentInput{
Messages: []Message{schema.UserMessage("Use the tool")},
}, cancelOpt)
assert.NotNil(t, iter)
assert.NotNil(t, cancelFn)
<-toolStarted
time.Sleep(100 * time.Millisecond)
handle, _ := cancelFn(WithAgentCancelMode(CancelAfterChatModel))
err = handle.Wait()
assert.NoError(t, err)
var events []*AgentEvent
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
continue
}
events = append(events, event)
}
assert.NotEmpty(t, events)
assert.True(t, atomic.LoadInt32(&st.callCount) >= 1, "Tool should have been called")
})
t.Run("CancelAfterToolCalls_CompletesToolExecution", func(t *testing.T) {
toolStarted := make(chan struct{}, 1)
st := &slowToolWithSignal{
name: "slow_tool",
delay: 500 * time.Millisecond,
result: "tool result",
startedChan: toolStarted,
}
modelWithToolCall := &simpleChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: modelWithToolCall,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
cancelOpt, cancelFn := WithCancel()
iter := agent.Run(ctx, &AgentInput{
Messages: []Message{schema.UserMessage("Use the tool")},
}, cancelOpt)
assert.NotNil(t, iter)
assert.NotNil(t, cancelFn)
<-toolStarted
time.Sleep(100 * time.Millisecond)
handle, _ := cancelFn(WithAgentCancelMode(CancelAfterToolCalls))
err = handle.Wait()
assert.NoError(t, err)
var events []*AgentEvent
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
continue
}
events = append(events, event)
}
assert.NotEmpty(t, events)
assert.True(t, atomic.LoadInt32(&st.callCount) >= 1, "Tool should have been called")
})
t.Run("NestedCancelPropagation", func(t *testing.T) {
cc := newCancelContext()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
child := cc.deriveAgentToolCancelContext(ctx)
assert.NotNil(t, child)
cc.setRecursive(true)
cc.setMode(CancelImmediate)
if atomic.CompareAndSwapInt32(&cc.state, stateRunning, stateCancelling) {
close(cc.cancelChan)
}
select {
case <-child.cancelChan:
case <-time.After(1 * time.Second):
t.Fatal("Child did not receive cancel signal")
}
assert.True(t, child.shouldCancel())
assert.Equal(t, CancelImmediate, child.getMode())
})
t.Run("DeepAgentIntegrationCancel", func(t *testing.T) {
ctx := context.Background()
modelStarted := make(chan struct{}, 1)
leafModel := &cancelTestChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "Leaf result",
},
startedChan: modelStarted,
doneChan: make(chan struct{}, 1),
}
leafModel.setDelay(500 * time.Millisecond)
leafAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "LeafAgent",
Description: "desc",
Model: leafModel,
})
assert.NoError(t, err)
rootModel := &cancelTestChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "LeafAgent",
Arguments: `{}`,
},
},
},
},
startedChan: make(chan struct{}, 1),
doneChan: make(chan struct{}, 1),
}
rootAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RootAgent",
Description: "desc",
Model: rootModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{NewAgentTool(ctx, leafAgent)},
},
},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: rootAgent,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("Run leaf")}, cancelOpt)
<-modelStarted
handle, _ := cancelFn(WithAgentCancelMode(CancelAfterChatModel), WithRecursive())
err = handle.Wait()
assert.NoError(t, err)
hasCancelError := false
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil {
var ce *CancelError
if errors.As(event.Err, &ce) {
hasCancelError = true
assert.NotNil(t, ce.interruptSignal, "CancelError should carry interrupt signal")
}
}
}
assert.True(t, hasCancelError, "Should have received CancelError")
})
}
type slowToolWithSignal struct {
name string
delay time.Duration
result string
callCount int32
startedChan chan struct{}
}
func (t *slowToolWithSignal) Info(ctx context.Context) (*schema.ToolInfo, error) {
return &schema.ToolInfo{
Name: t.name,
Desc: "A slow tool for testing",
ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
"input": {Type: "string", Desc: "Input parameter"},
}),
}, nil
}
func (t *slowToolWithSignal) InvokableRun(ctx context.Context, argumentsInJSON string, opts ...tool.Option) (string, error) {
atomic.AddInt32(&t.callCount, 1)
t.startedChan <- struct{}{}
time.Sleep(t.delay)
return t.result, nil
}
type simpleChatModel struct {
delay time.Duration
response *schema.Message
}
func (m *simpleChatModel) Generate(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
if m.delay > 0 {
select {
case <-time.After(m.delay):
case <-ctx.Done():
return nil, ctx.Err()
}
}
return m.response, nil
}
func (m *simpleChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
if m.delay > 0 {
select {
case <-time.After(m.delay):
case <-ctx.Done():
return nil, ctx.Err()
}
}
return schema.StreamReaderFromArray([]*schema.Message{m.response}), nil
}
func (m *simpleChatModel) BindTools(tools []*schema.ToolInfo) error {
return nil
}
func TestWithCancel_WithCheckpoint(t *testing.T) {
ctx := context.Background()
t.Run("CancelWithCheckpoint", func(t *testing.T) {
modelStarted := make(chan struct{}, 1)
st := newSlowTool("slow_tool", 100*time.Millisecond, "tool result")
slowModel := &cancelTestChatModel{
delayNs: int64(1 * time.Second),
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
startedChan: modelStarted,
doneChan: make(chan struct{}, 1),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: slowModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{
Agent: agent,
EnableStreaming: false,
CheckPointStore: store,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("Use the tool")}, cancelOpt, WithCheckPointID("cancel-1"))
<-modelStarted
handle, _ := cancelFn()
err = handle.Wait()
assert.NoError(t, err)
hasCancelError := false
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
hasCancelError = true
continue
}
}
assert.True(t, hasCancelError, "Should have CancelError event after cancel")
})
}
func TestAgentCancelFuncMultipleCalls(t *testing.T) {
ctx := context.Background()
t.Run("SecondCancelReturnsErrAgentFinished", func(t *testing.T) {
modelStarted := make(chan struct{}, 1)
st := newSlowTool("slow_tool", 100*time.Millisecond, "tool result")
slowModel := &cancelTestChatModel{
delayNs: int64(1 * time.Second),
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
startedChan: modelStarted,
doneChan: make(chan struct{}, 1),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: slowModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: agent,
EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("Use the tool")}, cancelOpt)
<-modelStarted
handle, _ := cancelFn()
cancelErr := handle.Wait()
assert.NoError(t, cancelErr)
for {
_, ok := iter.Next()
if !ok {
break
}
}
})
}
func TestWithCancel_Streaming(t *testing.T) {
ctx := context.Background()
t.Run("CancelImmediate_DuringModelStream", func(t *testing.T) {
modelStarted := make(chan struct{}, 1)
st := newSlowTool("slow_tool", 100*time.Millisecond, "tool result")
slowModel := &cancelTestChatModel{
delayNs: int64(2 * time.Second),
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
startedChan: modelStarted,
doneChan: make(chan struct{}, 1),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: slowModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: agent,
EnableStreaming: true,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("Use the tool")}, cancelOpt)
assert.NotNil(t, iter)
assert.NotNil(t, cancelFn)
eventsCh := make(chan []*AgentEvent, 1)
go func() {
var events []*AgentEvent
for {
event, ok := iter.Next()
if !ok {
break
}
events = append(events, event)
}
eventsCh <- events
}()
select {
case <-modelStarted:
case <-time.After(5 * time.Second):
t.Fatal("Model did not start within 5 seconds")
}
time.Sleep(100 * time.Millisecond)
handle, _ := cancelFn()
cancelErr := handle.Wait()
assert.NoError(t, cancelErr)
var events []*AgentEvent
select {
case events = <-eventsCh:
case <-time.After(5 * time.Second):
t.Fatal("Timed out waiting for events")
}
assert.NotEmpty(t, events)
assertHasCancelError(t, events)
})
t.Run("CancelAfterToolCalls_Streaming", func(t *testing.T) {
toolStarted := make(chan struct{}, 1)
st := &slowToolWithSignal{
name: "slow_tool",
delay: 500 * time.Millisecond,
result: "tool result",
startedChan: toolStarted,
}
modelWithToolCall := &simpleChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: modelWithToolCall,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: agent,
EnableStreaming: true,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("Use the tool")}, cancelOpt)
assert.NotNil(t, iter)
assert.NotNil(t, cancelFn)
<-toolStarted
time.Sleep(100 * time.Millisecond)
handle, _ := cancelFn(WithAgentCancelMode(CancelAfterToolCalls))
cancelErr := handle.Wait()
assert.NoError(t, cancelErr)
var events []*AgentEvent
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
continue
}
events = append(events, event)
}
assert.NotEmpty(t, events)
assert.True(t, atomic.LoadInt32(&st.callCount) >= 1, "Tool should have been called")
})
}
// TestWithCancel_Resume tests the workflow of Cancel followed by Resume.
//
// To avoid data races, we create new agent and runner instances for the Resume phase
// instead of reusing and modifying the original model instance.
func TestWithCancel_Resume(t *testing.T) {
ctx := context.Background()
t.Run("Cancel_ThenResume", func(t *testing.T) {
modelStarted := make(chan struct{}, 1)
modelCallCount := int32(0)
st := newSlowTool("slow_tool", 100*time.Millisecond, "tool result")
slowModel := &cancelTestChatModel{
delayNs: int64(500 * time.Millisecond),
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
startedChan: modelStarted,
doneChan: make(chan struct{}, 1),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: slowModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
store := newCancelTestStore()
checkpointID := "resume-cancel-test-1"
runner := NewRunner(ctx, RunnerConfig{
Agent: agent,
EnableStreaming: false,
CheckPointStore: store,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("Use the tool")}, cancelOpt, WithCheckPointID(checkpointID))
<-modelStarted
atomic.AddInt32(&modelCallCount, 1)
handle, _ := cancelFn()
cancelErr := handle.Wait()
assert.NoError(t, cancelErr)
hasCancelErr := false
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil {
var ce *CancelError
if errors.As(event.Err, &ce) {
hasCancelErr = true
continue
}
t.Fatalf("unexpected error: %v", event.Err)
}
}
assert.True(t, hasCancelErr, "Should have CancelError event after cancel")
newModelStarted := make(chan struct{}, 1)
slowModel2 := &cancelTestChatModel{
delayNs: int64(100 * time.Millisecond),
response: &schema.Message{
Role: schema.Assistant,
Content: "Final response after resume",
},
startedChan: newModelStarted,
doneChan: make(chan struct{}, 1),
}
agent2, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: slowModel2,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{
Agent: agent2,
EnableStreaming: false,
CheckPointStore: store,
})
resumeCancelOpt, _ := WithCancel()
resumeIter, err := runner2.Resume(ctx, checkpointID, resumeCancelOpt)
assert.NoError(t, err)
assert.NotNil(t, resumeIter)
var resumeEvents []*AgentEvent
for {
event, ok := resumeIter.Next()
if !ok {
break
}
assert.Nil(t, event.Err, "Should not have error event during resume")
resumeEvents = append(resumeEvents, event)
}
assert.NotEmpty(t, resumeEvents, "Resume should produce events")
})
t.Run("Resume_ThenCancel", func(t *testing.T) {
firstModelStarted := make(chan struct{}, 1)
modelCallCount := int32(0)
st := newSlowTool("slow_tool", 100*time.Millisecond, "tool result")
slowModel := &cancelTestChatModel{
delayNs: int64(500 * time.Millisecond),
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
startedChan: firstModelStarted,
doneChan: make(chan struct{}, 1),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: slowModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
store := newCancelTestStore()
checkpointID := "resume-then-cancel-test-1"
runner := NewRunner(ctx, RunnerConfig{
Agent: agent,
EnableStreaming: false,
CheckPointStore: store,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("Use the tool")}, cancelOpt, WithCheckPointID(checkpointID))
<-firstModelStarted
atomic.AddInt32(&modelCallCount, 1)
handle, _ := cancelFn()
cancelErr := handle.Wait()
assert.NoError(t, cancelErr)
for {
_, ok := iter.Next()
if !ok {
break
}
}
slowModel2 := newBlockingChatModel(toolCallMsg(toolCall("call_1", "slow_tool", `{"input": "test"}`)))
agent2, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent with tool",
Instruction: "You are a test assistant",
Model: slowModel2,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{
Agent: agent2,
EnableStreaming: false,
CheckPointStore: store,
})
resumeCancelOpt, resumeCancelFn := WithCancel()
resumeIter, err := runner2.Resume(ctx, checkpointID, resumeCancelOpt)
assert.NoError(t, err)
resumeEventsCh := make(chan []*AgentEvent, 1)
go func() {
var events []*AgentEvent
for {
event, ok := resumeIter.Next()
if !ok {
break
}
events = append(events, event)
}
resumeEventsCh <- events
}()
<-slowModel2.started
atomic.AddInt32(&modelCallCount, 1)
cancelHandle, _ := resumeCancelFn()
close(slowModel2.unblockCh)
err = cancelHandle.Wait()
assert.True(t, err == nil || errors.Is(err, ErrExecutionEnded), "unexpected cancel wait error: %v", err)
start := time.Now()
resumeEvents := <-resumeEventsCh
elapsed := time.Since(start)
assert.True(t, elapsed < 1*time.Second, "Resume should return quickly after cancel, elapsed: %v", elapsed)
assert.NotEmpty(t, resumeEvents)
hasCancelError := false
for _, e := range resumeEvents {
var ce *CancelError
if e.Err != nil && errors.As(e.Err, &ce) {
hasCancelError = true
}
}
executionCompletedBeforeCancel := errors.Is(err, ErrExecutionEnded)
assert.True(t, hasCancelError || executionCompletedBeforeCancel, "Resume should have CancelError event after cancel, or execution completed before cancel")
})
}
func TestCancelMonitoredToolHandler_StreamableToolCall(t *testing.T) {
t.Run("NoCancelContext_PassesThrough", func(t *testing.T) {
handler := &cancelMonitoredToolHandler{}
// Create a stream with some data
r, w := schema.Pipe[string](1)
go func() {
w.Send("chunk1", nil)
w.Send("chunk2", nil)
w.Close()
}()
next := func(ctx context.Context, input *compose.ToolInput) (*compose.StreamToolOutput, error) {
return &compose.StreamToolOutput{Result: r}, nil
}
wrapped := handler.WrapStreamableToolCall(next)
// No cancelContext in the Go context
output, err := wrapped(context.Background(), &compose.ToolInput{Name: "test"})
assert.NoError(t, err)
// Should get the original stream unchanged
chunk1, err := output.Result.Recv()
assert.NoError(t, err)
assert.Equal(t, "chunk1", chunk1)
chunk2, err := output.Result.Recv()
assert.NoError(t, err)
assert.Equal(t, "chunk2", chunk2)
_, err = output.Result.Recv()
assert.ErrorIs(t, err, io.EOF)
})
t.Run("WithCancelContext_NoCancel_StreamsNormally", func(t *testing.T) {
handler := &cancelMonitoredToolHandler{}
cc := newCancelContext()
r, w := schema.Pipe[string](1)
go func() {
w.Send("data1", nil)
w.Send("data2", nil)
w.Close()
}()
next := func(ctx context.Context, input *compose.ToolInput) (*compose.StreamToolOutput, error) {
return &compose.StreamToolOutput{Result: r}, nil
}
wrapped := handler.WrapStreamableToolCall(next)
ctx := withCancelContext(context.Background(), cc)
output, err := wrapped(ctx, &compose.ToolInput{Name: "test"})
assert.NoError(t, err)
chunk1, err := output.Result.Recv()
assert.NoError(t, err)
assert.Equal(t, "data1", chunk1)
chunk2, err := output.Result.Recv()
assert.NoError(t, err)
assert.Equal(t, "data2", chunk2)
_, err = output.Result.Recv()
assert.ErrorIs(t, err, io.EOF)
})
t.Run("WithCancelContext_ImmediateCancel_TerminatesStream", func(t *testing.T) {
handler := &cancelMonitoredToolHandler{}
cc := newCancelContext()
// Create a slow stream that we'll cancel mid-way
r, w := schema.Pipe[string](1)
go func() {
defer w.Close()
w.Send("chunk1", nil)
time.Sleep(200 * time.Millisecond)
w.Send("chunk2", nil)
}()
next := func(ctx context.Context, input *compose.ToolInput) (*compose.StreamToolOutput, error) {
return &compose.StreamToolOutput{Result: r}, nil
}
wrapped := handler.WrapStreamableToolCall(next)
ctx := withCancelContext(context.Background(), cc)
output, err := wrapped(ctx, &compose.ToolInput{Name: "test"})
assert.NoError(t, err)
// Read first chunk
chunk1, err := output.Result.Recv()
assert.NoError(t, err)
assert.Equal(t, "chunk1", chunk1)
// Fire immediate cancel
close(cc.immediateChan)
// Next recv should get ErrStreamCanceled
_, err = output.Result.Recv()
assert.ErrorIs(t, err, ErrStreamCanceled)
})
t.Run("WithCancelContext_AlreadyCancelled_TerminatesImmediately", func(t *testing.T) {
handler := &cancelMonitoredToolHandler{}
cc := newCancelContext()
close(cc.immediateChan) // Already canceled
r, w := schema.Pipe[string](1)
go func() {
w.Send("should-not-see", nil)
w.Close()
}()
next := func(ctx context.Context, input *compose.ToolInput) (*compose.StreamToolOutput, error) {
return &compose.StreamToolOutput{Result: r}, nil
}
wrapped := handler.WrapStreamableToolCall(next)
ctx := withCancelContext(context.Background(), cc)
output, err := wrapped(ctx, &compose.ToolInput{Name: "test"})
assert.NoError(t, err)
_, err = output.Result.Recv()
assert.ErrorIs(t, err, ErrStreamCanceled)
})
t.Run("NextReturnsError_PropagatesError", func(t *testing.T) {
handler := &cancelMonitoredToolHandler{}
cc := newCancelContext()
nextErr := errors.New("tool execution failed")
next := func(ctx context.Context, input *compose.ToolInput) (*compose.StreamToolOutput, error) {
return nil, nextErr
}
wrapped := handler.WrapStreamableToolCall(next)
ctx := withCancelContext(context.Background(), cc)
_, err := wrapped(ctx, &compose.ToolInput{Name: "test"})
assert.ErrorIs(t, err, nextErr)
})
}
func TestCancelMonitoredToolHandler_EnhancedStreamableToolCall(t *testing.T) {
t.Run("NoCancelContext_PassesThrough", func(t *testing.T) {
handler := &cancelMonitoredToolHandler{}
tr1 := &schema.ToolResult{Parts: []schema.ToolOutputPart{{Type: schema.ToolPartTypeText, Text: "chunk1"}}}
r, w := schema.Pipe[*schema.ToolResult](1)
go func() {
w.Send(tr1, nil)
w.Close()
}()
next := func(ctx context.Context, input *compose.ToolInput) (*compose.EnhancedStreamableToolOutput, error) {
return &compose.EnhancedStreamableToolOutput{Result: r}, nil
}
wrapped := handler.WrapEnhancedStreamableToolCall(next)
output, err := wrapped(context.Background(), &compose.ToolInput{Name: "test"})
assert.NoError(t, err)
result, err := output.Result.Recv()
assert.NoError(t, err)
assert.Equal(t, tr1, result)
_, err = output.Result.Recv()
assert.ErrorIs(t, err, io.EOF)
})
t.Run("WithCancelContext_ImmediateCancel_TerminatesStream", func(t *testing.T) {
handler := &cancelMonitoredToolHandler{}
cc := newCancelContext()
tr1 := &schema.ToolResult{Parts: []schema.ToolOutputPart{{Type: schema.ToolPartTypeText, Text: "chunk1"}}}
tr2 := &schema.ToolResult{Parts: []schema.ToolOutputPart{{Type: schema.ToolPartTypeText, Text: "chunk2"}}}
r, w := schema.Pipe[*schema.ToolResult](1)
go func() {
defer w.Close()
w.Send(tr1, nil)
time.Sleep(200 * time.Millisecond)
w.Send(tr2, nil)
}()
next := func(ctx context.Context, input *compose.ToolInput) (*compose.EnhancedStreamableToolOutput, error) {
return &compose.EnhancedStreamableToolOutput{Result: r}, nil
}
wrapped := handler.WrapEnhancedStreamableToolCall(next)
ctx := withCancelContext(context.Background(), cc)
output, err := wrapped(ctx, &compose.ToolInput{Name: "test"})
assert.NoError(t, err)
result, err := output.Result.Recv()
assert.NoError(t, err)
assert.Equal(t, tr1, result)
close(cc.immediateChan)
_, err = output.Result.Recv()
assert.ErrorIs(t, err, ErrStreamCanceled)
})
t.Run("NextReturnsError_PropagatesError", func(t *testing.T) {
handler := &cancelMonitoredToolHandler{}
cc := newCancelContext()
nextErr := errors.New("enhanced tool failed")
next := func(ctx context.Context, input *compose.ToolInput) (*compose.EnhancedStreamableToolOutput, error) {
return nil, nextErr
}
wrapped := handler.WrapEnhancedStreamableToolCall(next)
ctx := withCancelContext(context.Background(), cc)
_, err := wrapped(ctx, &compose.ToolInput{Name: "test"})
assert.ErrorIs(t, err, nextErr)
})
}
func TestCancelContextKey(t *testing.T) {
t.Run("WithAndGet_RoundTrips", func(t *testing.T) {
cc := newCancelContext()
ctx := withCancelContext(context.Background(), cc)
got := getCancelContext(ctx)
assert.Equal(t, cc, got)
})
t.Run("Get_NoValue_ReturnsNil", func(t *testing.T) {
got := getCancelContext(context.Background())
assert.Nil(t, got)
})
t.Run("With_NilCancelContext_ReturnsOriginalCtx", func(t *testing.T) {
ctx := context.Background()
result := withCancelContext(ctx, nil)
assert.Equal(t, ctx, result)
})
}
// -- Tests for cancel support across all agent types --
// cancelTestAgent is a ChatModelAgent-based agent where the model blocks until
// signalled, allowing tests to control exactly when to issue a cancel.
func newCancelTestAgent(t *testing.T, name string, modelDelay time.Duration, modelStarted chan struct{}) *ChatModelAgent {
t.Helper()
slowModel := &cancelTestChatModel{
delayNs: int64(modelDelay),
response: &schema.Message{
Role: schema.Assistant,
Content: "response from " + name,
},
startedChan: modelStarted,
doneChan: make(chan struct{}, 1),
}
agent, err := NewChatModelAgent(context.Background(), &ChatModelAgentConfig{
Name: name,
Description: "Test agent " + name,
Instruction: "You are a test assistant",
Model: slowModel,
})
assert.NoError(t, err)
return agent
}
func newCancelTestAgentWithTools(t *testing.T, name string, modelDelay time.Duration, modelStarted chan struct{}) *ChatModelAgent {
t.Helper()
toolName := name + "_tool"
slowModel := &cancelTestChatModel{
delayNs: int64(modelDelay),
response: &schema.Message{
Role: schema.Assistant,
ToolCalls: []schema.ToolCall{{
ID: "call_1", Type: "function",
Function: schema.FunctionCall{
Name: toolName,
Arguments: `{"input": "test"}`,
},
}},
},
startedChan: modelStarted,
doneChan: make(chan struct{}, 1),
}
st := newSlowTool(toolName, 10*time.Millisecond, "tool result")
agent, err := NewChatModelAgent(context.Background(), &ChatModelAgentConfig{
Name: name,
Description: "Test agent " + name,
Instruction: "You are a test assistant",
Model: slowModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
return agent
}
func newCancelTestAgentWithToolsFinalAnswer(t *testing.T, name string) *ChatModelAgent {
t.Helper()
toolName := name + "_tool"
finalModel := &cancelTestChatModel{
delayNs: int64(10 * time.Millisecond),
response: &schema.Message{
Role: schema.Assistant,
Content: "final response from " + name,
},
startedChan: make(chan struct{}, 1),
doneChan: make(chan struct{}, 1),
}
st := newSlowTool(toolName, 10*time.Millisecond, "tool result")
agent, err := NewChatModelAgent(context.Background(), &ChatModelAgentConfig{
Name: name,
Description: "Test agent " + name,
Instruction: "You are a test assistant",
Model: finalModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
return agent
}
func TestWithCancel_SequentialAgent(t *testing.T) {
ctx := context.Background()
t.Run("CancelImmediate_DuringSecondAgent", func(t *testing.T) {
// The first agent completes quickly. The second agent takes a long time.
// Cancel during the second agent's model call.
agent1Started := make(chan struct{}, 1)
agent2Started := make(chan struct{}, 1)
agent1 := newCancelTestAgent(t, "fast_agent", 50*time.Millisecond, agent1Started)
agent2 := newCancelTestAgent(t, "slow_agent", 5*time.Second, agent2Started)
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq_agent",
Description: "Sequential test",
SubAgents: []Agent{agent1, agent2},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: seqAgent,
EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt)
// Wait for second agent to start
select {
case <-agent2Started:
case <-time.After(10 * time.Second):
t.Fatal("Second agent did not start")
}
time.Sleep(50 * time.Millisecond)
// Cancel should NOT return ErrExecutionEnded (the bug before the fix)
handle, _ := cancelFn()
err = handle.Wait()
assert.NoError(t, err, "Cancel during second agent should succeed, not return ErrExecutionEnded")
drainEventsAndAssertCancelError(t, iter)
})
}
func TestWithCancel_LoopAgent(t *testing.T) {
ctx := context.Background()
t.Run("CancelImmediate_DuringIteration", func(t *testing.T) {
// Agent in a loop. Cancel during second iteration's model call.
modelStarted := make(chan struct{}, 10)
slowModel := &cancelTestChatModel{
delayNs: int64(3 * time.Second),
response: &schema.Message{
Role: schema.Assistant,
Content: "loop response",
},
startedChan: modelStarted,
doneChan: make(chan struct{}, 10),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "loop_inner",
Description: "Inner loop agent",
Instruction: "You are a test assistant",
Model: slowModel,
})
assert.NoError(t, err)
loopAgent, err := NewLoopAgent(ctx, &LoopAgentConfig{
Name: "loop_agent",
Description: "Loop test",
SubAgents: []Agent{agent},
MaxIterations: 10,
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: loopAgent,
EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt)
// Wait for first iteration's model call to start
select {
case <-modelStarted:
case <-time.After(10 * time.Second):
t.Fatal("Model did not start")
}
time.Sleep(50 * time.Millisecond)
// Cancel should succeed
handle, _ := cancelFn()
err = handle.Wait()
assert.NoError(t, err, "Cancel during loop iteration should succeed")
drainAndAssertCancelError(t, iter)
})
}
func TestWithCancel_ParallelAgent(t *testing.T) {
ctx := context.Background()
t.Run("CancelImmediate_InterruptsAllBranches", func(t *testing.T) {
agent1Started := make(chan struct{}, 1)
agent2Started := make(chan struct{}, 1)
// Both agents have long delays, so cancel should interrupt both.
agent1 := newCancelTestAgent(t, "par_agent1", 5*time.Second, agent1Started)
agent2 := newCancelTestAgent(t, "par_agent2", 5*time.Second, agent2Started)
parAgent, err := NewParallelAgent(ctx, &ParallelAgentConfig{
Name: "par_agent",
Description: "Parallel test",
SubAgents: []Agent{agent1, agent2},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: parAgent,
EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt)
// Wait for both agents to start
for i := 0; i < 2; i++ {
select {
case <-agent1Started:
case <-agent2Started:
case <-time.After(10 * time.Second):
t.Fatal("Parallel agents did not start")
}
}
time.Sleep(50 * time.Millisecond)
start := time.Now()
handle, _ := cancelFn()
err = handle.Wait()
assert.NoError(t, err, "Cancel during parallel agents should succeed")
events := drainEventsAndAssertCancelError(t, iter)
elapsed := time.Since(start)
_ = events
assert.True(t, elapsed < 3*time.Second, "Should complete quickly after cancel, elapsed: %v", elapsed)
})
}
func TestWithCancel_SupervisorAgent(t *testing.T) {
ctx := context.Background()
t.Run("CancelImmediate_DuringSubAgent", func(t *testing.T) {
// Supervisor delegates to a slow sub-agent via transfer.
// Cancel during the sub-agent's model call.
supervisorModelStarted := make(chan struct{}, 1)
subAgentModelStarted := make(chan struct{}, 1)
// The supervisor model returns a transfer_to_agent tool call
supervisorModel := &simpleChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: TransferToAgentToolName,
Arguments: `{"agent_name": "slow_sub"}`,
},
},
},
},
}
supervisorAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "supervisor",
Description: "Supervisor agent",
Instruction: "You are a supervisor",
Model: supervisorModel,
})
assert.NoError(t, err)
subAgent := newCancelTestAgent(t, "slow_sub", 5*time.Second, subAgentModelStarted)
agentWithSubAgents, err := SetSubAgents(ctx, supervisorAgent, []Agent{subAgent})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: agentWithSubAgents,
EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt)
// Ignore the supervisor model start, wait for the sub-agent model
// The supervisor model is fast (simpleChatModel), so it will start and finish quickly
_ = supervisorModelStarted
select {
case <-subAgentModelStarted:
case <-time.After(10 * time.Second):
t.Fatal("Sub-agent model did not start")
}
time.Sleep(50 * time.Millisecond)
start := time.Now()
handle, _ := cancelFn()
err = handle.Wait()
assert.NoError(t, err, "Cancel during sub-agent should succeed")
drainAndAssertCancelError(t, iter)
elapsed := time.Since(start)
assert.True(t, elapsed < 3*time.Second, "Should complete quickly after cancel, elapsed: %v", elapsed)
})
}
func TestFilterCancelOption(t *testing.T) {
t.Run("RemovesCancelOption", func(t *testing.T) {
cancelOpt, _ := WithCancel()
sessionOpt := WithSessionValues(map[string]any{"key": "value"})
opts := []AgentRunOption{cancelOpt, sessionOpt}
filtered := filterCancelOption(opts)
assert.Len(t, filtered, 1, "Should have removed the cancel option")
// Verify the remaining option is the session option
testOpt := &options{}
filtered[0].implSpecificOptFn.(func(*options))(testOpt)
assert.NotNil(t, testOpt.sessionValues)
assert.Nil(t, testOpt.cancelCtx)
})
t.Run("KeepsNonCancelOptions", func(t *testing.T) {
sessionOpt := WithSessionValues(map[string]any{"key": "value"})
callbackOpt := WithCallbacks()
opts := []AgentRunOption{sessionOpt, callbackOpt}
filtered := filterCancelOption(opts)
assert.Len(t, filtered, 2, "Should keep all non-cancel options")
})
t.Run("EmptyInput", func(t *testing.T) {
filtered := filterCancelOption(nil)
assert.Nil(t, filtered)
})
}
func wrapIterWithMarkDone(iter *AsyncIterator[*AgentEvent], cc *cancelContext) *AsyncIterator[*AgentEvent] {
if cc == nil {
return iter
}
outIter, outGen := NewAsyncIteratorPair[*AgentEvent]()
go func() {
defer cc.markDone()
defer outGen.Close()
for {
event, ok := iter.Next()
if !ok {
return
}
outGen.Send(event)
}
}()
return outIter
}
func TestWrapIterWithMarkDone(t *testing.T) {
t.Run("MarksDoneAfterDrain", func(t *testing.T) {
cc := newCancelContext()
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
go func() {
gen.Send(&AgentEvent{AgentName: "test"})
gen.Close()
}()
wrapped := wrapIterWithMarkDone(iter, cc)
event, ok := wrapped.Next()
assert.True(t, ok)
assert.Equal(t, "test", event.AgentName)
_, ok = wrapped.Next()
assert.False(t, ok)
// markDone should have been called, so doneChan should be closed
select {
case <-cc.doneChan:
// good
case <-time.After(time.Second):
t.Fatal("doneChan was not closed after drain")
}
})
t.Run("NilCancelContext_PassesThrough", func(t *testing.T) {
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
go func() {
gen.Send(&AgentEvent{AgentName: "test"})
gen.Close()
}()
wrapped := wrapIterWithMarkDone(iter, nil)
assert.Equal(t, iter, wrapped, "Should return same iter when cc is nil")
})
}
func TestGraphInterruptFuncs_Parallel(t *testing.T) {
t.Run("MultipleGraphInterruptFuncsAllCalled", func(t *testing.T) {
cc := newCancelContext()
var called1, called2 int32
cc.setGraphInterruptFunc(func(opts ...compose.GraphInterruptOption) {
atomic.AddInt32(&called1, 1)
})
cc.setGraphInterruptFunc(func(opts ...compose.GraphInterruptOption) {
atomic.AddInt32(&called2, 1)
})
// Simulate immediate cancel
cc.setMode(CancelImmediate)
atomic.CompareAndSwapInt32(&cc.state, stateRunning, stateCancelling)
close(cc.cancelChan)
cc.sendImmediateInterrupt()
assert.Equal(t, int32(1), atomic.LoadInt32(&called1), "First graph interrupt func should be called")
assert.Equal(t, int32(1), atomic.LoadInt32(&called2), "Second graph interrupt func should be called")
})
t.Run("RetroactiveFire_OnSetAfterCancel", func(t *testing.T) {
cc := newCancelContext()
// First set up cancel state with immediate interrupt
cc.setMode(CancelImmediate)
atomic.CompareAndSwapInt32(&cc.state, stateRunning, stateCancelling)
close(cc.cancelChan)
close(cc.immediateChan)
atomic.StoreInt32(&cc.interruptSent, interruptImmediate)
// Now register a new function - it should be retroactively fired
var called int32
cc.setGraphInterruptFunc(func(opts ...compose.GraphInterruptOption) {
atomic.AddInt32(&called, 1)
})
assert.Equal(t, int32(1), atomic.LoadInt32(&called), "setGraphInterruptFunc should retroactively fire new func")
})
}
// -- Tests for transition-point cancel (cancel between sub-agents) --
// gatedChatModel is a model that:
// - Signals doneChan when Generate completes
// - Optionally blocks on gateChan before returning (nil gateChan = no blocking)
// - Tracks call count via callCount
type gatedChatModel struct {
response *schema.Message
gateChan chan struct{} // if non-nil, blocks until closed before returning
doneChan chan struct{} // signalled after Generate completes
callCount int32
}
func (m *gatedChatModel) Generate(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
atomic.AddInt32(&m.callCount, 1)
if m.gateChan != nil {
select {
case <-m.gateChan:
case <-ctx.Done():
return nil, ctx.Err()
}
}
select {
case m.doneChan <- struct{}{}:
default:
}
return m.response, nil
}
func (m *gatedChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
msg, err := m.Generate(ctx, input, opts...)
if err != nil {
return nil, err
}
return schema.StreamReaderFromArray([]*schema.Message{msg}), nil
}
func (m *gatedChatModel) BindTools(tools []*schema.ToolInfo) error {
return nil
}
func TestCheckCancel_Sequential_BetweenSubAgents(t *testing.T) {
ctx := context.Background()
// CancelAfterToolCalls fires at transition boundaries between sub-agents.
// At a transition boundary, the completed sub-agent's entire execution
// (including any tool calls) is done, satisfying the CancelAfterToolCalls
// contract — even if this particular sub-agent had no tools.
model1 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent1 done"},
gateChan: make(chan struct{}),
doneChan: make(chan struct{}, 1),
}
model2 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent2 done"},
doneChan: make(chan struct{}, 1),
}
agent1, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent1", Description: "first", Instruction: "test", Model: model1,
})
assert.NoError(t, err)
agent2, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent2", Description: "second", Instruction: "test", Model: model2,
})
assert.NoError(t, err)
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq", Description: "sequential test", SubAgents: []Agent{agent1, agent2},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: seqAgent, EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("hello")}, cancelOpt)
for atomic.LoadInt32(&model1.callCount) == 0 {
runtime.Gosched()
}
cancelCalled, result := cancelAsync(cancelFn, WithAgentCancelMode(CancelAfterToolCalls))
waitForChan(t, cancelCalled, "cancelFn was not called")
close(model1.gateChan)
assert.NoError(t, result.waitDone(t), "CancelAfterToolCalls should succeed at transition boundary")
for {
_, ok := iter.Next()
if !ok {
break
}
}
assert.Equal(t, int32(1), atomic.LoadInt32(&model1.callCount), "Agent1 model should be invoked")
assert.Equal(t, int32(0), atomic.LoadInt32(&model2.callCount),
"Agent2 model should NOT be invoked (CancelAfterToolCalls caught at transition)")
}
func TestCheckCancel_Loop_BetweenIterations(t *testing.T) {
ctx := context.Background()
// CancelAfterToolCalls fires at loop iteration boundaries.
// After the first iteration completes, any tool calls it made are done,
// satisfying the CancelAfterToolCalls contract.
mdl := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "loop iter"},
gateChan: make(chan struct{}),
doneChan: make(chan struct{}, 10),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "loop_inner", Description: "inner", Instruction: "test", Model: mdl,
})
assert.NoError(t, err)
loopAgent, err := NewLoopAgent(ctx, &LoopAgentConfig{
Name: "loop", Description: "loop test", SubAgents: []Agent{agent}, MaxIterations: 3,
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: loopAgent, EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("hello")}, cancelOpt)
for atomic.LoadInt32(&mdl.callCount) == 0 {
runtime.Gosched()
}
cancelCalled, result := cancelAsync(cancelFn, WithAgentCancelMode(CancelAfterToolCalls))
waitForChan(t, cancelCalled, "cancelFn was not called")
close(mdl.gateChan)
assert.NoError(t, result.waitDone(t), "CancelAfterToolCalls should succeed at loop transition boundary")
for {
_, ok := iter.Next()
if !ok {
break
}
}
assert.Equal(t, int32(1), atomic.LoadInt32(&mdl.callCount),
"Model should be called once; second iteration caught at transition")
}
func TestCheckCancel_Parallel_PreSpawn(t *testing.T) {
ctx := context.Background()
// Cancel fires before Run is called. Neither model should be invoked.
model1 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "par1"},
doneChan: make(chan struct{}, 1),
}
model2 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "par2"},
doneChan: make(chan struct{}, 1),
}
agent1, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "par1", Description: "first", Instruction: "test", Model: model1,
})
assert.NoError(t, err)
agent2, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "par2", Description: "second", Instruction: "test", Model: model2,
})
assert.NoError(t, err)
parAgent, err := NewParallelAgent(ctx, &ParallelAgentConfig{
Name: "par", Description: "parallel test", SubAgents: []Agent{agent1, agent2},
})
assert.NoError(t, err)
// Fire cancel in goroutine (cancelFn blocks until handled)
cancelOpt, cancelFn := WithCancel()
cancelDone := make(chan error, 1)
go func() {
handle, _ := cancelFn()
cancelDone <- handle.Wait()
}()
// Wait for cancelChan to be closed (happens synchronously before the blocking doneChan wait)
time.Sleep(20 * time.Millisecond)
runner := NewRunner(ctx, RunnerConfig{
Agent: parAgent, EnableStreaming: false,
})
iter := runner.Run(ctx, []Message{schema.UserMessage("hello")}, cancelOpt)
var cancelErr *CancelError
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
cancelErr = ce
}
}
// cancelFn should have completed
select {
case err = <-cancelDone:
assert.NoError(t, err)
case <-time.After(5 * time.Second):
t.Fatal("cancelFn did not return")
}
assert.NotNil(t, cancelErr, "Should have CancelError")
assert.Equal(t, int32(0), atomic.LoadInt32(&model1.callCount), "First model should never be invoked")
assert.Equal(t, int32(0), atomic.LoadInt32(&model2.callCount), "Second model should never be invoked")
}
func TestCheckCancel_Transfer_BeforeTarget(t *testing.T) {
ctx := context.Background()
// Supervisor CMA returns a transfer action (instantly).
// Cancel fires after transfer action but before target runs.
// Target model should never be invoked.
supervisorModel := &simpleChatModel{
response: &schema.Message{
Role: schema.Assistant,
ToolCalls: []schema.ToolCall{{
ID: "call_1", Type: "function",
Function: schema.FunctionCall{
Name: TransferToAgentToolName,
Arguments: `{"agent_name": "target"}`,
},
}},
},
}
targetModel := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "target done"},
doneChan: make(chan struct{}, 1),
}
supervisorAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "supervisor", Description: "supervisor", Instruction: "test", Model: supervisorModel,
})
assert.NoError(t, err)
targetAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "target", Description: "target", Instruction: "test", Model: targetModel,
})
assert.NoError(t, err)
agentWithSub, err := SetSubAgents(ctx, supervisorAgent, []Agent{targetAgent})
assert.NoError(t, err)
// Fire cancel in goroutine (cancelFn blocks until handled)
cancelOpt, cancelFn := WithCancel()
cancelDone := make(chan error, 1)
go func() {
handle, _ := cancelFn()
cancelDone <- handle.Wait()
}()
time.Sleep(20 * time.Millisecond)
runner := NewRunner(ctx, RunnerConfig{
Agent: agentWithSub, EnableStreaming: false,
})
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt)
var cancelErr *CancelError
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
cancelErr = ce
}
}
select {
case err = <-cancelDone:
assert.NoError(t, err)
case <-time.After(5 * time.Second):
t.Fatal("cancelFn did not return")
}
assert.NotNil(t, cancelErr, "Should have CancelError")
assert.Equal(t, int32(0), atomic.LoadInt32(&targetModel.callCount), "Target model should never be invoked")
}
func TestCheckCancel_AlreadyHandled_NoDuplicate(t *testing.T) {
ctx := context.Background()
// In a sequential agent, if the first CMA handles the cancel (graph interrupt),
// the workflow's transition check should NOT emit a duplicate CancelError.
// Use a slow model so cancel fires during its execution (handled by CMA).
modelStarted := make(chan struct{}, 1)
model1 := &cancelTestChatModel{
delayNs: int64(2 * time.Second),
response: &schema.Message{Role: schema.Assistant, Content: "agent1"},
startedChan: modelStarted,
doneChan: make(chan struct{}, 1),
}
model2 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent2"},
doneChan: make(chan struct{}, 1),
}
agent1, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent1", Description: "first", Instruction: "test", Model: model1,
})
assert.NoError(t, err)
agent2, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent2", Description: "second", Instruction: "test", Model: model2,
})
assert.NoError(t, err)
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq", Description: "sequential", SubAgents: []Agent{agent1, agent2},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: seqAgent, EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt)
// Wait for model to start, then cancel during model execution
select {
case <-modelStarted:
case <-time.After(5 * time.Second):
t.Fatal("Model did not start")
}
time.Sleep(50 * time.Millisecond)
handle, _ := cancelFn()
err = handle.Wait()
assert.NoError(t, err)
cancelCount := 0
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
cancelCount++
}
}
assert.Equal(t, 1, cancelCount, "Should have exactly one CancelError, no duplicate from workflow transition")
assert.Equal(t, int32(0), atomic.LoadInt32(&model2.callCount), "Second agent should not run")
}
// Tests for CancelAfterChatModel/CancelAfterToolCalls in nested workflow structures.
// These verify that safe-point cancel modes propagate through the entire agent hierarchy
// and fire at whichever nested level reaches the safe-point first.
func TestCancel_SequentialWorkflow_CancelAfterChatModel(t *testing.T) {
ctx := context.Background()
agent1Started := make(chan struct{}, 1)
agent1 := newCancelTestAgentWithTools(t, "seq_slow", 500*time.Millisecond, agent1Started)
agent2 := newCancelTestAgentWithTools(t, "seq_fast", 50*time.Millisecond, make(chan struct{}, 1))
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq_agent",
Description: "Sequential workflow",
SubAgents: []Agent{agent1, agent2},
})
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{
Agent: seqAgent,
CheckPointStore: store,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt, WithCheckPointID("seq-cancel-1"))
select {
case <-agent1Started:
case <-time.After(10 * time.Second):
t.Fatal("First agent did not start")
}
handle, contributed := cancelFn(WithAgentCancelMode(CancelAfterChatModel))
assert.True(t, contributed, "Cancel should contribute")
err = handle.Wait()
assert.NoError(t, err)
hasCancelError := false
var cancelErr *CancelError
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil && errors.As(event.Err, &cancelErr) {
hasCancelError = true
}
}
assert.True(t, hasCancelError, "Should have CancelError")
assert.Equal(t, CancelAfterChatModel, cancelErr.Info.Mode)
assert.NotNil(t, cancelErr.interruptSignal, "CancelError should have interrupt signal for checkpoint")
resumeAgent1 := newCancelTestAgentWithToolsFinalAnswer(t, "seq_slow")
resumeAgent2 := newCancelTestAgentWithToolsFinalAnswer(t, "seq_fast")
resumeSeq, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq_agent",
Description: "Sequential workflow",
SubAgents: []Agent{resumeAgent1, resumeAgent2},
})
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{
Agent: resumeSeq,
CheckPointStore: store,
})
resumeIter, err := runner2.Resume(ctx, "seq-cancel-1")
assert.NoError(t, err)
assert.NotNil(t, resumeIter)
var resumeEvents []*AgentEvent
for {
event, ok := resumeIter.Next()
if !ok {
break
}
assert.Nil(t, event.Err, "Should not have error during resume")
resumeEvents = append(resumeEvents, event)
}
assert.NotEmpty(t, resumeEvents, "Resume should produce events")
}
func TestCancelImmediate_OrphanedToolGoroutine_NoPanic(t *testing.T) {
t.Run("unit_send_after_close", func(t *testing.T) {
_, gen := NewAsyncIteratorPair[*AgentEvent]()
cc := newCancelContext()
cc.setMode(CancelImmediate)
close(cc.cancelChan)
close(cc.immediateChan)
gen.Close()
execCtx := &chatModelAgentExecCtx{
generator: gen,
cancelCtx: cc,
}
assert.NotPanics(t, func() {
execCtx.send(&AgentEvent{AgentName: "test"})
}, "send after generator.Close must not panic")
})
t.Run("unit_send_after_close_without_cancel_ctx", func(t *testing.T) {
_, gen := NewAsyncIteratorPair[*AgentEvent]()
gen.Close()
execCtx := &chatModelAgentExecCtx{
generator: gen,
}
assert.NotPanics(t, func() {
execCtx.send(&AgentEvent{AgentName: "test"})
}, "send after generator.Close must not panic even without cancelCtx (trySend safety net)")
})
t.Run("unit_send_nil_execCtx", func(t *testing.T) {
var execCtx *chatModelAgentExecCtx
assert.NotPanics(t, func() {
execCtx.send(&AgentEvent{AgentName: "test"})
}, "send on nil execCtx must not panic")
})
t.Run("unit_send_nil_generator", func(t *testing.T) {
execCtx := &chatModelAgentExecCtx{}
assert.NotPanics(t, func() {
execCtx.send(&AgentEvent{AgentName: "test"})
}, "send with nil generator must not panic")
})
t.Run("unit_isImmediateCancelled_nil_cancelContext", func(t *testing.T) {
var cc *cancelContext
assert.False(t, cc.isImmediateCancelled(), "nil cancelContext should return false")
})
t.Run("unit_trySend_race_window", func(t *testing.T) {
_, gen := NewAsyncIteratorPair[*AgentEvent]()
cc := newCancelContext()
gen.Close()
execCtx := &chatModelAgentExecCtx{
generator: gen,
cancelCtx: cc,
}
assert.NotPanics(t, func() {
execCtx.send(&AgentEvent{AgentName: "test"})
}, "trySend must handle the case where isImmediateCancelled is false but generator is closed")
})
t.Run("unit_SendEvent_after_close", func(t *testing.T) {
_, gen := NewAsyncIteratorPair[*AgentEvent]()
cc := newCancelContext()
cc.setMode(CancelImmediate)
close(cc.cancelChan)
close(cc.immediateChan)
gen.Close()
execCtx := &chatModelAgentExecCtx{
generator: gen,
cancelCtx: cc,
}
ctx := withTypedChatModelAgentExecCtx(context.Background(), execCtx)
assert.NotPanics(t, func() {
err := SendEvent(ctx, &AgentEvent{AgentName: "test"})
assert.NoError(t, err)
}, "SendEvent after generator.Close must not panic")
})
t.Run("unit_SendEvent_no_execCtx", func(t *testing.T) {
err := SendEvent(context.Background(), &AgentEvent{AgentName: "test"})
assert.Error(t, err, "SendEvent without execCtx should return error")
})
t.Run("integration_cancel_escalation_orphans_tool", func(t *testing.T) {
ctx := context.Background()
toolStarted := make(chan struct{}, 1)
toolDone := make(chan struct{}, 1)
st := &slowToolWithSignal{
name: "orphan_tool",
delay: 2 * time.Second,
result: "tool result",
startedChan: toolStarted,
}
mdl := &simpleChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_orphan_1",
Type: "function",
Function: schema.FunctionCall{
Name: "orphan_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "OrphanTestAgent",
Description: "Test agent for orphaned tool goroutine panic",
Model: mdl,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
cancelOpt, cancelFn := WithCancel()
iter := agent.Run(ctx, &AgentInput{
Messages: []Message{schema.UserMessage("Use the tool")},
}, cancelOpt)
assert.NotNil(t, iter)
select {
case <-toolStarted:
case <-time.After(10 * time.Second):
t.Fatal("Tool did not start")
}
timeout := 50 * time.Millisecond
handle, contributed := cancelFn(
WithAgentCancelMode(CancelAfterChatModel),
WithAgentCancelTimeout(timeout),
)
assert.True(t, contributed, "Cancel should contribute")
err = handle.Wait()
assert.True(t, err == nil || errors.Is(err, ErrCancelTimeout),
"handle.Wait should return nil or ErrCancelTimeout, got: %v", err)
for {
_, ok := iter.Next()
if !ok {
break
}
}
go func() {
time.Sleep(3 * time.Second)
select {
case toolDone <- struct{}{}:
default:
}
}()
runtime.Gosched()
time.Sleep(3 * time.Second)
select {
case <-toolDone:
default:
}
})
}
// -- Tests for CancelImmediate in nested agent structures --
func newTestChatModel(response *schema.Message, delay time.Duration) *cancelTestChatModel {
m := &cancelTestChatModel{
response: response,
startedChan: make(chan struct{}, 1),
doneChan: make(chan struct{}, 1),
}
if delay > 0 {
m.setDelay(delay)
}
return m
}
func newToolCallResponse(toolName string) *schema.Message {
return &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{ID: "call_1", Type: "function", Function: schema.FunctionCall{Name: toolName, Arguments: `{}`}},
},
}
}
func newAgentWithTool(t *testing.T, ctx context.Context, name string, mdl model.BaseChatModel, subAgent Agent) (Agent, error) {
t.Helper()
return NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: name,
Description: name,
Model: mdl,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{NewAgentTool(ctx, subAgent)},
},
},
})
}
func waitForChan(t *testing.T, ch <-chan struct{}, msg string) {
t.Helper()
select {
case <-ch:
case <-time.After(10 * time.Second):
t.Fatal(msg)
}
}
func drainCancelError(t *testing.T, iter *AsyncIterator[*AgentEvent]) *CancelError {
t.Helper()
var cancelErr *CancelError
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil {
errors.As(event.Err, &cancelErr)
}
}
return cancelErr
}
func drainResumeErrors(t *testing.T, iter *AsyncIterator[*AgentEvent]) []error {
t.Helper()
var errs []error
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil {
errs = append(errs, event.Err)
}
}
return errs
}
type cancelResult struct {
err error
contributed bool
done chan struct{}
}
func cancelAsync(cancelFn AgentCancelFunc, opts ...AgentCancelOption) (cancelCalled chan struct{}, result *cancelResult) {
cancelCalled = make(chan struct{})
result = &cancelResult{done: make(chan struct{})}
go func() {
handle, contributed := cancelFn(opts...)
result.contributed = contributed
close(cancelCalled)
result.err = handle.Wait()
close(result.done)
}()
return
}
func (r *cancelResult) waitDone(t *testing.T) error {
t.Helper()
select {
case <-r.done:
return r.err
case <-time.After(10 * time.Second):
t.Fatal("cancel did not complete")
return nil
}
}
func TestCancelImmediate_AgentTool_PreservesChildCheckpoint(t *testing.T) {
ctx := context.Background()
leafModel := newTestChatModel(
&schema.Message{Role: schema.Assistant, Content: "leaf response"}, 2*time.Second)
leafAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "leaf_agent", Description: "Leaf agent in agentTool", Model: leafModel,
})
assert.NoError(t, err)
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "inner_seq", Description: "Inner sequential workflow", SubAgents: []Agent{leafAgent},
})
assert.NoError(t, err)
rootModel := newTestChatModel(newToolCallResponse("inner_seq"), 0)
rootAgent, err := newAgentWithTool(t, ctx, "root_agent", rootModel, seqAgent)
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{Agent: rootAgent, CheckPointStore: store})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt, WithCheckPointID("immediate-agent-tool-1"))
waitForChan(t, leafModel.startedChan, "Leaf agent model did not start")
handle, contributed := cancelFn(WithRecursive())
assert.True(t, contributed)
assert.NoError(t, handle.Wait())
cancelErr := drainCancelError(t, iter)
assert.NotNil(t, cancelErr, "Should have CancelError from CancelImmediate through agentTool")
assert.NotNil(t, cancelErr.interruptSignal)
resumeLeaf, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "leaf_agent", Description: "Leaf agent in agentTool",
Model: newTestChatModel(&schema.Message{Role: schema.Assistant, Content: "resumed leaf"}, 0),
})
assert.NoError(t, err)
resumeSeq, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "inner_seq", Description: "Inner sequential workflow", SubAgents: []Agent{resumeLeaf},
})
assert.NoError(t, err)
resumeRoot, err := newAgentWithTool(t, ctx, "root_agent",
newTestChatModel(&schema.Message{Role: schema.Assistant, Content: "resumed root"}, 0), resumeSeq)
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{Agent: resumeRoot, CheckPointStore: store})
resumeIter, err := runner2.Resume(ctx, "immediate-agent-tool-1")
assert.NoError(t, err)
assert.Empty(t, drainResumeErrors(t, resumeIter), "Resume should complete without errors")
}
func TestCancelImmediate_ParallelWorkflow_WithAgentTool(t *testing.T) {
ctx := context.Background()
leafModel := newTestChatModel(
&schema.Message{Role: schema.Assistant, Content: "leaf response"}, 2*time.Second)
leafAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "leaf_agent", Description: "Leaf agent in agentTool", Model: leafModel,
})
assert.NoError(t, err)
agentWithTool, err := newAgentWithTool(t, ctx, "agent_with_tool",
newTestChatModel(newToolCallResponse("leaf_agent"), 0), leafAgent)
assert.NoError(t, err)
simpleStarted := make(chan struct{}, 1)
simpleAgent := newCancelTestAgent(t, "simple_agent", 2*time.Second, simpleStarted)
parAgent, err := NewParallelAgent(ctx, &ParallelAgentConfig{
Name: "par_agent", Description: "Parallel with agentTool and simple agent",
SubAgents: []Agent{agentWithTool, simpleAgent},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{Agent: parAgent, EnableStreaming: false})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt)
waitForChan(t, leafModel.startedChan, "Leaf agent did not start")
waitForChan(t, simpleStarted, "Simple agent did not start")
start := time.Now()
handle, _ := cancelFn()
assert.NoError(t, handle.Wait())
cancelErr := drainCancelError(t, iter)
elapsed := time.Since(start)
assert.NotNil(t, cancelErr, "Should have CancelError from parallel with agentTool")
assert.True(t, elapsed < 5*time.Second, "Should complete quickly after cancel, elapsed: %v", elapsed)
}
type cancelUnawareAgent struct {
name string
desc string
delay time.Duration
response string
}
type multiResponseGatedModel struct {
responses []*schema.Message
gateChan chan struct{}
gateOnce bool
gated int32
doneChan chan struct{}
callCount int32
}
func (m *multiResponseGatedModel) Generate(ctx context.Context, msgs []*schema.Message, opts ...model.Option) (*schema.Message, error) {
idx := atomic.AddInt32(&m.callCount, 1)
if m.gateChan != nil && (!m.gateOnce || atomic.CompareAndSwapInt32(&m.gated, 0, 1)) {
select {
case <-m.gateChan:
case <-ctx.Done():
return nil, ctx.Err()
}
}
if len(m.responses) == 0 {
return nil, fmt.Errorf("multiResponseGatedModel: no responses configured")
}
resp := m.responses[(int(idx)-1)%len(m.responses)]
if m.doneChan != nil {
select {
case m.doneChan <- struct{}{}:
default:
}
}
return resp, nil
}
func (m *multiResponseGatedModel) Stream(ctx context.Context, msgs []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
resp, err := m.Generate(ctx, msgs, opts...)
if err != nil {
return nil, err
}
return schema.StreamReaderFromArray([]*schema.Message{resp}), nil
}
func (m *multiResponseGatedModel) BindTools(tools []*schema.ToolInfo) error { return nil }
func (a *cancelUnawareAgent) Name(_ context.Context) string { return a.name }
func (a *cancelUnawareAgent) Description(_ context.Context) string { return a.desc }
func (a *cancelUnawareAgent) Run(_ context.Context, _ *AgentInput, _ ...AgentRunOption) *AsyncIterator[*AgentEvent] {
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
go func() {
defer gen.Close()
// Intentionally ignores ctx.Done() — simulates a custom agent that
// does not participate in the cancel protocol at all.
// Delay is kept short (relative to grace period) to avoid goroutine
// leak lasting long after the test completes.
time.Sleep(a.delay)
}()
return iter
}
func TestCancelImmediate_CustomAgent_GracePeriodFallback(t *testing.T) {
ctx := context.Background()
customAgent := &cancelUnawareAgent{
name: "custom_slow", desc: "A custom agent that ignores cancel",
delay: 5 * time.Second, response: "custom response",
}
rootModel := newTestChatModel(newToolCallResponse("custom_slow"), 0)
rootAgent, err := newAgentWithTool(t, ctx, "root_agent", rootModel, customAgent)
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{Agent: rootAgent, EnableStreaming: false})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt)
waitForChan(t, rootModel.startedChan, "Root model did not start")
waitForChan(t, rootModel.doneChan, "Root model did not finish")
start := time.Now()
handle, _ := cancelFn()
assert.NoError(t, handle.Wait())
cancelErr := drainCancelError(t, iter)
elapsed := time.Since(start)
assert.NotNil(t, cancelErr, "Should have CancelError (from grace period fallback)")
assert.True(t, elapsed < 5*time.Second,
"Should complete within grace period + overhead, elapsed: %v", elapsed)
}
func TestCancelImmediate_MultiLevelNesting(t *testing.T) {
ctx := context.Background()
innerLeafModel := newTestChatModel(
&schema.Message{Role: schema.Assistant, Content: "inner leaf response"}, 2*time.Second)
innerLeafAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "inner_leaf", Description: "Innermost leaf agent", Model: innerLeafModel,
})
assert.NoError(t, err)
middleAgent, err := newAgentWithTool(t, ctx, "middle_agent",
newTestChatModel(newToolCallResponse("inner_leaf"), 0), innerLeafAgent)
assert.NoError(t, err)
rootAgent, err := newAgentWithTool(t, ctx, "root_agent",
newTestChatModel(newToolCallResponse("middle_agent"), 0), middleAgent)
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{Agent: rootAgent, CheckPointStore: store})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt, WithCheckPointID("multi-level-1"))
waitForChan(t, innerLeafModel.startedChan, "Inner leaf model did not start")
start := time.Now()
handle, contributed := cancelFn()
assert.True(t, contributed)
assert.NoError(t, handle.Wait())
cancelErr := drainCancelError(t, iter)
elapsed := time.Since(start)
assert.NotNil(t, cancelErr, "Should have CancelError from multi-level nesting")
assert.NotNil(t, cancelErr.interruptSignal)
assert.True(t, elapsed < 5*time.Second, "Should complete quickly, elapsed: %v", elapsed)
resumeInnerLeaf, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "inner_leaf", Description: "Innermost leaf agent",
Model: newTestChatModel(&schema.Message{Role: schema.Assistant, Content: "resumed inner leaf"}, 0),
})
assert.NoError(t, err)
resumeMiddle, err := newAgentWithTool(t, ctx, "middle_agent",
newTestChatModel(&schema.Message{Role: schema.Assistant, Content: "resumed middle"}, 0), resumeInnerLeaf)
assert.NoError(t, err)
resumeRoot, err := newAgentWithTool(t, ctx, "root_agent",
newTestChatModel(&schema.Message{Role: schema.Assistant, Content: "resumed root"}, 0), resumeMiddle)
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{Agent: resumeRoot, CheckPointStore: store})
resumeIter, err := runner2.Resume(ctx, "multi-level-1")
assert.NoError(t, err)
assert.Empty(t, drainResumeErrors(t, resumeIter), "Resume should complete without errors")
}
func TestCancelImmediate_SequentialTransitionBoundary(t *testing.T) {
ctx := context.Background()
model1 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent1 done"},
gateChan: make(chan struct{}),
doneChan: make(chan struct{}, 1),
}
model2 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent2 done"},
doneChan: make(chan struct{}, 1),
}
agent1, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent1", Description: "first", Instruction: "test", Model: model1,
})
assert.NoError(t, err)
agent2, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent2", Description: "second", Instruction: "test", Model: model2,
})
assert.NoError(t, err)
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq", Description: "sequential test", SubAgents: []Agent{agent1, agent2},
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: seqAgent, EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("hello")}, cancelOpt)
for atomic.LoadInt32(&model1.callCount) == 0 {
runtime.Gosched()
}
cancelCalled, result := cancelAsync(cancelFn)
waitForChan(t, cancelCalled, "cancelFn was not called")
close(model1.gateChan)
assert.NoError(t, result.waitDone(t), "CancelImmediate should succeed at transition")
cancelErr := drainCancelError(t, iter)
assert.NotNil(t, cancelErr, "Should have CancelError at transition boundary")
assert.Equal(t, int32(1), atomic.LoadInt32(&model1.callCount), "Agent1 model should be invoked")
assert.Equal(t, int32(0), atomic.LoadInt32(&model2.callCount), "Agent2 model should NOT be invoked (caught at transition)")
}
func TestCancelImmediate_LoopTransitionBoundary(t *testing.T) {
ctx := context.Background()
mdl := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "loop iter"},
gateChan: make(chan struct{}),
doneChan: make(chan struct{}, 10),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "loop_inner", Description: "inner", Instruction: "test", Model: mdl,
})
assert.NoError(t, err)
loopAgent, err := NewLoopAgent(ctx, &LoopAgentConfig{
Name: "loop", Description: "loop test", SubAgents: []Agent{agent}, MaxIterations: 5,
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: loopAgent, EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("hello")}, cancelOpt)
for atomic.LoadInt32(&mdl.callCount) == 0 {
runtime.Gosched()
}
cancelCalled, result := cancelAsync(cancelFn)
waitForChan(t, cancelCalled, "cancelFn was not called")
close(mdl.gateChan)
assert.NoError(t, result.waitDone(t), "CancelImmediate should succeed at loop transition")
for {
_, ok := iter.Next()
if !ok {
break
}
}
assert.Equal(t, int32(1), atomic.LoadInt32(&mdl.callCount),
"Model should be called once; second iteration caught at transition")
}
func TestCancelAfterChatModel_SequentialTransitionBoundary(t *testing.T) {
ctx := context.Background()
model1 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent1 done"},
gateChan: make(chan struct{}),
doneChan: make(chan struct{}, 1),
}
model2 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent2 done"},
doneChan: make(chan struct{}, 1),
}
agent1, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent1", Description: "first", Instruction: "test", Model: model1,
})
assert.NoError(t, err)
agent2, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent2", Description: "second", Instruction: "test", Model: model2,
})
assert.NoError(t, err)
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq", Description: "sequential test", SubAgents: []Agent{agent1, agent2},
})
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{
Agent: seqAgent,
EnableStreaming: false,
CheckPointStore: store,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("hello")}, cancelOpt, WithCheckPointID("chatmodel-transition-1"))
for atomic.LoadInt32(&model1.callCount) == 0 {
runtime.Gosched()
}
cancelCalled, result := cancelAsync(cancelFn, WithAgentCancelMode(CancelAfterChatModel))
waitForChan(t, cancelCalled, "cancelFn was not called")
close(model1.gateChan)
assert.NoError(t, result.waitDone(t), "CancelAfterChatModel should succeed at transition boundary")
var cancelErr *CancelError
for {
event, ok := iter.Next()
if !ok {
break
}
var ce *CancelError
if event.Err != nil && errors.As(event.Err, &ce) {
cancelErr = ce
}
}
assert.NotNil(t, cancelErr, "Should have CancelError at transition boundary")
assert.Equal(t, CancelAfterChatModel, cancelErr.Info.Mode)
assert.Equal(t, int32(1), atomic.LoadInt32(&model1.callCount), "Agent1 model should be invoked")
assert.Equal(t, int32(0), atomic.LoadInt32(&model2.callCount),
"Agent2 model should NOT be invoked (CancelAfterChatModel caught at transition)")
}
func TestCancelAfterChatModel_Sequential_Agent1CompletesCancelBeforeAgent2Resume(t *testing.T) {
ctx := context.Background()
model1 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent1 done"},
gateChan: make(chan struct{}),
doneChan: make(chan struct{}, 1),
}
model2 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent2 done"},
doneChan: make(chan struct{}, 1),
}
model3 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "agent3 done"},
doneChan: make(chan struct{}, 1),
}
agent1, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent1", Description: "first", Instruction: "test", Model: model1,
})
assert.NoError(t, err)
agent2, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent2", Description: "second", Instruction: "test", Model: model2,
})
assert.NoError(t, err)
agent3, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent3", Description: "third", Instruction: "test", Model: model3,
})
assert.NoError(t, err)
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq", Description: "3-agent sequential", SubAgents: []Agent{agent1, agent2, agent3},
})
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{
Agent: seqAgent, CheckPointStore: store, EnableStreaming: false,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("hello")}, cancelOpt,
WithCheckPointID("seq-transition-resume-1"))
for atomic.LoadInt32(&model1.callCount) == 0 {
runtime.Gosched()
}
cancelCalled, result := cancelAsync(cancelFn, WithAgentCancelMode(CancelAfterChatModel))
waitForChan(t, cancelCalled, "cancelFn was not called")
close(model1.gateChan)
assert.NoError(t, result.waitDone(t))
cancelErr := drainCancelError(t, iter)
assert.NotNil(t, cancelErr, "Should have CancelError")
assert.Equal(t, CancelAfterChatModel, cancelErr.Info.Mode)
assert.Equal(t, int32(1), atomic.LoadInt32(&model1.callCount))
assert.Equal(t, int32(0), atomic.LoadInt32(&model2.callCount),
"Agent2 should NOT run (cancel caught at transition after agent1)")
assert.Equal(t, int32(0), atomic.LoadInt32(&model3.callCount))
resumeModel2 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "resumed agent2"},
doneChan: make(chan struct{}, 1),
}
resumeModel3 := &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "resumed agent3"},
doneChan: make(chan struct{}, 1),
}
resumeAgent1, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent1", Description: "first", Instruction: "test",
Model: &gatedChatModel{
response: &schema.Message{Role: schema.Assistant, Content: "should not run"},
doneChan: make(chan struct{}, 1),
},
})
assert.NoError(t, err)
resumeAgent2, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent2", Description: "second", Instruction: "test", Model: resumeModel2,
})
assert.NoError(t, err)
resumeAgent3, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent3", Description: "third", Instruction: "test", Model: resumeModel3,
})
assert.NoError(t, err)
resumeSeq, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq", Description: "3-agent sequential",
SubAgents: []Agent{resumeAgent1, resumeAgent2, resumeAgent3},
})
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{
Agent: resumeSeq, CheckPointStore: store, EnableStreaming: false,
})
resumeIter, err := runner2.Resume(ctx, "seq-transition-resume-1")
assert.NoError(t, err)
assert.Empty(t, drainResumeErrors(t, resumeIter), "Resume should complete without errors")
assert.Equal(t, int32(1), atomic.LoadInt32(&resumeModel2.callCount),
"Agent2 should run on resume")
assert.Equal(t, int32(1), atomic.LoadInt32(&resumeModel3.callCount),
"Agent3 should run on resume")
}
func TestCancelAfterToolCalls_LoopTransitionBoundary(t *testing.T) {
ctx := context.Background()
// Model that returns tool calls on odd calls and no tools on even calls.
// This completes one ReAct cycle per pair of calls:
// call 1 (gated): returns tool call → tool runs → call 2: returns no tools → END
// The gate only blocks the very first call. After that, all calls proceed instantly.
mdl := &multiResponseGatedModel{
responses: []*schema.Message{
{Role: schema.Assistant, ToolCalls: []schema.ToolCall{{
ID: "call_1", Type: "function",
Function: schema.FunctionCall{Name: "loop_tool", Arguments: `{"input": "test"}`},
}}},
{Role: schema.Assistant, Content: "iteration done"},
},
gateChan: make(chan struct{}),
gateOnce: true,
doneChan: make(chan struct{}, 10),
}
st := &slowTool{
name: "loop_tool",
delay: 10 * time.Millisecond,
result: "tool done",
startedChan: make(chan struct{}, 10),
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "loop_inner", Description: "inner", Instruction: "test", Model: mdl,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
loopAgent, err := NewLoopAgent(ctx, &LoopAgentConfig{
Name: "loop", Description: "loop test", SubAgents: []Agent{agent}, MaxIterations: 10,
})
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{Agent: loopAgent, CheckPointStore: store})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt, WithCheckPointID("toolcalls-loop-1"))
// Wait for the model to be entered (blocked on gate)
for atomic.LoadInt32(&mdl.callCount) == 0 {
runtime.Gosched()
}
// Fire cancel, wait for it to be registered, then release the gate
cancelCalled, result := cancelAsync(cancelFn, WithAgentCancelMode(CancelAfterToolCalls))
waitForChan(t, cancelCalled, "cancelFn was not called")
close(mdl.gateChan)
// Iteration 1 completes fully (model→tool→model-no-tools→END).
// The CancelAfterToolCalls safe-point inside ReAct fires after tool calls,
// OR the transition boundary catches it before iteration 2.
// Note: this test doesn't deterministically distinguish which path fires —
// both are semantically correct for CancelAfterToolCalls. The transition-
// boundary code path for CancelAfterToolCalls in loops is not definitively
// covered here because the ReAct safe-point may handle it first.
assert.NoError(t, result.waitDone(t))
cancelErr := drainCancelError(t, iter)
assert.NotNil(t, cancelErr, "Should have CancelError from CancelAfterToolCalls in loop")
assert.Equal(t, CancelAfterToolCalls, cancelErr.Info.Mode)
}
func TestCancelContext_RecursiveGraceBoundary(t *testing.T) {
t.Run("AgentToolDescendantMarker_PropagatesToParents", func(t *testing.T) {
parent := newCancelContext()
ctx := context.Background()
child := parent.deriveAgentToolCancelContext(ctx)
grandchild := child.deriveAgentToolCancelContext(ctx)
t.Cleanup(func() {
grandchild.markDone()
child.markDone()
})
assert.False(t, parent.hasAgentToolDescendant())
assert.False(t, child.hasAgentToolDescendant())
grandchild.markAgentToolDescendant()
assert.True(t, grandchild.hasAgentToolDescendant())
assert.True(t, child.hasAgentToolDescendant())
assert.True(t, parent.hasAgentToolDescendant())
})
}
func TestCancel_ParallelWorkflow_CancelAfterChatModel(t *testing.T) {
ctx := context.Background()
slowStarted := make(chan struct{}, 1)
slowAgent := newCancelTestAgentWithTools(t, "par_slow", 1*time.Second, slowStarted)
fastAgent := newCancelTestAgentWithTools(t, "par_fast", 50*time.Millisecond, make(chan struct{}, 1))
parAgent, err := NewParallelAgent(ctx, &ParallelAgentConfig{
Name: "par_agent",
Description: "Parallel workflow",
SubAgents: []Agent{slowAgent, fastAgent},
})
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{
Agent: parAgent,
CheckPointStore: store,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt, WithCheckPointID("par-cancel-1"))
select {
case <-slowStarted:
case <-time.After(10 * time.Second):
t.Fatal("Slow agent did not start")
}
handle, contributed := cancelFn(WithAgentCancelMode(CancelAfterChatModel))
assert.True(t, contributed, "Cancel should contribute")
err = handle.Wait()
assert.NoError(t, err)
hasCancelError := false
var cancelErr *CancelError
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil && errors.As(event.Err, &cancelErr) {
hasCancelError = true
}
}
assert.True(t, hasCancelError, "Should have CancelError from parallel workflow")
assert.Equal(t, CancelAfterChatModel, cancelErr.Info.Mode)
resumeSlow := newCancelTestAgentWithToolsFinalAnswer(t, "par_slow")
resumeFast := newCancelTestAgentWithToolsFinalAnswer(t, "par_fast")
resumePar, err := NewParallelAgent(ctx, &ParallelAgentConfig{
Name: "par_agent",
Description: "Parallel workflow",
SubAgents: []Agent{resumeSlow, resumeFast},
})
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{
Agent: resumePar,
CheckPointStore: store,
})
resumeIter, err := runner2.Resume(ctx, "par-cancel-1")
assert.NoError(t, err)
assert.NotNil(t, resumeIter)
var resumeErrors []error
for {
event, ok := resumeIter.Next()
if !ok {
break
}
if event.Err != nil {
resumeErrors = append(resumeErrors, event.Err)
}
}
assert.Empty(t, resumeErrors, "Resume should complete without errors")
}
func TestCancel_LoopWorkflow_CancelAfterChatModel(t *testing.T) {
ctx := context.Background()
modelStarted := make(chan struct{}, 10)
agent := newCancelTestAgentWithTools(t, "loop_inner", 500*time.Millisecond, modelStarted)
loopAgent, err := NewLoopAgent(ctx, &LoopAgentConfig{
Name: "loop_agent",
Description: "Loop workflow",
SubAgents: []Agent{agent},
MaxIterations: 10,
})
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{
Agent: loopAgent,
CheckPointStore: store,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt, WithCheckPointID("loop-cancel-1"))
select {
case <-modelStarted:
case <-time.After(10 * time.Second):
t.Fatal("Model did not start")
}
handle, contributed := cancelFn(WithAgentCancelMode(CancelAfterChatModel))
assert.True(t, contributed, "Cancel should contribute")
err = handle.Wait()
assert.NoError(t, err)
hasCancelError := false
var cancelErr *CancelError
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil && errors.As(event.Err, &cancelErr) {
hasCancelError = true
}
}
assert.True(t, hasCancelError, "Should have CancelError from loop workflow")
assert.Equal(t, CancelAfterChatModel, cancelErr.Info.Mode)
resumeAgent := newCancelTestAgentWithToolsFinalAnswer(t, "loop_inner")
resumeLoop, err := NewLoopAgent(ctx, &LoopAgentConfig{
Name: "loop_agent",
Description: "Loop workflow",
SubAgents: []Agent{resumeAgent},
MaxIterations: 10,
})
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{
Agent: resumeLoop,
CheckPointStore: store,
})
resumeIter, err := runner2.Resume(ctx, "loop-cancel-1")
assert.NoError(t, err)
assert.NotNil(t, resumeIter)
var resumeEvents []*AgentEvent
for {
event, ok := resumeIter.Next()
if !ok {
break
}
assert.Nil(t, event.Err, "Should not have error during resume")
resumeEvents = append(resumeEvents, event)
}
assert.NotEmpty(t, resumeEvents, "Resume should produce events")
}
func TestCancel_NestedWorkflow_AgentTool_CancelAfterChatModel(t *testing.T) {
// Structure: Runner -> RootCMA (with tools) -> agentTool -> flowAgent -> seqWorkflow -> LeafCMA
ctx := context.Background()
leafStarted := make(chan struct{}, 1)
leafModel := &cancelTestChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "leaf response",
},
startedChan: leafStarted,
doneChan: make(chan struct{}, 1),
}
leafModel.setDelay(500 * time.Millisecond)
leafAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "leaf_agent",
Description: "Leaf agent in workflow",
Model: leafModel,
})
assert.NoError(t, err)
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "inner_seq",
Description: "Inner sequential workflow",
SubAgents: []Agent{leafAgent},
})
assert.NoError(t, err)
rootModel := &cancelTestChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "inner_seq",
Arguments: `{}`,
},
},
},
},
startedChan: make(chan struct{}, 1),
doneChan: make(chan struct{}, 1),
}
rootAgent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "root_agent",
Description: "Root agent",
Model: rootModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{NewAgentTool(ctx, seqAgent)},
},
},
})
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{
Agent: rootAgent,
CheckPointStore: store,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("test")}, cancelOpt, WithCheckPointID("nested-cancel-1"))
select {
case <-leafStarted:
case <-time.After(10 * time.Second):
t.Fatal("Leaf agent model did not start")
}
handle, contributed := cancelFn(WithAgentCancelMode(CancelAfterChatModel), WithRecursive())
assert.True(t, contributed, "Cancel should contribute")
err = handle.Wait()
assert.NoError(t, err)
hasCancelError := false
var cancelErr *CancelError
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil && errors.As(event.Err, &cancelErr) {
hasCancelError = true
}
}
assert.True(t, hasCancelError, "Should have CancelError from deeply nested workflow")
assert.Equal(t, CancelAfterChatModel, cancelErr.Info.Mode)
assert.NotNil(t, cancelErr.interruptSignal, "CancelError should carry interrupt signal through agent tree")
// Phase 2: Resume from checkpoint — new instances to avoid data races
resumeLeafModel := &cancelTestChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "resumed leaf response",
},
startedChan: make(chan struct{}, 1),
doneChan: make(chan struct{}, 1),
}
resumeLeaf, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "leaf_agent",
Description: "Leaf agent in workflow",
Model: resumeLeafModel,
})
assert.NoError(t, err)
resumeSeq, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "inner_seq",
Description: "Inner sequential workflow",
SubAgents: []Agent{resumeLeaf},
})
assert.NoError(t, err)
resumeRootModel := &cancelTestChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "resumed root response",
},
startedChan: make(chan struct{}, 1),
doneChan: make(chan struct{}, 1),
}
resumeRoot, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "root_agent",
Description: "Root agent",
Model: resumeRootModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{NewAgentTool(ctx, resumeSeq)},
},
},
})
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{
Agent: resumeRoot,
CheckPointStore: store,
})
resumeIter, err := runner2.Resume(ctx, "nested-cancel-1")
assert.NoError(t, err)
assert.NotNil(t, resumeIter)
var resumeErrors []error
for {
event, ok := resumeIter.Next()
if !ok {
break
}
if event.Err != nil {
resumeErrors = append(resumeErrors, event.Err)
}
}
assert.Empty(t, resumeErrors, "Resume should complete without errors")
}
func TestCancel_CancelAfterToolCalls_InSequentialWorkflow(t *testing.T) {
ctx := context.Background()
toolStarted := make(chan struct{}, 1)
st := &slowTool{
name: "slow_tool",
delay: 200 * time.Millisecond,
result: "tool done",
startedChan: toolStarted,
}
modelWithToolCall := &simpleChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "",
ToolCalls: []schema.ToolCall{
{
ID: "call_1",
Type: "function",
Function: schema.FunctionCall{
Name: "slow_tool",
Arguments: `{"input": "test"}`,
},
},
},
},
}
agentWithTools, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent_with_tools",
Description: "Agent with slow tool",
Model: modelWithToolCall,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{st},
},
},
})
assert.NoError(t, err)
seqAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq_agent",
Description: "Sequential workflow with tool agent",
SubAgents: []Agent{agentWithTools},
})
assert.NoError(t, err)
store := newCancelTestStore()
runner := NewRunner(ctx, RunnerConfig{
Agent: seqAgent,
CheckPointStore: store,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("Use the tool")}, cancelOpt, WithCheckPointID("tool-cancel-1"))
select {
case <-toolStarted:
case <-time.After(10 * time.Second):
t.Fatal("Tool did not start")
}
// Cancel after tool calls — should wait for the tool to finish, then cancel
handle, contributed := cancelFn(WithAgentCancelMode(CancelAfterToolCalls))
assert.True(t, contributed, "Cancel should contribute")
err = handle.Wait()
assert.NoError(t, err)
hasCancelError := false
var cancelErr *CancelError
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil && errors.As(event.Err, &cancelErr) {
hasCancelError = true
}
}
assert.True(t, hasCancelError, "Should have CancelError after tool calls complete")
assert.Equal(t, CancelAfterToolCalls, cancelErr.Info.Mode)
// Phase 2: Resume from checkpoint — new instances
resumeTool := &slowTool{
name: "slow_tool",
delay: 50 * time.Millisecond,
result: "resumed tool done",
startedChan: make(chan struct{}, 1),
}
resumeModel := &simpleChatModel{
response: &schema.Message{
Role: schema.Assistant,
Content: "resumed response after tool",
},
}
resumeAgentWithTools, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "agent_with_tools",
Description: "Agent with slow tool",
Model: resumeModel,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{resumeTool},
},
},
})
assert.NoError(t, err)
resumeSeq, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "seq_agent",
Description: "Sequential workflow with tool agent",
SubAgents: []Agent{resumeAgentWithTools},
})
assert.NoError(t, err)
runner2 := NewRunner(ctx, RunnerConfig{
Agent: resumeSeq,
CheckPointStore: store,
})
resumeIter, err := runner2.Resume(ctx, "tool-cancel-1")
assert.NoError(t, err)
assert.NotNil(t, resumeIter)
var resumeEvents []*AgentEvent
for {
event, ok := resumeIter.Next()
if !ok {
break
}
assert.Nil(t, event.Err, "Should not have error during resume")
resumeEvents = append(resumeEvents, event)
}
assert.NotEmpty(t, resumeEvents, "Resume should produce events")
}
// TestCancel_SafePointNeverFires_ErrExecutionEnded verifies the waitForCompletion
// path where a safe-point cancel is submitted while the agent is running, but
// the agent finishes without hitting the requested safe-point (e.g.
// CancelAfterToolCalls on an agent with no tool calls). The cancel CAS succeeds
// (stateRunning → stateCancelling), but the agent completes normally (markDone →
// stateDone), so waitForCompletion returns ErrExecutionEnded.
func TestCancel_SafePointNeverFires_ErrExecutionEnded(t *testing.T) {
ctx := context.Background()
gate := make(chan struct{})
done := make(chan struct{}, 1)
m := &gatedChatModel{
gateChan: gate,
doneChan: done,
response: &schema.Message{
Role: schema.Assistant,
Content: "Final answer, no tool calls",
},
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "NoToolAgent",
Description: "Agent with no tools",
Instruction: "You are a test assistant",
Model: m,
})
assert.NoError(t, err)
runner := NewRunner(ctx, RunnerConfig{
Agent: agent,
})
cancelOpt, cancelFn := WithCancel()
iter := runner.Run(ctx, []Message{schema.UserMessage("hello")}, cancelOpt)
// Wait a moment for the agent to enter Generate and block on gateChan.
runtime.Gosched()
time.Sleep(50 * time.Millisecond)
// Submit a safe-point cancel for tool calls. The agent has no tools,
// so this safe-point will never fire.
handle, submitted := cancelFn(WithAgentCancelMode(CancelAfterToolCalls))
assert.True(t, submitted)
// Let the model complete. The agent finishes without hitting the tool
// calls safe-point → markDone → stateDone → waitForCompletion returns
// ErrExecutionEnded.
close(gate)
waitErr := handle.Wait()
assert.ErrorIs(t, waitErr, ErrExecutionEnded)
for {
_, ok := iter.Next()
if !ok {
break
}
}
}
// TestBuildCancelFunc_StateDoneUnderLock exercises the race-condition path
// in buildCancelFunc where the state transitions to stateDone between the
// lockless check and the locked check (cancel.go L732-734).
func TestBuildCancelFunc_StateDoneUnderLock(t *testing.T) {
cc := newCancelContext()
cancelFn := cc.buildCancelFunc()
// Hold cancelMu so the cancel func blocks when it tries to acquire the lock.
cc.cancelMu.Lock()
type result struct {
handle *CancelHandle
ok bool
}
ch := make(chan result, 1)
go func() {
h, ok := cancelFn(WithAgentCancelMode(CancelAfterToolCalls))
ch <- result{h, ok}
}()
// Give the goroutine time to reach the Lock() call.
runtime.Gosched()
time.Sleep(20 * time.Millisecond)
// Transition to stateDone while the cancel goroutine is blocked on the lock.
cc.markDone()
// Release the lock. The cancel func resumes and finds stateDone.
cc.cancelMu.Unlock()
r := <-ch
assert.False(t, r.ok, "cancel should not be accepted when execution already done")
assert.ErrorIs(t, r.handle.Wait(), ErrExecutionEnded)
}
// TestBuildCancelFunc_CASFailStateDone exercises the race-condition path
// in buildCancelFunc where the CAS on stateRunning→stateCancelling fails
// because markDone transitioned stateRunning→stateDone concurrently
// (cancel.go L742-743).
func TestBuildCancelFunc_CASFailStateDone(t *testing.T) {
// Exercises cancel.go L742-743: CAS(stateRunning→stateCancelling) fails
// because markDone transitions stateRunning→stateDone concurrently.
//
// The window between the state check (L738) and CAS (L739) is extremely
// tight. We maximize the chance by having the cancel goroutine block on
// cancelMu, then racing markDone with the lock release.
hit := false
for i := 0; i < 100000 && !hit; i++ {
cc := newCancelContext()
cancelFn := cc.buildCancelFunc()
// Hold cancelMu so the cancel goroutine blocks at L725.
cc.cancelMu.Lock()
cancelDone := make(chan struct{})
var h *CancelHandle
var ok bool
go func() {
defer close(cancelDone)
h, ok = cancelFn(WithAgentCancelMode(CancelAfterToolCalls))
}()
// Let the cancel goroutine reach the Lock() call.
runtime.Gosched()
// Release lock and fire markDone concurrently. The cancel goroutine
// will acquire the lock and race with markDone on the CAS.
go cc.markDone()
cc.cancelMu.Unlock()
<-cancelDone
if !ok && errors.Is(h.Wait(), ErrExecutionEnded) {
hit = true
}
}
if hit {
t.Log("Successfully hit CAS-fail → stateDone path")
} else {
t.Log("CAS race path not triggered (L743 remains a theoretical race edge)")
}
}