Files
cloudwego--eino/adk/chatmodel_retry_test.go
2026-07-13 12:49:10 +08:00

3209 lines
97 KiB
Go

/*
* Copyright 2025 CloudWeGo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package adk
import (
"context"
"errors"
"io"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"github.com/cloudwego/eino/components/model"
"github.com/cloudwego/eino/components/tool"
"github.com/cloudwego/eino/compose"
mockModel "github.com/cloudwego/eino/internal/mock/components/model"
"github.com/cloudwego/eino/schema"
)
var errRetryAble = errors.New("retry-able error")
var errNonRetryAble = errors.New("non-retry-able error")
var instantBackoff = func(_ context.Context, _ int) time.Duration { return time.Millisecond }
type agentEvent struct {
Err error
Output *AgentOutput
StreamContent string
}
func drainAgentEvents(t *testing.T, iterator *AsyncIterator[*AgentEvent]) []agentEvent {
t.Helper()
var events []agentEvent
for {
event, ok := iterator.Next()
if !ok {
break
}
events = append(events, agentEvent{Err: event.Err, Output: event.Output})
}
return events
}
func drainStreamingAgentEvents(t *testing.T, iterator *AsyncIterator[*AgentEvent]) (events []agentEvent, streamTermErrs []error) {
t.Helper()
for {
event, ok := iterator.Next()
if !ok {
break
}
ae := agentEvent{Err: event.Err, Output: event.Output}
if event.Output != nil && event.Output.MessageOutput != nil {
mo := event.Output.MessageOutput
if mo.IsStreaming && mo.MessageStream != nil {
var chunks []string
for {
msg, recvErr := mo.MessageStream.Recv()
if recvErr != nil {
streamTermErrs = append(streamTermErrs, recvErr)
break
}
if msg != nil {
chunks = append(chunks, msg.Content)
}
}
ae.StreamContent = strings.Join(chunks, "")
}
}
events = append(events, ae)
}
return events, streamTermErrs
}
func TestChatModelAgentRetry_NoTools_DirectError_Generate(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
if count < 3 {
return nil, errRetryAble
}
return schema.AssistantMessage("Success after retry", nil), nil
}).Times(3)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.Nil(t, event.Err)
assert.NotNil(t, event.Output)
assert.Equal(t, "Success after retry", event.Output.MessageOutput.Message.Content)
_, ok = iterator.Next()
assert.False(t, ok)
assert.Equal(t, int32(3), atomic.LoadInt32(&callCount))
}
func TestChatModelAgentRetry_NoTools_DirectError_Stream(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
count := atomic.AddInt32(&callCount, 1)
if count < 2 {
return nil, errRetryAble
}
return schema.StreamReaderFromArray([]*schema.Message{
schema.AssistantMessage("Success", nil),
}), nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.Nil(t, event.Err)
assert.NotNil(t, event.Output)
assert.True(t, event.Output.MessageOutput.IsStreaming)
_, ok = iterator.Next()
assert.False(t, ok)
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
}
type streamErrorModel struct {
callCount int32
failAtChunk int
maxFailures int
tools []*schema.ToolInfo
returnTool bool
}
func (m *streamErrorModel) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) {
return schema.AssistantMessage("Generated", nil), nil
}
func (m *streamErrorModel) Stream(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) {
count := atomic.AddInt32(&m.callCount, 1)
sr, sw := schema.Pipe[*schema.Message](10)
go func() {
defer sw.Close()
for i := 0; i < 5; i++ {
if i == m.failAtChunk && int(count) <= m.maxFailures {
sw.Send(nil, errRetryAble)
return
}
if m.returnTool && i == 0 {
sw.Send(schema.AssistantMessage("", []schema.ToolCall{{
ID: "call-1",
Function: schema.FunctionCall{Name: "test_tool", Arguments: "{}"},
}}), nil)
} else {
sw.Send(schema.AssistantMessage("chunk", nil), nil)
}
}
}()
return sr, nil
}
func (m *streamErrorModel) WithTools(tools []*schema.ToolInfo) (model.ToolCallingChatModel, error) {
m.tools = tools
return m, nil
}
func TestChatModelAgentRetry_StreamError(t *testing.T) {
t.Run("WithTools", func(t *testing.T) {
ctx := context.Background()
m := &streamErrorModel{
failAtChunk: 2,
maxFailures: 2,
returnTool: false,
}
config := &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: m,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{&fakeToolForTest{tarCount: 0}},
},
},
}
agent, err := NewChatModelAgent(ctx, config)
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
var events []*AgentEvent
for {
event, ok := iterator.Next()
if !ok {
break
}
events = append(events, event)
}
assert.Equal(t, 3, len(events))
var streamErrEventCount int
var errs []error
for i, event := range events {
if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming {
sr := event.Output.MessageOutput.MessageStream
for {
msg, err := sr.Recv()
if err == io.EOF {
break
}
if err != nil {
streamErrEventCount++
errs = append(errs, err)
t.Logf("event %d: err: %v", i, err)
break
}
t.Logf("event %d: %v", i, msg.Content)
}
}
}
assert.Equal(t, 2, streamErrEventCount)
assert.Equal(t, 2, len(errs))
var willRetryErr *WillRetryError
assert.True(t, errors.As(errs[0], &willRetryErr))
assert.True(t, errors.As(errs[1], &willRetryErr))
assert.Equal(t, int32(3), atomic.LoadInt32(&m.callCount))
})
t.Run("NoTools", func(t *testing.T) {
ctx := context.Background()
m := &streamErrorModel{
failAtChunk: 2,
maxFailures: 2,
returnTool: false,
}
config := &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: m,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
}
agent, err := NewChatModelAgent(ctx, config)
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
var events []*AgentEvent
for {
event, ok := iterator.Next()
if !ok {
break
}
events = append(events, event)
}
assert.Equal(t, 3, len(events))
var streamErrEventCount int
var errs []error
for i, event := range events {
if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming {
sr := event.Output.MessageOutput.MessageStream
for {
msg, err := sr.Recv()
if err == io.EOF {
break
}
if err != nil {
streamErrEventCount++
errs = append(errs, err)
t.Logf("event %d: err: %v", i, err)
break
}
t.Logf("event %d: %v", i, msg.Content)
}
}
}
assert.Equal(t, 2, streamErrEventCount)
assert.Equal(t, 2, len(errs))
var willRetryErr *WillRetryError
assert.True(t, errors.As(errs[0], &willRetryErr))
assert.True(t, errors.As(errs[1], &willRetryErr))
assert.Equal(t, int32(3), atomic.LoadInt32(&m.callCount))
})
}
func TestChatModelAgentRetry_WithTools_DirectError_Generate(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
if count < 2 {
return nil, errRetryAble
}
return schema.AssistantMessage("Success after retry", nil), nil
}).Times(2)
cm.EXPECT().WithTools(gomock.Any()).Return(cm, nil).AnyTimes()
fakeTool := &fakeToolForTest{tarCount: 0}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: cm,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{fakeTool},
},
},
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.Nil(t, event.Err)
assert.NotNil(t, event.Output)
assert.Equal(t, "Success after retry", event.Output.MessageOutput.Message.Content)
_, ok = iterator.Next()
assert.False(t, ok)
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
}
func TestChatModelAgentRetry_NonRetryableError(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(nil, errNonRetryAble).Times(1)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.NotNil(t, event.Err)
assert.True(t, errors.Is(event.Err, errNonRetryAble))
_, ok = iterator.Next()
assert.False(t, ok)
}
type inputCapturingModel struct {
capturedInputs [][]Message
}
func (m *inputCapturingModel) Generate(_ context.Context, input []*schema.Message, _ ...model.Option) (*schema.Message, error) {
m.capturedInputs = append(m.capturedInputs, input)
return schema.AssistantMessage("Response from capturing model", nil), nil
}
func (m *inputCapturingModel) Stream(_ context.Context, input []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) {
m.capturedInputs = append(m.capturedInputs, input)
return schema.StreamReaderFromArray([]*schema.Message{
schema.AssistantMessage("Response from capturing model", nil),
}), nil
}
func (m *inputCapturingModel) WithTools(_ []*schema.ToolInfo) (model.ToolCallingChatModel, error) {
return m, nil
}
func TestChatModelAgentRetry_MaxRetriesExhausted(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(nil, errRetryAble).Times(4)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.NotNil(t, event.Err)
assert.True(t, errors.Is(event.Err, ErrExceedMaxRetries))
var retryErr *RetryExhaustedError
assert.True(t, errors.As(event.Err, &retryErr))
assert.True(t, errors.Is(retryErr.LastErr, errRetryAble))
_, ok = iterator.Next()
assert.False(t, ok)
}
func TestChatModelAgentRetry_BackoffFunction(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var backoffCalls []int
var callCount int32
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
if count < 3 {
return nil, errRetryAble
}
return schema.AssistantMessage("Success", nil), nil
}).Times(3)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
BackoffFunc: func(ctx context.Context, attempt int) time.Duration {
backoffCalls = append(backoffCalls, attempt)
return time.Millisecond
},
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.Nil(t, event.Err)
_, ok = iterator.Next()
assert.False(t, ok)
assert.Equal(t, []int{1, 2}, backoffCalls)
}
func TestChatModelAgentRetry_NoRetryConfig(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(nil, errRetryAble).Times(1)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Test agent without retry config",
Instruction: "You are a helpful assistant.",
Model: cm,
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.NotNil(t, event.Err)
assert.True(t, errors.Is(event.Err, errRetryAble))
_, ok = iterator.Next()
assert.False(t, ok)
}
func TestChatModelAgentRetry_WithTools_NonRetryAbleStreamError(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
Return(nil, errNonRetryAble).Times(1)
cm.EXPECT().WithTools(gomock.Any()).Return(cm, nil).AnyTimes()
fakeTool := &fakeToolForTest{tarCount: 0}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: cm,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{fakeTool},
},
},
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.NotNil(t, event.Err)
assert.True(t, errors.Is(event.Err, errNonRetryAble))
_, ok = iterator.Next()
assert.False(t, ok)
}
type nonRetryAbleStreamErrorModel struct {
tools []*schema.ToolInfo
}
func (m *nonRetryAbleStreamErrorModel) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) {
return schema.AssistantMessage("Generated", nil), nil
}
func (m *nonRetryAbleStreamErrorModel) Stream(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) {
sr, sw := schema.Pipe[*schema.Message](10)
go func() {
defer sw.Close()
sw.Send(schema.AssistantMessage("chunk1", nil), nil)
sw.Send(nil, errNonRetryAble)
}()
return sr, nil
}
func (m *nonRetryAbleStreamErrorModel) WithTools(tools []*schema.ToolInfo) (model.ToolCallingChatModel, error) {
m.tools = tools
return m, nil
}
func TestChatModelAgentRetry_NoTools_NonRetryAbleStreamError(t *testing.T) {
ctx := context.Background()
m := &nonRetryAbleStreamErrorModel{}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent for retry functionality",
Instruction: "You are a helpful assistant.",
Model: m,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
var events []*AgentEvent
for {
event, ok := iterator.Next()
if !ok {
break
}
events = append(events, event)
}
assert.Equal(t, 2, len(events))
event0 := events[0]
assert.NotNil(t, event0.Output)
assert.NotNil(t, event0.Output.MessageOutput)
assert.True(t, event0.Output.MessageOutput.IsStreaming)
sr := event0.Output.MessageOutput.MessageStream
var streamErr error
for {
_, err := sr.Recv()
if err == io.EOF {
break
}
if err != nil {
streamErr = err
break
}
}
assert.NotNil(t, streamErr)
assert.True(t, errors.Is(streamErr, errNonRetryAble), "Stream error should be the original error")
event1 := events[1]
assert.NotNil(t, event1.Err)
assert.True(t, errors.Is(event1.Err, errNonRetryAble))
}
func TestDefaultBackoff(t *testing.T) {
ctx := context.Background()
d1 := defaultBackoff(ctx, 1)
d2 := defaultBackoff(ctx, 2)
d3 := defaultBackoff(ctx, 3)
t.Logf("Backoff delays: d1=%v, d2=%v, d3=%v", d1, d2, d3)
assert.True(t, d1 >= 100*time.Millisecond && d1 < 150*time.Millisecond,
"First retry should be ~100ms + jitter (0-50ms), got %v", d1)
assert.True(t, d2 >= 200*time.Millisecond && d2 < 300*time.Millisecond,
"Second retry should be ~200ms + jitter (0-100ms), got %v", d2)
assert.True(t, d3 >= 400*time.Millisecond && d3 < 600*time.Millisecond,
"Third retry should be ~400ms + jitter (0-200ms), got %v", d3)
d10 := defaultBackoff(ctx, 10)
t.Logf("Backoff delay for attempt 10: %v", d10)
assert.True(t, d10 >= 10*time.Second && d10 <= 15*time.Second,
"Delay should be capped at 10s + jitter (0-5s), got %v", d10)
d100 := defaultBackoff(ctx, 100)
t.Logf("Backoff delay for attempt 100: %v", d100)
assert.True(t, d100 >= 10*time.Second && d100 <= 15*time.Second,
"Delay should still be capped at 10s + jitter for very high attempts, got %v", d100)
}
type customError struct {
code int
msg string
}
func (e *customError) Error() string {
return e.msg
}
func TestWillRetryError_Unwrap(t *testing.T) {
originalErr := &customError{code: 500, msg: "internal error"}
willRetry := &WillRetryError{ErrStr: originalErr.Error(), RetryAttempt: 1, err: originalErr}
assert.True(t, errors.Is(willRetry, originalErr))
var targetErr *customError
assert.True(t, errors.As(willRetry, &targetErr))
assert.Equal(t, 500, targetErr.code)
assert.Equal(t, "internal error", targetErr.msg)
}
func TestChatModelAgentRetry_DefaultIsRetryAble(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
if count < 2 {
return nil, errors.New("any error should be retried")
}
return schema.AssistantMessage("Success", nil), nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTestAgent",
Description: "Test agent with default IsRetryAble",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.Nil(t, event.Err)
assert.Equal(t, "Success", event.Output.MessageOutput.Message.Content)
_, ok = iterator.Next()
assert.False(t, ok)
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
}
func TestSequentialWorkflow_RetryAbleStreamError_SuccessfulRetry(t *testing.T) {
ctx := context.Background()
retryModel := &streamErrorModel{
failAtChunk: 2,
maxFailures: 2,
}
agentA, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "AgentA",
Description: "Agent A with retry that emits stream errors then succeeds",
Instruction: "You are agent A.",
Model: retryModel,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
})
assert.NoError(t, err)
capturingModel := &inputCapturingModel{}
agentB, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "AgentB",
Description: "Agent B that captures input",
Instruction: "You are agent B.",
Model: capturingModel,
})
assert.NoError(t, err)
sequentialAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "SequentialAgent",
Description: "Sequential agent A->B",
SubAgents: []Agent{agentA, agentB},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
ctx, _ = initRunCtx(ctx, sequentialAgent.Name(ctx), input)
iterator := sequentialAgent.Run(ctx, input)
var willRetryErrCount int
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming {
sr := event.Output.MessageOutput.MessageStream
for {
_, err := sr.Recv()
if err == io.EOF {
break
}
if err != nil {
var retryErr *WillRetryError
if errors.As(err, &retryErr) {
willRetryErrCount++
}
break
}
}
}
}
assert.Equal(t, 2, willRetryErrCount, "End-user should receive 2 WillRetryError events")
assert.Equal(t, 1, len(capturingModel.capturedInputs), "Agent B should be called exactly once")
successorInput := capturingModel.capturedInputs[0]
var hasSuccessfulMessage bool
for _, msg := range successorInput {
if strings.Contains(msg.Content, "chunkchunkchunkchunkchunk") {
hasSuccessfulMessage = true
break
}
}
assert.True(t, hasSuccessfulMessage, "Agent B should receive the successful message from Agent A")
for _, msg := range successorInput {
assert.NotContains(t, msg.Content, "retry-able error", "Agent B should not receive failed stream messages")
}
}
type streamErrorModelNoRetry struct {
callCount int32
}
func (m *streamErrorModelNoRetry) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) {
return schema.AssistantMessage("Generated", nil), nil
}
func (m *streamErrorModelNoRetry) Stream(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) {
atomic.AddInt32(&m.callCount, 1)
sr, sw := schema.Pipe[*schema.Message](10)
go func() {
defer sw.Close()
sw.Send(schema.AssistantMessage("chunk1", nil), nil)
sw.Send(schema.AssistantMessage("chunk2", nil), nil)
sw.Send(nil, errRetryAble)
}()
return sr, nil
}
func (m *streamErrorModelNoRetry) WithTools(_ []*schema.ToolInfo) (model.ToolCallingChatModel, error) {
return m, nil
}
func TestSequentialWorkflow_NonRetryAbleStreamError_StopsFlow(t *testing.T) {
ctx := context.Background()
nonRetryModel := &nonRetryAbleStreamErrorModel{}
agentA, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "AgentA",
Description: "Agent A that emits non-retryable stream error",
Instruction: "You are agent A.",
Model: nonRetryModel,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(ctx context.Context, err error) bool { return errors.Is(err, errRetryAble) },
},
})
assert.NoError(t, err)
capturingModel := &inputCapturingModel{}
agentB, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "AgentB",
Description: "Agent B that captures input",
Instruction: "You are agent B.",
Model: capturingModel,
})
assert.NoError(t, err)
sequentialAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "SequentialAgent",
Description: "Sequential agent A->B",
SubAgents: []Agent{agentA, agentB},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
ctx, _ = initRunCtx(ctx, sequentialAgent.Name(ctx), input)
iterator := sequentialAgent.Run(ctx, input)
var streamErrFound bool
var finalErrEvent *AgentEvent
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Err != nil {
finalErrEvent = event
}
if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming {
sr := event.Output.MessageOutput.MessageStream
for {
_, err := sr.Recv()
if err == io.EOF {
break
}
if err != nil {
streamErrFound = true
assert.True(t, errors.Is(err, errNonRetryAble), "Stream error should be the original error")
break
}
}
}
}
assert.True(t, streamErrFound, "End-user should receive stream error")
assert.NotNil(t, finalErrEvent, "Should receive a final error event")
assert.True(t, errors.Is(finalErrEvent.Err, errNonRetryAble), "Final error should be the non-retryable error")
assert.Equal(t, 0, len(capturingModel.capturedInputs), "Agent B should NOT be called due to error")
}
func TestSequentialWorkflow_NoRetryConfig_StreamError_StopsFlow(t *testing.T) {
ctx := context.Background()
noRetryModel := &streamErrorModelNoRetry{}
agentA, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "AgentA",
Description: "Agent A without retry config that emits stream error",
Instruction: "You are agent A.",
Model: noRetryModel,
})
assert.NoError(t, err)
capturingModel := &inputCapturingModel{}
agentB, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "AgentB",
Description: "Agent B that captures input",
Instruction: "You are agent B.",
Model: capturingModel,
})
assert.NoError(t, err)
sequentialAgent, err := NewSequentialAgent(ctx, &SequentialAgentConfig{
Name: "SequentialAgent",
Description: "Sequential agent A->B",
SubAgents: []Agent{agentA, agentB},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
ctx, _ = initRunCtx(ctx, sequentialAgent.Name(ctx), input)
iterator := sequentialAgent.Run(ctx, input)
var streamErrFound bool
var finalErrEvent *AgentEvent
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Err != nil {
finalErrEvent = event
}
if event.Output != nil && event.Output.MessageOutput != nil && event.Output.MessageOutput.IsStreaming {
sr := event.Output.MessageOutput.MessageStream
for {
_, err := sr.Recv()
if err == io.EOF {
break
}
if err != nil {
streamErrFound = true
assert.True(t, errors.Is(err, errRetryAble), "Stream error should be the original error")
break
}
}
}
}
assert.True(t, streamErrFound, "End-user should receive stream error")
assert.NotNil(t, finalErrEvent, "Should receive a final error event")
assert.True(t, errors.Is(finalErrEvent.Err, errRetryAble), "Final error should be the original error")
assert.Equal(t, 0, len(capturingModel.capturedInputs), "Agent B should NOT be called due to error")
assert.Equal(t, int32(1), atomic.LoadInt32(&noRetryModel.callCount), "Model should only be called once (no retry)")
}
// failThenToolCallStreamModel is a ChatModel that:
// - First Stream() call: yields a partial chunk then fails with a retryable error mid-stream.
// - Second Stream() call (retry): yields a tool-call message (success).
// - Third Generate() call (after tool result): yields a final assistant message.
//
// This exercises the path where the eventSenderModel copies the first stream,
// wraps its error as WillRetryError, and sends it as an event to the session.
// The retryModelWrapper then retries, gets a clean stream with a tool call,
// the tool interrupts, and checkpoint save needs to gob-encode the session
// (which still contains the unconsumed WillRetryError event stream).
type failThenToolCallStreamModel struct {
streamCallCount int32
genCallCount int32
}
func (m *failThenToolCallStreamModel) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) {
atomic.AddInt32(&m.genCallCount, 1)
return schema.AssistantMessage("final answer", nil), nil
}
func (m *failThenToolCallStreamModel) Stream(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) {
count := atomic.AddInt32(&m.streamCallCount, 1)
sr, sw := schema.Pipe[*schema.Message](10)
go func() {
defer sw.Close()
if count == 1 {
// First call: yield a partial chunk then fail.
sw.Send(schema.AssistantMessage("partial", nil), nil)
sw.Send(nil, errRetryAble)
return
}
// Second call (retry): yield a tool-call message.
sw.Send(schema.AssistantMessage("", []schema.ToolCall{{
ID: "call-1",
Function: schema.FunctionCall{
Name: "interrupt_tool",
Arguments: `{}`,
},
}}), nil)
}()
return sr, nil
}
func (m *failThenToolCallStreamModel) WithTools(_ []*schema.ToolInfo) (model.ToolCallingChatModel, error) {
return m, nil
}
// interruptToolForRetryTest is a tool that always interrupts.
type interruptToolForRetryTest struct{}
func (t *interruptToolForRetryTest) Info(_ context.Context) (*schema.ToolInfo, error) {
return &schema.ToolInfo{
Name: "interrupt_tool",
Desc: "tool that interrupts",
ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
"input": {Type: "string"},
}),
}, nil
}
func (t *interruptToolForRetryTest) InvokableRun(ctx context.Context, _ string, _ ...tool.Option) (string, error) {
return "", tool.Interrupt(ctx, "interrupted by tool")
}
// TestCheckpointSave_WillRetryError_StreamNotConsumed verifies that checkpoint
// saving succeeds when the session contains an event with an unconsumed stream
// that ends with WillRetryError.
//
// Scenario:
// 1. ChatModelAgent with retry (MaxRetries=1) and a tool that always interrupts
// 2. Model.Stream() #1 yields "partial" then errRetryAble mid-stream
// → eventSenderModel copies the stream, wraps the error as WillRetryError,
// sends the event to the session (stream NOT consumed by anyone yet)
// → retryModelWrapper detects error on its copy, retries
// 3. Model.Stream() #2 succeeds with a tool-call message
// 4. Tool executes → interrupts
// 5. Runner.handleIter sees the interrupt → saveCheckPoint → gob encodes runSession
// 6. The session has the WillRetryError event with an unconsumed stream
// → agentEventWrapper.GobEncode proactively consumes the stream via
// getMessageFromWrappedEvent, so MessageVariant.GobEncode sees an error-free
// array and succeeds
func TestCheckpointSave_WillRetryError_StreamNotConsumed(t *testing.T) {
ctx := context.Background()
mdl := &failThenToolCallStreamModel{}
itool := &interruptToolForRetryTest{}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "TestAgent",
Description: "Agent for checkpoint stream error test",
Instruction: "You are a test agent.",
Model: mdl,
ToolsConfig: ToolsConfig{
ToolsNodeConfig: compose.ToolsNodeConfig{
Tools: []tool.BaseTool{itool},
},
},
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 1,
IsRetryAble: func(_ context.Context, err error) bool {
return errors.Is(err, errRetryAble)
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
store := newMyStore()
runner := NewRunner(ctx, RunnerConfig{
Agent: agent,
EnableStreaming: true,
CheckPointStore: store,
})
iter := runner.Run(ctx,
[]Message{schema.UserMessage("hello")},
WithCheckPointID("ckpt-1"),
)
for {
event, ok := iter.Next()
if !ok {
break
}
if event.Err != nil {
t.Logf("event error: %v", event.Err)
}
}
// Verify the checkpoint was saved successfully.
_, exists, _ := store.Get(ctx, "ckpt-1")
assert.True(t, exists, "checkpoint should be saved successfully; "+
"if this fails, the WillRetryError stream in the session caused gob encoding to fail")
// Sanity: the model should have been called twice for Stream (fail + retry).
assert.Equal(t, int32(2), atomic.LoadInt32(&mdl.streamCallCount),
"model should be called twice: first fail, then retry success")
}
func TestChatModelAgentRetry_ShouldRetry_RejectMessage_Stream(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
count := atomic.AddInt32(&callCount, 1)
r, w := schema.Pipe[*schema.Message](1)
go func() {
if count < 2 {
_ = w.Send(schema.AssistantMessage("bad stream content", nil), nil)
} else {
_ = w.Send(schema.AssistantMessage("good stream content", nil), nil)
}
w.Close()
}()
return r, nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "ShouldRetryStreamTestAgent",
Description: "Test ShouldRetry message rejection in stream mode",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{Retry: true}
}
if retryCtx.OutputMessage != nil && strings.Contains(retryCtx.OutputMessage.Content, "bad") {
return &RetryDecision{Retry: true}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
events, _ := drainStreamingAgentEvents(t, iterator)
var foundGoodContent bool
for _, e := range events {
if e.StreamContent == "good stream content" {
foundGoodContent = true
}
}
require.True(t, foundGoodContent, "should have received good stream content")
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
}
func TestShouldRetry_Generate(t *testing.T) {
t.Run("RetryContext_Fields", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
if count < 2 {
return schema.AssistantMessage("bad", nil), nil
}
return schema.AssistantMessage("good", nil), nil
}).Times(2)
var capturedContexts []*RetryContext
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryContextFieldsAgent",
Description: "Test that RetryContext fields are correctly populated",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
capturedContexts = append(capturedContexts, retryCtx)
if retryCtx.OutputMessage != nil && retryCtx.OutputMessage.Content == "bad" {
return &RetryDecision{Retry: true}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
for {
event, ok := iterator.Next()
if !ok {
break
}
_ = event
}
assert.Len(t, capturedContexts, 2, "ShouldRetry should be called twice")
assert.Equal(t, 1, capturedContexts[0].RetryAttempt)
assert.Len(t, capturedContexts[0].InputMessages, 2)
assert.True(t, len(capturedContexts[0].Options) > 0, "should have default options")
assert.Equal(t, "bad", capturedContexts[0].OutputMessage.Content)
assert.Nil(t, capturedContexts[0].Err)
assert.Equal(t, 2, capturedContexts[1].RetryAttempt)
assert.Equal(t, "good", capturedContexts[1].OutputMessage.Content)
assert.Nil(t, capturedContexts[1].Err)
})
t.Run("RewriteError_OnMessage", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(schema.AssistantMessage("unrecoverable bad message", nil), nil).Times(1)
fatalErr := errors.New("fatal: unrecoverable model output")
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RewriteErrorTestAgent",
Description: "Test ShouldRetry RewriteError on message",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.OutputMessage != nil && strings.Contains(retryCtx.OutputMessage.Content, "unrecoverable") {
return &RetryDecision{
Retry: false,
RewriteError: fatalErr,
}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
events := drainAgentEvents(t, iterator)
require.NotEmpty(t, events)
foundErr := false
for _, e := range events {
if e.Err != nil && errors.Is(e.Err, fatalErr) {
foundErr = true
}
}
require.True(t, foundErr, "should have received the fatal rewrite error")
})
t.Run("RewriteError_OnError", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
origErr := errors.New("original transient error")
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(nil, origErr).Times(1)
wrappedErr := errors.New("wrapped: original transient error with more context")
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RewriteErrorOnErrorTestAgent",
Description: "Test ShouldRetry RewriteError replacing original error",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{
Retry: false,
RewriteError: wrappedErr,
}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
events := drainAgentEvents(t, iterator)
require.NotEmpty(t, events)
foundErr := false
for _, e := range events {
if e.Err != nil && errors.Is(e.Err, wrappedErr) {
foundErr = true
}
}
require.True(t, foundErr, "should have received the wrapped rewrite error")
})
t.Run("AdditionalOptions", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
var capturedOpts [][]model.Option
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
capturedOpts = append(capturedOpts, opts)
if count < 2 {
return nil, errRetryAble
}
return schema.AssistantMessage("success", nil), nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "AdditionalOptionsTestAgent",
Description: "Test ShouldRetry AdditionalOptions",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{
Retry: true,
AdditionalOptions: []model.Option{model.WithMaxTokens(8192)},
}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.Nil(t, event.Err)
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
assert.Equal(t, 2, len(capturedOpts))
assert.Equal(t, len(capturedOpts[0])+1, len(capturedOpts[1]))
})
t.Run("ModifiedInputMessages_NoPersist", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
var capturedInputs [][]*schema.Message
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
inputCopy := make([]*schema.Message, len(input))
copy(inputCopy, input)
capturedInputs = append(capturedInputs, inputCopy)
if count < 2 {
return nil, errRetryAble
}
return schema.AssistantMessage("success", nil), nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "ModifiedInputNoPersistAgent",
Description: "Test ShouldRetry ModifiedInputMessages without persistence",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{
Retry: true,
ModifiedInputMessages: []*schema.Message{
schema.SystemMessage("compressed instruction"),
schema.UserMessage("Hello"),
},
PersistModifiedInputMessages: false,
}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.Nil(t, event.Err)
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
assert.Equal(t, 2, len(capturedInputs))
assert.Equal(t, "compressed instruction", capturedInputs[1][0].Content, "second call should use modified input")
})
t.Run("Backoff", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
if count < 2 {
return nil, errRetryAble
}
return schema.AssistantMessage("success", nil), nil
}).Times(2)
customBackoff := 50 * time.Millisecond
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "BackoffTestAgent",
Description: "Test ShouldRetry custom Backoff in decision",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{
Retry: true,
Backoff: customBackoff,
}
}
return &RetryDecision{Retry: false}
},
},
})
assert.NoError(t, err)
start := time.Now()
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
event, ok := iterator.Next()
assert.True(t, ok)
assert.NotNil(t, event)
assert.Nil(t, event.Err)
elapsed := time.Since(start)
assert.True(t, elapsed >= customBackoff && elapsed < customBackoff+200*time.Millisecond, "expected backoff ~%v, got %v", customBackoff, elapsed)
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
})
t.Run("SuppressFlag_Rejected_NoEvent", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
if count == 1 {
return schema.AssistantMessage("bad", nil), nil
}
return schema.AssistantMessage("good", nil), nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "SuppressRejected",
Description: "Test suppress flag rejects first then accepts",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 1,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.OutputMessage != nil && retryCtx.OutputMessage.Content == "bad" {
return &RetryDecision{Retry: true}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
var msgEvents []*AgentEvent
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil {
msgEvents = append(msgEvents, event)
}
}
assert.Equal(t, 1, len(msgEvents), "should have exactly 1 message event (suppressed rejected)")
assert.Equal(t, "good", msgEvents[0].Output.MessageOutput.Message.Content)
})
t.Run("SuppressFlag_AllRejected_NoEvents", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(schema.AssistantMessage("always bad", nil), nil).Times(3)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "SuppressAllRejected",
Description: "Test suppress flag all rejected no events",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
return &RetryDecision{Retry: true}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
events := drainAgentEvents(t, iterator)
var msgEventCount int
var foundExhaustedErr bool
for _, e := range events {
if e.Output != nil && e.Output.MessageOutput != nil {
msgEventCount++
}
if e.Err != nil && errors.Is(e.Err, ErrExceedMaxRetries) {
foundExhaustedErr = true
}
}
assert.Equal(t, 0, msgEventCount, "no message events should be emitted when all are rejected")
require.True(t, foundExhaustedErr, "final event should have RetryExhaustedError")
})
t.Run("SuppressFlag_Accepted_FirstAttempt", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(schema.AssistantMessage("perfect", nil), nil).Times(1)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "SuppressAcceptedFirst",
Description: "Test suppress flag accepted first attempt",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
var msgEvents []*AgentEvent
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil {
msgEvents = append(msgEvents, event)
}
}
assert.Equal(t, 1, len(msgEvents), "should have exactly 1 event")
assert.Equal(t, "perfect", msgEvents[0].Output.MessageOutput.Message.Content)
})
t.Run("ContextCanceled_DuringSleep", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
atomic.AddInt32(&callCount, 1)
return nil, errors.New("transient")
}).Times(1)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "ContextCancelDuringSleep",
Description: "Test context cancellation during backoff sleep",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 5,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
return &RetryDecision{Retry: true}
},
BackoffFunc: func(_ context.Context, _ int) time.Duration { return 10 * time.Second },
},
})
require.NoError(t, err)
go func() {
time.Sleep(50 * time.Millisecond)
cancel()
}()
start := time.Now()
iterator := agent.Run(ctx, &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
})
events := drainAgentEvents(t, iterator)
elapsed := time.Since(start)
require.True(t, elapsed < 2*time.Second, "should not block for full backoff; elapsed: %v", elapsed)
assert.Equal(t, int32(1), atomic.LoadInt32(&callCount))
var foundCtxErr bool
for _, e := range events {
if e.Err != nil && errors.Is(e.Err, context.Canceled) {
foundCtxErr = true
}
}
require.True(t, foundCtxErr, "should have received context.Canceled error")
})
}
func TestShouldRetry_Stream(t *testing.T) {
t.Run("ErrorRetry", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
streamErr := errors.New("stream unavailable")
var callCount int32
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
count := atomic.AddInt32(&callCount, 1)
if count < 2 {
return nil, streamErr
}
r, w := schema.Pipe[*schema.Message](1)
go func() {
_ = w.Send(schema.AssistantMessage("recovered stream", nil), nil)
w.Close()
}()
return r, nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamErrorRetryAgent",
Description: "Test ShouldRetry when Stream returns error (nil stream)",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{Retry: true}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
events, _ := drainStreamingAgentEvents(t, iterator)
var foundContent bool
for _, e := range events {
if e.StreamContent == "recovered stream" {
foundContent = true
}
}
require.True(t, foundContent, "should have received recovered stream content after error retry")
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
})
t.Run("ErrorRewrite", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
streamErr := errors.New("model overloaded")
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
Return(nil, streamErr).Times(1)
fatalErr := errors.New("fatal: model overloaded, aborting")
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamErrorRewriteAgent",
Description: "Test ShouldRetry RewriteError when Stream returns error",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil && strings.Contains(retryCtx.Err.Error(), "overloaded") {
return &RetryDecision{
Retry: false,
RewriteError: fatalErr,
}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
events := drainAgentEvents(t, iterator)
require.NotEmpty(t, events)
foundErr := false
for _, e := range events {
if e.Err != nil && errors.Is(e.Err, fatalErr) {
foundErr = true
}
}
require.True(t, foundErr, "should have received the fatal rewrite error from stream")
})
t.Run("RewriteError_OnMessage", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
r, w := schema.Pipe[*schema.Message](1)
go func() {
_ = w.Send(schema.AssistantMessage("hallucinated garbage output", nil), nil)
w.Close()
}()
return r, nil
}).Times(1)
fatalErr := errors.New("fatal: hallucinated output detected")
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamRewriteOnMessageAgent",
Description: "Test ShouldRetry RewriteError on successful stream with bad content",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.OutputMessage != nil && strings.Contains(retryCtx.OutputMessage.Content, "hallucinated") {
return &RetryDecision{
Retry: false,
RewriteError: fatalErr,
}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
events := drainAgentEvents(t, iterator)
require.NotEmpty(t, events)
foundErr := false
for _, e := range events {
if e.Err != nil && errors.Is(e.Err, fatalErr) {
foundErr = true
}
}
require.True(t, foundErr, "should have received fatal rewrite error from stream message inspection")
})
t.Run("PartialStreamError", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
partialErr := errors.New("connection reset mid-stream")
var callCount int32
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
count := atomic.AddInt32(&callCount, 1)
r, w := schema.Pipe[*schema.Message](1)
go func() {
_ = w.Send(schema.AssistantMessage("partial chunk", nil), nil)
if count < 2 {
w.Send(nil, partialErr)
} else {
_ = w.Send(schema.AssistantMessage(" complete", nil), nil)
w.Close()
}
}()
return r, nil
}).Times(2)
var capturedContexts []*RetryContext
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamPartialErrorAgent",
Description: "Test ShouldRetry when stream has partial content then error",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
capturedContexts = append(capturedContexts, retryCtx)
if retryCtx.Err != nil {
return &RetryDecision{Retry: true}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil {
mo := event.Output.MessageOutput
if mo.IsStreaming && mo.MessageStream != nil {
for {
_, err := mo.MessageStream.Recv()
if err != nil {
break
}
}
}
}
}
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
assert.Equal(t, 2, len(capturedContexts))
assert.NotNil(t, capturedContexts[0].Err, "first attempt should have stream error")
assert.NotNil(t, capturedContexts[0].OutputMessage, "first attempt should have partial message despite error")
assert.Equal(t, "partial chunk", capturedContexts[0].OutputMessage.Content)
})
t.Run("ModifiedInputsAndOptions_WithPersist", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
var capturedInputs [][]*schema.Message
var capturedOptLens []int
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
count := atomic.AddInt32(&callCount, 1)
inputCopy := make([]*schema.Message, len(input))
copy(inputCopy, input)
capturedInputs = append(capturedInputs, inputCopy)
capturedOptLens = append(capturedOptLens, len(opts))
r, w := schema.Pipe[*schema.Message](1)
go func() {
if count < 2 {
_ = w.Send(schema.AssistantMessage("too long response exceeds limit", nil), nil)
} else {
_ = w.Send(schema.AssistantMessage("good response", nil), nil)
}
w.Close()
}()
return r, nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamModifiedInputsPersistAgent",
Description: "Test ShouldRetry with ModifiedInputMessages (persist) and AdditionalOptions in stream mode",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.OutputMessage != nil && strings.Contains(retryCtx.OutputMessage.Content, "too long") {
return &RetryDecision{
Retry: true,
ModifiedInputMessages: []*schema.Message{
schema.SystemMessage("compressed instruction"),
schema.UserMessage("summarized history"),
},
PersistModifiedInputMessages: true,
AdditionalOptions: []model.Option{model.WithMaxTokens(16384)},
}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
events, _ := drainStreamingAgentEvents(t, iterator)
var foundGood bool
for _, e := range events {
if e.StreamContent == "good response" {
foundGood = true
}
}
require.True(t, foundGood, "should have received good response after retry with modified inputs")
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
assert.Equal(t, 2, len(capturedInputs))
assert.Equal(t, "compressed instruction", capturedInputs[1][0].Content, "second call should use modified input")
assert.Equal(t, "summarized history", capturedInputs[1][1].Content)
assert.Equal(t, capturedOptLens[0]+1, capturedOptLens[1])
})
t.Run("VerdictSignal_CleanStream_Rejected", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
count := atomic.AddInt32(&callCount, 1)
if count == 1 {
return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("bad", nil)}), nil
}
return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("good", nil)}), nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "VerdictCleanRejected",
Description: "Test verdict signal on clean stream rejected",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 1,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.OutputMessage != nil && retryCtx.OutputMessage.Content == "bad" {
return &RetryDecision{Retry: true}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
var streamEvents []int
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil {
mo := event.Output.MessageOutput
if mo.IsStreaming && mo.MessageStream != nil {
idx := len(streamEvents)
streamEvents = append(streamEvents, idx)
var lastErr error
for {
_, recvErr := mo.MessageStream.Recv()
if recvErr != nil {
lastErr = recvErr
break
}
}
if idx == 0 {
var willRetryErr *WillRetryError
assert.True(t, errors.As(lastErr, &willRetryErr), "first stream should end with WillRetryError")
} else {
assert.ErrorIs(t, lastErr, io.EOF, "second stream should end with io.EOF")
}
}
}
}
assert.Equal(t, 2, len(streamEvents), "should have exactly 2 stream events")
assert.Equal(t, int32(2), atomic.LoadInt32(&callCount))
})
t.Run("VerdictSignal_StreamError_Rejected", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
streamErr := errors.New("mid-stream error")
var callCount int32
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
count := atomic.AddInt32(&callCount, 1)
if count == 1 {
r, w := schema.Pipe[*schema.Message](1)
go func() {
_ = w.Send(schema.AssistantMessage("partial", nil), nil)
w.Send(nil, streamErr)
}()
return r, nil
}
return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("good", nil)}), nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "VerdictStreamErrorRejected",
Description: "Test verdict signal on stream error rejected",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 1,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{Retry: true}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
var firstEventHasWillRetry bool
var eventCount int
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil {
mo := event.Output.MessageOutput
if mo.IsStreaming && mo.MessageStream != nil {
eventCount++
for {
_, recvErr := mo.MessageStream.Recv()
if recvErr != nil {
if eventCount == 1 {
var willRetryErr *WillRetryError
if errors.As(recvErr, &willRetryErr) {
firstEventHasWillRetry = true
}
}
break
}
}
}
}
}
assert.True(t, firstEventHasWillRetry, "first event stream should end with WillRetryError via errWrapper path")
assert.Equal(t, 2, eventCount, "should have 2 stream events")
})
t.Run("VerdictSignal_Accepted_FirstAttempt", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("perfect", nil)}), nil
}).Times(1)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "VerdictAcceptedFirst",
Description: "Test verdict signal accepted first attempt",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
var eventCount int
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil {
mo := event.Output.MessageOutput
if mo.IsStreaming && mo.MessageStream != nil {
eventCount++
var lastErr error
for {
_, recvErr := mo.MessageStream.Recv()
if recvErr != nil {
lastErr = recvErr
break
}
}
assert.ErrorIs(t, lastErr, io.EOF, "accepted stream should end with io.EOF")
}
}
}
assert.Equal(t, 1, eventCount, "should have exactly 1 event")
})
t.Run("VerdictSignal_AllRejected_Exhausted", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("always bad", nil)}), nil
}).Times(3)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "VerdictAllRejected",
Description: "Test verdict signal all rejected exhausted",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
return &RetryDecision{Retry: true}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
events, streamTermErrs := drainStreamingAgentEvents(t, iterator)
var willRetryCount int
var foundExhaustedErr bool
for _, e := range events {
if e.Err != nil && errors.Is(e.Err, ErrExceedMaxRetries) {
foundExhaustedErr = true
}
}
for _, termErr := range streamTermErrs {
var willRetryErr *WillRetryError
if errors.As(termErr, &willRetryErr) {
willRetryCount++
}
}
assert.Equal(t, 3, willRetryCount, "all 3 stream events should end with WillRetryError")
require.True(t, foundExhaustedErr, "final error should be RetryExhaustedError")
})
t.Run("ShouldRetry_Panics_VerdictStillSent", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("trigger panic", nil)}), nil
}).Times(1)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "ShouldRetryPanicsAgent",
Description: "Test that ShouldRetry panic sends verdict signal and does not deadlock",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 1,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
panic("deliberate panic in ShouldRetry")
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
done := make(chan struct{})
var events []agentEvent
go func() {
defer close(done)
iterator := agent.Run(ctx, input)
events = drainAgentEvents(t, iterator)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("test deadlocked — verdict signal was not sent after ShouldRetry panic")
}
require.NotEmpty(t, events)
var foundPanicErr bool
for _, e := range events {
if e.Err != nil && strings.Contains(e.Err.Error(), "panic") {
foundPanicErr = true
}
}
assert.True(t, foundPanicErr, "should have received a panic error event")
})
}
func TestErrStreamCanceled(t *testing.T) {
t.Run("Stream_ShouldRetry_NeverRetried", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
r, w := schema.Pipe[*schema.Message](1)
go func() {
_ = w.Send(schema.AssistantMessage("partial", nil), nil)
w.Send(nil, ErrStreamCanceled)
}()
return r, nil
}).Times(1)
var shouldRetryCalled int32
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamCanceledShouldRetry",
Description: "Test ErrStreamCanceled never retried with ShouldRetry",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
atomic.AddInt32(&shouldRetryCalled, 1)
return &RetryDecision{Retry: true}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil {
mo := event.Output.MessageOutput
if mo.IsStreaming && mo.MessageStream != nil {
for {
_, recvErr := mo.MessageStream.Recv()
if recvErr != nil {
break
}
}
}
}
}
assert.Equal(t, int32(0), atomic.LoadInt32(&shouldRetryCalled), "ShouldRetry should never be called for ErrStreamCanceled")
})
t.Run("Stream_LegacyIsRetryAble_NeverRetried", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
r, w := schema.Pipe[*schema.Message](1)
go func() {
_ = w.Send(schema.AssistantMessage("partial", nil), nil)
w.Send(nil, ErrStreamCanceled)
}()
return r, nil
}).Times(1)
var isRetryAbleCalled int32
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamCanceledLegacy",
Description: "Test ErrStreamCanceled never retried with legacy IsRetryAble",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
IsRetryAble: func(_ context.Context, err error) bool {
atomic.AddInt32(&isRetryAbleCalled, 1)
return true
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil {
mo := event.Output.MessageOutput
if mo.IsStreaming && mo.MessageStream != nil {
for {
_, recvErr := mo.MessageStream.Recv()
if recvErr != nil {
break
}
}
}
}
}
assert.Equal(t, int32(0), atomic.LoadInt32(&isRetryAbleCalled), "IsRetryAble should never be called for ErrStreamCanceled")
})
t.Run("Generate_ShouldRetry_NeverRetried", func(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(nil, ErrStreamCanceled).Times(1)
var shouldRetryCalled int32
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "GenCanceledShouldRetry",
Description: "Test ErrStreamCanceled in Generate never retried",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
atomic.AddInt32(&shouldRetryCalled, 1)
return &RetryDecision{Retry: true}
},
BackoffFunc: instantBackoff,
},
})
assert.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
}
iterator := agent.Run(ctx, input)
for {
_, ok := iterator.Next()
if !ok {
break
}
}
assert.Equal(t, int32(0), atomic.LoadInt32(&shouldRetryCalled), "ShouldRetry should never be called for ErrStreamCanceled")
})
}
func TestAttack_ShouldRetry_NilDecisionOnEveryCall(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(schema.AssistantMessage("ok", nil), nil).Times(1)
var shouldRetryCalls int32
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "NilDecisionAgent",
Description: "ShouldRetry always returns nil — should accept on first call",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
atomic.AddInt32(&shouldRetryCalls, 1)
return nil
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
iterator := agent.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("Hello")}})
events := drainAgentEvents(t, iterator)
require.NotEmpty(t, events)
assert.Equal(t, int32(1), atomic.LoadInt32(&shouldRetryCalls))
var foundOK bool
for _, e := range events {
if e.Output != nil && e.Output.MessageOutput != nil && e.Output.MessageOutput.Message != nil {
if e.Output.MessageOutput.Message.Content == "ok" {
foundOK = true
}
}
}
assert.True(t, foundOK, "nil decision should accept the message as-is")
}
func TestAttack_ShouldRetry_MaxRetriesZero_RejectFirstAttempt(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
Return(schema.AssistantMessage("bad", nil), nil).Times(1)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "MaxZeroRejectAgent",
Description: "MaxRetries=0 with ShouldRetry rejecting — should exhaust immediately",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 0,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
return &RetryDecision{Retry: true}
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
iterator := agent.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("Hello")}})
events := drainAgentEvents(t, iterator)
var foundExhausted bool
for _, e := range events {
if e.Err != nil {
var exhaustedErr *RetryExhaustedError
if errors.As(e.Err, &exhaustedErr) {
foundExhausted = true
}
}
}
assert.True(t, foundExhausted, "MaxRetries=0 with Retry:true should produce RetryExhaustedError")
}
func TestAttack_ShouldRetry_RetryTrueWithRewriteError_IgnoresRewrite(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
var callCount int32
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
if count == 1 {
return nil, errors.New("transient")
}
return schema.AssistantMessage("success", nil), nil
}).Times(2)
rewriteErr := errors.New("this should be ignored")
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "RetryTrueRewriteAgent",
Description: "Retry=true with RewriteError should ignore the rewrite",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 3,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{Retry: true, RewriteError: rewriteErr}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
iterator := agent.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("Hello")}})
events := drainAgentEvents(t, iterator)
var foundSuccess bool
for _, e := range events {
if e.Err != nil && errors.Is(e.Err, rewriteErr) {
t.Fatal("RewriteError should be ignored when Retry=true")
}
if e.Output != nil && e.Output.MessageOutput != nil && e.Output.MessageOutput.Message != nil {
if e.Output.MessageOutput.Message.Content == "success" {
foundSuccess = true
}
}
}
assert.True(t, foundSuccess, "should eventually succeed after retry, ignoring RewriteError")
}
func TestAttack_ShouldRetry_OptionsAccumulateAcrossRetries(t *testing.T) {
ctx := context.Background()
var capturedOpts [][]model.Option
var callCount int32
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Generate(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
count := atomic.AddInt32(&callCount, 1)
capturedOpts = append(capturedOpts, opts)
if count <= 2 {
return nil, errors.New("needs retry")
}
return schema.AssistantMessage("done", nil), nil
}).Times(3)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "OptsAccumulateAgent",
Description: "Verify options accumulate across retries",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 5,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{
Retry: true,
AdditionalOptions: []model.Option{model.WithMaxTokens(100 * retryCtx.RetryAttempt)},
}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
iterator := agent.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("Hello")}})
drainAgentEvents(t, iterator)
require.Len(t, capturedOpts, 3)
assert.True(t, len(capturedOpts[1]) > len(capturedOpts[0]),
"second call should have more options than first (accumulated AdditionalOptions)")
assert.True(t, len(capturedOpts[2]) > len(capturedOpts[1]),
"third call should have more options than second (accumulated AdditionalOptions)")
}
func TestAttack_ShouldRetry_Stream_NilDecisionAccepts(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("stream ok", nil)}), nil
}).Times(1)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamNilDecisionAgent",
Description: "ShouldRetry returns nil in stream mode — should accept",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
return nil
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
iterator := agent.Run(ctx, input)
events, streamTermErrs := drainStreamingAgentEvents(t, iterator)
var foundStreamContent bool
for _, e := range events {
if e.StreamContent == "stream ok" {
foundStreamContent = true
}
}
assert.True(t, foundStreamContent, "nil decision should accept the stream")
for _, termErr := range streamTermErrs {
assert.Equal(t, io.EOF, termErr, "stream should terminate with clean EOF, not error")
}
}
func TestAttack_ShouldRetry_Stream_MaxRetriesZero_Exhausted(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("rejected", nil)}), nil
}).Times(1)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamMaxZeroAgent",
Description: "Stream mode with MaxRetries=0 rejecting — should exhaust immediately",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 0,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
return &RetryDecision{Retry: true}
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
done := make(chan struct{})
var events []agentEvent
go func() {
defer close(done)
iterator := agent.Run(ctx, input)
events, _ = drainStreamingAgentEvents(t, iterator)
}()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("test deadlocked — Stream MaxRetries=0 with reject should not hang")
}
var foundExhausted bool
for _, e := range events {
if e.Err != nil {
var exhaustedErr *RetryExhaustedError
if errors.As(e.Err, &exhaustedErr) {
foundExhausted = true
}
}
}
assert.True(t, foundExhausted, "MaxRetries=0 stream reject should produce RetryExhaustedError")
}
func TestAttack_ShouldRetry_Stream_RewriteErrorOnCleanStream(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
return schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("looks good but bad", nil)}), nil
}).Times(1)
fatalErr := errors.New("fatal: content policy violation")
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "StreamRewriteCleanAgent",
Description: "Stream returns cleanly but ShouldRetry rewrites to error",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 2,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
return &RetryDecision{Retry: false, RewriteError: fatalErr}
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
done := make(chan struct{})
var events []agentEvent
go func() {
defer close(done)
iterator := agent.Run(ctx, input)
events, _ = drainStreamingAgentEvents(t, iterator)
}()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("test deadlocked")
}
var foundFatal bool
for _, e := range events {
if e.Err != nil && errors.Is(e.Err, fatalErr) {
foundFatal = true
}
}
assert.True(t, foundFatal, "RewriteError on clean stream should propagate the fatal error")
}
func TestAttack_ShouldRetry_ConcatMessagesFails_EmptyStream(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
r, w := schema.Pipe[*schema.Message](1)
w.Close()
return r, nil
}).Times(1)
var capturedCtx *RetryContext
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "EmptyStreamAgent",
Description: "Stream returns zero chunks — both OutputMessage and Err should be nil",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 1,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
capturedCtx = retryCtx
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
done := make(chan struct{})
go func() {
defer close(done)
iterator := agent.Run(ctx, input)
drainStreamingAgentEvents(t, iterator)
}()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("test deadlocked on empty stream")
}
require.NotNil(t, capturedCtx)
assert.NotNil(t, capturedCtx.OutputMessage, "empty stream should have non-nil OutputMessage from ConcatMessages")
assert.Nil(t, capturedCtx.Err, "empty stream should have nil Err")
}
func TestAttack_ShouldRetry_Stream_MidStreamError_VerdictDoubleRead(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cm := mockModel.NewMockToolCallingChatModel(ctrl)
midStreamErr := errors.New("mid-stream transient error")
cm.EXPECT().Stream(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
r, w := schema.Pipe[*schema.Message](1)
go func() {
defer w.Close()
_ = w.Send(schema.AssistantMessage("chunk1", nil), nil)
_ = w.Send(nil, midStreamErr)
}()
return r, nil
}).Times(2)
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "DoubleReadBugAgent",
Description: "Reproduces signal.ch double-read when event stream hits mid-stream error then EOF",
Instruction: "You are a helpful assistant.",
Model: cm,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 1,
ShouldRetry: func(ctx context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{Retry: true}
}
return &RetryDecision{Retry: false}
},
BackoffFunc: instantBackoff,
},
})
require.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("Hello")},
EnableStreaming: true,
}
done := make(chan struct{})
go func() {
defer close(done)
iterator := agent.Run(ctx, input)
for {
event, ok := iterator.Next()
if !ok {
break
}
if event.Output != nil && event.Output.MessageOutput != nil {
mo := event.Output.MessageOutput
if mo.IsStreaming && mo.MessageStream != nil {
for {
_, recvErr := mo.MessageStream.Recv()
if recvErr == io.EOF {
break
}
}
}
}
}
}()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("goroutine leak: onEOF blocked on signal.ch after errWrapper already drained the verdict")
}
}
type rejectReasonTestModel struct {
streamFn func(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error)
}
func (m *rejectReasonTestModel) Generate(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.Message, error) {
return schema.AssistantMessage("generated", nil), nil
}
func (m *rejectReasonTestModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
return m.streamFn(ctx, input, opts...)
}
func TestRejectReason_StreamPath(t *testing.T) {
ctx := context.Background()
type rejectInfo struct {
Reason string
Attempt int
}
streamErr := errors.New("bad output")
var streamCallCount int32
m := &rejectReasonTestModel{
streamFn: func(_ context.Context, _ []*schema.Message, _ ...model.Option) (*schema.StreamReader[*schema.Message], error) {
n := atomic.AddInt32(&streamCallCount, 1)
if n == 1 {
return streamWithMidError(
[]*schema.Message{schema.AssistantMessage("rejected chunk", nil)},
streamErr,
), nil
}
sr, sw := schema.Pipe[*schema.Message](1)
go func() {
defer sw.Close()
sw.Send(schema.AssistantMessage("accepted", nil), nil)
}()
return sr, nil
},
}
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
Name: "reject-reason-agent",
Description: "test reject reason",
Instruction: "test",
Model: m,
ModelRetryConfig: &ModelRetryConfig{
MaxRetries: 1,
ShouldRetry: func(_ context.Context, retryCtx *RetryContext) *RetryDecision {
if retryCtx.Err != nil {
return &RetryDecision{
Retry: true,
RejectReason: rejectInfo{
Reason: "output quality too low",
Attempt: retryCtx.RetryAttempt,
},
}
}
return nil
},
BackoffFunc: func(_ context.Context, _ int) time.Duration { return time.Millisecond },
},
})
require.NoError(t, err)
input := &AgentInput{
Messages: []Message{schema.UserMessage("hello")},
EnableStreaming: true,
}
ctx, _ = initRunCtx(ctx, agent.Name(ctx), input)
iter := agent.Run(ctx, input)
var capturedRejectReasons []any
var finalContent string
for {
ev, ok := iter.Next()
if !ok {
break
}
if ev.Err != nil {
continue
}
if ev.Output != nil && ev.Output.MessageOutput != nil && ev.Output.MessageOutput.IsStreaming {
sr := ev.Output.MessageOutput.MessageStream
for {
chunk, recvErr := sr.Recv()
if recvErr != nil {
var willRetry *WillRetryError
if errors.As(recvErr, &willRetry) {
capturedRejectReasons = append(capturedRejectReasons, willRetry.RejectReason())
}
break
}
if chunk != nil {
finalContent = chunk.Content
}
}
}
}
assert.Contains(t, finalContent, "accepted")
require.NotEmpty(t, capturedRejectReasons, "should have at least one WillRetryError with RejectReason from stream Recv()")
for _, reason := range capturedRejectReasons {
require.NotNil(t, reason)
ri, ok := reason.(rejectInfo)
require.True(t, ok, "RejectReason should be rejectInfo type, got %T", reason)
assert.Equal(t, "output quality too low", ri.Reason)
assert.Equal(t, 1, ri.Attempt)
}
}
func TestWillRetryError_RejectReason(t *testing.T) {
t.Run("nil when not set", func(t *testing.T) {
wrErr := &WillRetryError{ErrStr: "test", RetryAttempt: 1, err: errors.New("test")}
assert.Nil(t, wrErr.RejectReason(), "RejectReason should be nil when not set")
})
t.Run("returns value when set", func(t *testing.T) {
reason := map[string]string{"key": "value"}
wrErr := &WillRetryError{
ErrStr: "rejected",
RetryAttempt: 2,
rejectReason: reason,
err: errors.New("inner"),
}
assert.Equal(t, reason, wrErr.RejectReason())
assert.Equal(t, "rejected", wrErr.Error())
assert.Equal(t, 2, wrErr.RetryAttempt)
})
}