6025 lines
178 KiB
Go
6025 lines
178 KiB
Go
/*
|
|
* Copyright 2025 CloudWeGo Authors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package adk
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/cloudwego/eino/schema"
|
|
)
|
|
|
|
type turnLoopMockAgent struct {
|
|
name string
|
|
events []*AgentEvent
|
|
runFunc func(ctx context.Context, input *AgentInput) (*AgentOutput, error)
|
|
}
|
|
|
|
func (a *turnLoopMockAgent) Name(_ context.Context) string { return a.name }
|
|
func (a *turnLoopMockAgent) Description(_ context.Context) string { return "mock agent" }
|
|
func (a *turnLoopMockAgent) Run(ctx context.Context, input *AgentInput, _ ...AgentRunOption) *AsyncIterator[*AgentEvent] {
|
|
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
|
|
|
|
if a.runFunc != nil {
|
|
go func() {
|
|
defer gen.Close()
|
|
output, err := a.runFunc(ctx, input)
|
|
if err != nil {
|
|
gen.Send(&AgentEvent{Err: err})
|
|
return
|
|
}
|
|
gen.Send(&AgentEvent{Output: output})
|
|
}()
|
|
return iter
|
|
}
|
|
|
|
go func() {
|
|
defer gen.Close()
|
|
for _, e := range a.events {
|
|
gen.Send(e)
|
|
}
|
|
}()
|
|
return iter
|
|
}
|
|
|
|
type turnLoopCheckpointStore struct {
|
|
m map[string][]byte
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func (s *turnLoopCheckpointStore) Set(_ context.Context, key string, value []byte) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.m[key] = value
|
|
return nil
|
|
}
|
|
|
|
func (s *turnLoopCheckpointStore) Get(_ context.Context, key string) ([]byte, bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
v, ok := s.m[key]
|
|
return v, ok, nil
|
|
}
|
|
|
|
type turnLoopCancellableMockAgent struct {
|
|
name string
|
|
runFunc func(ctx context.Context, input *AgentInput) (*AgentOutput, error)
|
|
onCancel func(cc *cancelContext)
|
|
cancel context.CancelFunc
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func (a *turnLoopCancellableMockAgent) Name(_ context.Context) string { return a.name }
|
|
func (a *turnLoopCancellableMockAgent) Description(_ context.Context) string { return "mock agent" }
|
|
|
|
func (a *turnLoopCancellableMockAgent) Run(ctx context.Context, input *AgentInput, opts ...AgentRunOption) *AsyncIterator[*AgentEvent] {
|
|
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
|
|
|
|
o := getCommonOptions(nil, opts...)
|
|
cc := o.cancelCtx
|
|
|
|
a.mu.Lock()
|
|
var cancelCtx context.Context
|
|
cancelCtx, a.cancel = context.WithCancel(ctx)
|
|
a.mu.Unlock()
|
|
|
|
go func() {
|
|
defer gen.Close()
|
|
if cc != nil {
|
|
go func() {
|
|
<-cc.cancelChan
|
|
// CRITICAL: call onCancel BEFORE cancel() to avoid race condition.
|
|
// If cancel() fires first, the runFunc returns immediately,
|
|
// flowAgent's defer calls markDone(), and doneChan closes
|
|
// before onCancel can read cc.config.
|
|
if a.onCancel != nil {
|
|
a.onCancel(cc)
|
|
}
|
|
a.mu.Lock()
|
|
if a.cancel != nil {
|
|
a.cancel()
|
|
}
|
|
a.mu.Unlock()
|
|
}()
|
|
}
|
|
|
|
output, err := a.runFunc(cancelCtx, input)
|
|
if err != nil {
|
|
gen.Send(&AgentEvent{Err: err})
|
|
return
|
|
}
|
|
gen.Send(&AgentEvent{Output: output})
|
|
}()
|
|
return iter
|
|
}
|
|
|
|
type turnLoopStopModeProbeAgent struct {
|
|
ccCh chan *cancelContext
|
|
}
|
|
|
|
func (a *turnLoopStopModeProbeAgent) Name(_ context.Context) string { return "probe" }
|
|
func (a *turnLoopStopModeProbeAgent) Description(_ context.Context) string { return "probe" }
|
|
func (a *turnLoopStopModeProbeAgent) Run(_ context.Context, _ *AgentInput, opts ...AgentRunOption) *AsyncIterator[*AgentEvent] {
|
|
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
|
|
o := getCommonOptions(nil, opts...)
|
|
cc := o.cancelCtx
|
|
a.ccCh <- cc
|
|
go func() {
|
|
defer gen.Close()
|
|
<-cc.cancelChan
|
|
for {
|
|
if cc.getMode() == CancelImmediate {
|
|
gen.Send(&AgentEvent{Err: cc.createCancelError()})
|
|
return
|
|
}
|
|
time.Sleep(1 * time.Millisecond)
|
|
}
|
|
}()
|
|
return iter
|
|
}
|
|
|
|
func newAndRunTurnLoop[T any, M MessageType](ctx context.Context, cfg TurnLoopConfig[T, M]) *TurnLoop[T, M] {
|
|
l := NewTurnLoop(cfg)
|
|
l.Run(ctx)
|
|
return l
|
|
}
|
|
|
|
func genInputConsumeAll(_ context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
}
|
|
|
|
func genInputConsumeAllWithMsg(_ context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
}
|
|
|
|
func genInputConsumeFirst(_ context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: []string{items[0]},
|
|
Remaining: items[1:],
|
|
}, nil
|
|
}
|
|
|
|
var prepareTestAgent = func(_ context.Context, _ *TurnLoop[string, *schema.Message], _ []string) (Agent, error) {
|
|
return &turnLoopMockAgent{name: "test"}, nil
|
|
}
|
|
|
|
func prepareAgent(a Agent) func(context.Context, *TurnLoop[string, *schema.Message], []string) (Agent, error) {
|
|
return func(_ context.Context, _ *TurnLoop[string, *schema.Message], _ []string) (Agent, error) {
|
|
return a, nil
|
|
}
|
|
}
|
|
|
|
func waitOrFail(t *testing.T, ch <-chan struct{}, msg string) {
|
|
t.Helper()
|
|
select {
|
|
case <-ch:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal(msg)
|
|
}
|
|
}
|
|
|
|
func newTestStore() *turnLoopCheckpointStore {
|
|
return &turnLoopCheckpointStore{m: make(map[string][]byte)}
|
|
}
|
|
|
|
func newPreemptTestLoop(t *testing.T, agent *turnLoopCancellableMockAgent) *TurnLoop[string, *schema.Message] {
|
|
t.Helper()
|
|
|
|
agentStarted := make(chan struct{})
|
|
agentStartedOnce := sync.Once{}
|
|
|
|
originalRunFunc := agent.runFunc
|
|
agent.runFunc = func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
agentStartedOnce.Do(func() { close(agentStarted) })
|
|
return originalRunFunc(ctx, input)
|
|
}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: genInputConsumeFirst,
|
|
})
|
|
|
|
loop.Push("first")
|
|
|
|
waitOrFail(t, agentStarted, "agent did not start")
|
|
|
|
return loop
|
|
}
|
|
|
|
func TestTurnLoop_StopIsIdempotent(t *testing.T) {
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Stop()
|
|
loop.Stop()
|
|
loop.Stop()
|
|
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_WaitMultipleGoroutines(t *testing.T) {
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Stop()
|
|
|
|
var wg sync.WaitGroup
|
|
results := make([]*TurnLoopExitState[string, *schema.Message], 3)
|
|
|
|
for i := 0; i < 3; i++ {
|
|
i := i
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
results[i] = loop.Wait()
|
|
}()
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
assert.Equal(t, results[0], results[1])
|
|
assert.Equal(t, results[1], results[2])
|
|
}
|
|
|
|
func TestTurnLoop_UnhandledItemsOnStop(t *testing.T) {
|
|
started := make(chan struct{})
|
|
blocked := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
close(started)
|
|
<-blocked
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items[:1],
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
loop.Push("msg2")
|
|
loop.Push("msg3")
|
|
|
|
<-started
|
|
|
|
loop.Stop()
|
|
close(blocked)
|
|
|
|
result := loop.Wait()
|
|
assert.NotEmpty(t, result.UnhandledItems, "should return unhandled items")
|
|
}
|
|
|
|
func TestTurnLoop_GenInputError(t *testing.T) {
|
|
genErr := errors.New("gen input error")
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return nil, genErr
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, genErr)
|
|
}
|
|
|
|
func TestTurnLoop_GetAgentError(t *testing.T) {
|
|
agentErr := errors.New("get agent error")
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return nil, agentErr
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, agentErr)
|
|
}
|
|
|
|
func TestTurnLoop_BatchProcessing(t *testing.T) {
|
|
var batches [][]string
|
|
var mu sync.Mutex
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
mu.Lock()
|
|
batches = append(batches, items)
|
|
mu.Unlock()
|
|
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items[:1],
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
loop.Push("msg2")
|
|
loop.Push("msg3")
|
|
|
|
time.Sleep(200 * time.Millisecond)
|
|
|
|
loop.Stop()
|
|
loop.Wait()
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
|
|
assert.NotEmpty(t, batches, "should have processed at least one batch")
|
|
}
|
|
|
|
func TestTurnLoop_StopWithMode(t *testing.T) {
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Stop(WithGraceful())
|
|
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_Preempt_CancelsCurrentAgent(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentCancelled := make(chan struct{})
|
|
agentStartedOnce := sync.Once{}
|
|
agentCancelledOnce := sync.Once{}
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
agentStartedOnce.Do(func() {
|
|
close(agentStarted)
|
|
})
|
|
<-ctx.Done()
|
|
agentCancelledOnce.Do(func() {
|
|
close(agentCancelled)
|
|
})
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
genInputCalls := int32(0)
|
|
secondGenInputCalled := make(chan struct{})
|
|
secondGenInputOnce := sync.Once{}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
count := atomic.AddInt32(&genInputCalls, 1)
|
|
if count >= 2 {
|
|
secondGenInputOnce.Do(func() {
|
|
close(secondGenInputCalled)
|
|
})
|
|
}
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: []string{items[0]},
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("first")
|
|
|
|
waitOrFail(t, agentStarted, "agent did not start")
|
|
|
|
loop.Push("urgent", WithPreempt[string, *schema.Message](AnySafePoint))
|
|
|
|
waitOrFail(t, agentCancelled, "agent was not cancelled by preempt")
|
|
|
|
waitOrFail(t, secondGenInputCalled, "second GenInput was not called after preempt")
|
|
|
|
loop.Stop(WithImmediate())
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.GreaterOrEqual(t, atomic.LoadInt32(&genInputCalls), int32(2))
|
|
}
|
|
|
|
func TestTurnLoop_Preempt_DiscardsConsumedItems(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentDone := make(chan struct{})
|
|
agentStartedOnce := sync.Once{}
|
|
agentDoneOnce := sync.Once{}
|
|
firstAgentRun := true
|
|
var firstRunMu sync.Mutex
|
|
|
|
genInputResults := make([][]string, 0)
|
|
var mu sync.Mutex
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
firstRunMu.Lock()
|
|
isFirst := firstAgentRun
|
|
firstAgentRun = false
|
|
firstRunMu.Unlock()
|
|
|
|
if isFirst {
|
|
agentStartedOnce.Do(func() {
|
|
close(agentStarted)
|
|
})
|
|
<-ctx.Done()
|
|
} else {
|
|
agentDoneOnce.Do(func() {
|
|
close(agentDone)
|
|
})
|
|
}
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
mu.Lock()
|
|
genInputResults = append(genInputResults, items)
|
|
mu.Unlock()
|
|
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: []string{items[0]},
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("first")
|
|
|
|
waitOrFail(t, agentStarted, "agent did not start")
|
|
|
|
loop.Push("urgent", WithPreempt[string, *schema.Message](AnySafePoint))
|
|
|
|
waitOrFail(t, agentDone, "second agent run did not complete")
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
require.GreaterOrEqual(t, len(genInputResults), 2)
|
|
assert.NotContains(t, genInputResults[1], "first")
|
|
assert.Contains(t, genInputResults[1], "urgent")
|
|
}
|
|
|
|
func TestTurnLoop_Preempt_WithAgentCancelMode(t *testing.T) {
|
|
cancelFuncCalled := make(chan struct{})
|
|
cancelFuncCalledOnce := sync.Once{}
|
|
firstCancelModeUsed := CancelImmediate
|
|
var cancelModeMu sync.Mutex
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
return &AgentOutput{}, nil
|
|
},
|
|
onCancel: func(cc *cancelContext) {
|
|
cancelModeMu.Lock()
|
|
cancelFuncCalledOnce.Do(func() {
|
|
firstCancelModeUsed = cc.getMode()
|
|
close(cancelFuncCalled)
|
|
})
|
|
cancelModeMu.Unlock()
|
|
},
|
|
}
|
|
|
|
loop := newPreemptTestLoop(t, agent)
|
|
|
|
loop.Push("urgent", WithPreempt[string, *schema.Message](AfterToolCalls))
|
|
|
|
waitOrFail(t, cancelFuncCalled, "cancelFunc was not called by preempt")
|
|
|
|
loop.Stop(WithImmediate())
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
cancelModeMu.Lock()
|
|
actualMode := firstCancelModeUsed
|
|
cancelModeMu.Unlock()
|
|
assert.Equal(t, CancelAfterToolCalls, actualMode)
|
|
}
|
|
|
|
func TestTurnLoop_PreemptAck_ClosesAfterCancelIsInitiated(t *testing.T) {
|
|
cancelObserved := make(chan struct{})
|
|
agentFinishGate := make(chan struct{})
|
|
cancelObservedOnce := sync.Once{}
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
<-agentFinishGate
|
|
return &AgentOutput{}, nil
|
|
},
|
|
onCancel: func(cc *cancelContext) {
|
|
cancelObservedOnce.Do(func() { close(cancelObserved) })
|
|
},
|
|
}
|
|
|
|
loop := newPreemptTestLoop(t, agent)
|
|
|
|
ok, ack := loop.Push("urgent", WithPreempt[string, *schema.Message](AfterToolCalls))
|
|
assert.True(t, ok)
|
|
assert.NotNil(t, ack)
|
|
|
|
waitOrFail(t, ack, "preempt ack was not closed")
|
|
|
|
waitOrFail(t, cancelObserved, "cancel was not initiated")
|
|
|
|
close(agentFinishGate)
|
|
|
|
loop.Stop(WithImmediate())
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_PreemptAck_ClosesImmediatelyIfLoopNotStarted(t *testing.T) {
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
ok, ack := loop.Push("urgent", WithPreempt[string, *schema.Message](AnySafePoint))
|
|
assert.True(t, ok)
|
|
assert.NotNil(t, ack)
|
|
|
|
waitOrFail(t, ack, "preempt ack was not closed")
|
|
}
|
|
|
|
func TestTurnLoop_Preempt_EscalatesOnSecondPreempt(t *testing.T) {
|
|
firstCancelSeen := make(chan struct{})
|
|
agentFinishGate := make(chan struct{})
|
|
firstCancelOnce := sync.Once{}
|
|
|
|
var ccPtr atomic.Value
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
<-agentFinishGate
|
|
return &AgentOutput{}, nil
|
|
},
|
|
onCancel: func(cc *cancelContext) {
|
|
ccPtr.Store(cc)
|
|
firstCancelOnce.Do(func() { close(firstCancelSeen) })
|
|
},
|
|
}
|
|
|
|
loop := newPreemptTestLoop(t, agent)
|
|
|
|
loop.Push("urgent1", WithPreempt[string, *schema.Message](AfterChatModel))
|
|
waitOrFail(t, firstCancelSeen, "first preempt did not trigger cancel")
|
|
|
|
loop.Push("urgent2", WithPreemptTimeout[string, *schema.Message](AnySafePoint, time.Millisecond))
|
|
|
|
wantMode := CancelAfterChatModel | CancelAfterToolCalls
|
|
deadline := time.Now().Add(1 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
v := ccPtr.Load()
|
|
if v == nil {
|
|
time.Sleep(5 * time.Millisecond)
|
|
continue
|
|
}
|
|
cc := v.(*cancelContext)
|
|
if cc.getMode() == wantMode && atomic.LoadInt32(&cc.escalated) == 1 {
|
|
break
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
|
|
v := ccPtr.Load()
|
|
if v == nil {
|
|
t.Fatal("cancel context was not captured")
|
|
}
|
|
cc := v.(*cancelContext)
|
|
assert.Equal(t, wantMode, cc.getMode())
|
|
assert.Equal(t, int32(1), atomic.LoadInt32(&cc.escalated))
|
|
|
|
close(agentFinishGate)
|
|
|
|
loop.Stop(WithImmediate())
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_Preempt_JoinsSafePointModesOnSecondPreempt(t *testing.T) {
|
|
firstCancelSeen := make(chan struct{})
|
|
agentFinishGate := make(chan struct{})
|
|
firstCancelOnce := sync.Once{}
|
|
|
|
var ccPtr atomic.Value
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
<-agentFinishGate
|
|
return &AgentOutput{}, nil
|
|
},
|
|
onCancel: func(cc *cancelContext) {
|
|
ccPtr.Store(cc)
|
|
firstCancelOnce.Do(func() { close(firstCancelSeen) })
|
|
},
|
|
}
|
|
|
|
loop := newPreemptTestLoop(t, agent)
|
|
|
|
loop.Push("urgent1", WithPreempt[string, *schema.Message](AfterChatModel))
|
|
waitOrFail(t, firstCancelSeen, "first preempt did not trigger cancel")
|
|
|
|
loop.Push("urgent2", WithPreempt[string, *schema.Message](AfterToolCalls))
|
|
|
|
want := CancelAfterChatModel | CancelAfterToolCalls
|
|
deadline := time.Now().Add(1 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
v := ccPtr.Load()
|
|
if v == nil {
|
|
time.Sleep(5 * time.Millisecond)
|
|
continue
|
|
}
|
|
cc := v.(*cancelContext)
|
|
if cc.getMode() == want {
|
|
break
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
|
|
v := ccPtr.Load()
|
|
if v == nil {
|
|
t.Fatal("cancel context was not captured")
|
|
}
|
|
cc := v.(*cancelContext)
|
|
assert.Equal(t, want, cc.getMode())
|
|
|
|
close(agentFinishGate)
|
|
|
|
loop.Stop(WithImmediate())
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_Push_WithoutPreempt_DoesNotCancel(t *testing.T) {
|
|
agentRunCount := 0
|
|
agentDone := make(chan struct{})
|
|
|
|
agent := &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
agentRunCount++
|
|
if agentRunCount == 1 {
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
if agentRunCount == 2 {
|
|
close(agentDone)
|
|
}
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: genInputConsumeFirst,
|
|
})
|
|
|
|
loop.Push("first")
|
|
time.Sleep(20 * time.Millisecond)
|
|
loop.Push("second")
|
|
|
|
waitOrFail(t, agentDone, "second agent run did not complete")
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.Equal(t, 2, agentRunCount)
|
|
}
|
|
|
|
func TestTurnLoop_PreemptDelay_NoMispreemptOnNaturalCompletion(t *testing.T) {
|
|
agent1Started := make(chan struct{})
|
|
allowAgent1Done := make(chan struct{})
|
|
agent1Done := make(chan struct{})
|
|
agent2Started := make(chan struct{})
|
|
allowAgent2Done := make(chan struct{})
|
|
agent2Done := make(chan struct{})
|
|
agent1StartedOnce := sync.Once{}
|
|
agent1DoneOnce := sync.Once{}
|
|
agent2StartedOnce := sync.Once{}
|
|
agent2DoneOnce := sync.Once{}
|
|
|
|
var agentRunCount int32
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
count := atomic.AddInt32(&agentRunCount, 1)
|
|
switch count {
|
|
case 1:
|
|
agent1StartedOnce.Do(func() { close(agent1Started) })
|
|
<-allowAgent1Done
|
|
agent1DoneOnce.Do(func() { close(agent1Done) })
|
|
case 2:
|
|
agent2StartedOnce.Do(func() { close(agent2Started) })
|
|
select {
|
|
case <-allowAgent2Done:
|
|
case <-ctx.Done():
|
|
t.Error("Agent2 was unexpectedly cancelled")
|
|
return nil, ctx.Err()
|
|
}
|
|
agent2DoneOnce.Do(func() { close(agent2Done) })
|
|
}
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: genInputConsumeFirst,
|
|
})
|
|
|
|
loop.Push("first")
|
|
|
|
waitOrFail(t, agent1Started, "agent1 did not start")
|
|
|
|
_, ack := loop.Push("second", WithPreempt[string, *schema.Message](AnySafePoint), WithPreemptDelay[string, *schema.Message](500*time.Millisecond))
|
|
require.NotNil(t, ack)
|
|
close(allowAgent1Done)
|
|
|
|
waitOrFail(t, agent1Done, "agent1 did not complete naturally")
|
|
|
|
waitOrFail(t, agent2Started, "agent2 did not start")
|
|
|
|
requireAckClosed(t, ack)
|
|
close(allowAgent2Done)
|
|
|
|
waitOrFail(t, agent2Done, "agent2 did not complete - may have been incorrectly preempted")
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.Equal(t, int32(2), atomic.LoadInt32(&agentRunCount))
|
|
}
|
|
|
|
func TestTurnLoop_PreemptDuringPlanningCancelsUpcomingAgent(t *testing.T) {
|
|
genInputEntered := make(chan struct{})
|
|
allowGenInput := make(chan struct{})
|
|
cancelSubmitted := make(chan struct{})
|
|
agentCancelled := make(chan struct{})
|
|
genInputOnce := sync.Once{}
|
|
cancelOnce := sync.Once{}
|
|
agentCancelledOnce := sync.Once{}
|
|
|
|
var agentRunCount int32
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
count := atomic.AddInt32(&agentRunCount, 1)
|
|
if count == 1 {
|
|
<-ctx.Done()
|
|
agentCancelledOnce.Do(func() { close(agentCancelled) })
|
|
return &AgentOutput{}, nil
|
|
}
|
|
return &AgentOutput{}, nil
|
|
},
|
|
onCancel: func(cc *cancelContext) {
|
|
cancelOnce.Do(func() { close(cancelSubmitted) })
|
|
},
|
|
}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
genInputOnce.Do(func() {
|
|
close(genInputEntered)
|
|
<-allowGenInput
|
|
})
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: []string{items[0]},
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("first")
|
|
|
|
waitOrFail(t, genInputEntered, "GenInput did not enter planning phase")
|
|
|
|
ok, ack := loop.Push("urgent", WithPreempt[string, *schema.Message](AnySafePoint))
|
|
require.True(t, ok)
|
|
require.NotNil(t, ack)
|
|
requireAckOpen(t, ack)
|
|
|
|
close(allowGenInput)
|
|
|
|
waitOrFail(t, cancelSubmitted, "planning-phase preempt did not submit cancel")
|
|
requireAckClosed(t, ack)
|
|
|
|
waitOrFail(t, agentCancelled, "agent was not cancelled by planning-phase preempt")
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.GreaterOrEqual(t, atomic.LoadInt32(&agentRunCount), int32(1))
|
|
}
|
|
|
|
func TestTurnLoop_ConcurrentPush(t *testing.T) {
|
|
var count int32
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
atomic.AddInt32(&count, int32(len(items)))
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 10; i++ {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
for j := 0; j < 10; j++ {
|
|
_, _ = loop.Push(fmt.Sprintf("msg-%d-%d", i, j))
|
|
}
|
|
}(i)
|
|
}
|
|
|
|
wg.Wait()
|
|
time.Sleep(200 * time.Millisecond)
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
|
|
processed := atomic.LoadInt32(&count)
|
|
unhandled := len(result.UnhandledItems)
|
|
|
|
assert.True(t, processed > 0, "should have processed some items")
|
|
assert.True(t, int(processed)+unhandled <= 100, "total should not exceed pushed amount")
|
|
}
|
|
|
|
func TestTurnLoop_StopAfterReceive_RecoverItem(t *testing.T) {
|
|
receiveStarted := make(chan struct{})
|
|
cancelDone := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
close(receiveStarted)
|
|
<-cancelDone
|
|
time.Sleep(50 * time.Millisecond)
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-receiveStarted
|
|
|
|
loop.Stop()
|
|
close(cancelDone)
|
|
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_StopAfterGenInput_RecoverConsumed(t *testing.T) {
|
|
genInputDone := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
close(genInputDone)
|
|
time.Sleep(50 * time.Millisecond)
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items[:1],
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
time.Sleep(100 * time.Millisecond)
|
|
return &turnLoopMockAgent{name: "test"}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
loop.Push("msg2")
|
|
|
|
<-genInputDone
|
|
|
|
time.Sleep(60 * time.Millisecond)
|
|
loop.Stop()
|
|
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_GetAgentError_RecoverConsumed(t *testing.T) {
|
|
agentErr := errors.New("get agent error")
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items[:1],
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], c []string) (Agent, error) {
|
|
return nil, agentErr
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
loop.Push("msg2")
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, agentErr)
|
|
assert.NotEmpty(t, result.UnhandledItems, "should recover at least the consumed item and remaining")
|
|
}
|
|
|
|
func TestTurnLoop_GenInputError_RecoverItems(t *testing.T) {
|
|
genErr := errors.New("gen input error")
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return nil, genErr
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
loop.Push("msg2")
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, genErr)
|
|
assert.Len(t, result.UnhandledItems, 2, "should recover all items when GenInput fails")
|
|
assert.Contains(t, result.UnhandledItems, "msg1")
|
|
assert.Contains(t, result.UnhandledItems, "msg2")
|
|
}
|
|
|
|
func TestTurnLoop_PrepareAgentError_RecoverItemsInOrder(t *testing.T) {
|
|
agentErr := errors.New("prepare agent error")
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
var urgent string
|
|
remaining := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
if item == "urgent" {
|
|
urgent = item
|
|
} else {
|
|
remaining = append(remaining, item)
|
|
}
|
|
}
|
|
if urgent != "" {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: []string{urgent},
|
|
Remaining: remaining,
|
|
}, nil
|
|
}
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items[:1],
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return nil, agentErr
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
loop.Push("urgent")
|
|
loop.Push("msg2")
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, agentErr)
|
|
assert.Len(t, result.UnhandledItems, 3, "should recover all items")
|
|
assert.Equal(t, []string{"msg1", "urgent", "msg2"}, result.UnhandledItems,
|
|
"should preserve original push order even when GenInput selects non-prefix items")
|
|
}
|
|
|
|
// Context cancel tests: the TurnLoop monitors context cancellation by closing
|
|
// the internal buffer when ctx.Done() fires, which unblocks the blocking
|
|
// Receive() call. The loop then checks ctx.Err() and exits with the context error.
|
|
|
|
func TestTurnLoop_ContextCancel(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
genInputStarted := make(chan struct{})
|
|
genInputDone := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
close(genInputStarted)
|
|
<-genInputDone
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
<-genInputStarted
|
|
cancel()
|
|
close(genInputDone)
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, context.Canceled)
|
|
}
|
|
|
|
func TestTurnLoop_ContextDeadlineExceeded(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
|
defer cancel()
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
select {
|
|
case <-time.After(100 * time.Millisecond):
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, context.DeadlineExceeded)
|
|
}
|
|
|
|
func TestTurnLoop_ContextCancelBeforeReceive(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
// Push before Run to guarantee the item is buffered before the
|
|
// context-monitoring goroutine can close the buffer.
|
|
_, _ = loop.Push("msg1")
|
|
loop.Run(ctx)
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, context.Canceled)
|
|
assert.Len(t, result.UnhandledItems, 1)
|
|
}
|
|
|
|
func TestTurnLoop_ContextCancelDuringBlockingReceive(t *testing.T) {
|
|
// When context is cancelled while Receive() is blocking (no items in buffer),
|
|
// the context monitoring goroutine closes the buffer, which unblocks Receive().
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
// Don't push any items — let Receive() block
|
|
time.Sleep(50 * time.Millisecond)
|
|
cancel()
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, context.Canceled)
|
|
}
|
|
|
|
func TestTurnLoop_ContextCancelAfterGenInput_RecoverItems(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
genInputCount := 0
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
genInputCount++
|
|
if genInputCount == 1 {
|
|
cancel()
|
|
}
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items[:1],
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], c []string) (Agent, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
return &turnLoopMockAgent{name: "test"}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
loop.Push("msg2")
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, context.Canceled)
|
|
assert.NotEmpty(t, result.UnhandledItems, "should recover consumed and remaining items")
|
|
}
|
|
|
|
func TestTurnLoop_OnAgentEventsReceivesEvents(t *testing.T) {
|
|
var receivedEvents []*AgentEvent
|
|
var receivedConsumed []string
|
|
var mu sync.Mutex
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
mu.Lock()
|
|
receivedConsumed = append(receivedConsumed, tc.Consumed...)
|
|
mu.Unlock()
|
|
|
|
for {
|
|
event, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
mu.Lock()
|
|
receivedEvents = append(receivedEvents, event)
|
|
mu.Unlock()
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
|
|
assert.NoError(t, result.ExitReason)
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
assert.NotEmpty(t, receivedConsumed, "should have received consumed items")
|
|
}
|
|
|
|
// TestTurnLoop_BareStop_AgentRunsToCompletion verifies the core contract of
|
|
// bare Stop(): the running agent finishes naturally with an uncanceled context,
|
|
// the loop exits cleanly (ExitReason == nil), and no new turn starts even when
|
|
// additional items are buffered.
|
|
func TestTurnLoop_BareStop_AgentRunsToCompletion(t *testing.T) {
|
|
const agentWorkDuration = 200 * time.Millisecond
|
|
|
|
agentStarted := make(chan struct{})
|
|
agentCtxErr := make(chan error, 1)
|
|
agentOutput := make(chan string, 1)
|
|
|
|
turnsExecuted := int32(0)
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeFirst,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "worker",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
atomic.AddInt32(&turnsExecuted, 1)
|
|
close(agentStarted)
|
|
|
|
// Simulate real work (NOT blocking on <-ctx.Done())
|
|
time.Sleep(agentWorkDuration)
|
|
|
|
// Record context state AFTER work completes
|
|
agentCtxErr <- ctx.Err()
|
|
agentOutput <- "work-done"
|
|
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
// Push two items so the loop has a reason to start a second turn.
|
|
loop.Push("task1")
|
|
loop.Push("task2")
|
|
|
|
// Wait for the agent to start processing task1.
|
|
waitOrFail(t, agentStarted, "agent did not start")
|
|
|
|
// Call bare Stop() while the agent is doing work.
|
|
loop.Stop()
|
|
|
|
result := loop.Wait()
|
|
|
|
// 1. Agent's context was NOT canceled.
|
|
select {
|
|
case err := <-agentCtxErr:
|
|
assert.NoError(t, err, "bare Stop must not cancel the agent's context")
|
|
default:
|
|
t.Fatal("agent never reported context state")
|
|
}
|
|
|
|
// 2. Agent completed its work.
|
|
select {
|
|
case out := <-agentOutput:
|
|
assert.Equal(t, "work-done", out)
|
|
default:
|
|
t.Fatal("agent never produced output")
|
|
}
|
|
|
|
// 3. ExitReason is nil (clean exit, not a CancelError).
|
|
assert.NoError(t, result.ExitReason)
|
|
|
|
// 4. InterruptedItems is empty (agent was not interrupted).
|
|
assert.Empty(t, result.InterruptedItems)
|
|
|
|
// 5. Only one turn executed; the second item is unhandled.
|
|
assert.Equal(t, int32(1), atomic.LoadInt32(&turnsExecuted),
|
|
"bare Stop must prevent new turns from starting after the current one completes")
|
|
assert.Equal(t, []string{"task2"}, result.UnhandledItems,
|
|
"the second item should appear in UnhandledItems")
|
|
}
|
|
|
|
func TestTurnLoop_StopCheckPointIDInCancelError(t *testing.T) {
|
|
ctx := context.Background()
|
|
modelStarted := make(chan struct{}, 1)
|
|
checkpointID := "turn-loop-cancel-ckpt-1"
|
|
store := newTestStore()
|
|
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: checkpointID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
<-modelStarted
|
|
loop.Stop(WithImmediate())
|
|
|
|
result := loop.Wait()
|
|
|
|
var cancelErr *CancelError
|
|
assert.True(t, errors.As(result.ExitReason, &cancelErr), "ExitReason should be a *CancelError")
|
|
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
_, ok := store.m[checkpointID]
|
|
assert.True(t, ok, "checkpoint should be saved under the configured CheckpointID")
|
|
}
|
|
|
|
// TestTurnLoop_CancelError_CapturedIndependentlyOfCallback verifies that the TurnLoop
|
|
// correctly reports *CancelError as ExitReason and populates InterruptedItems even when
|
|
// the user's custom OnAgentEvents callback swallows the CancelError (returns nil).
|
|
// This tests the documented guarantee: "the callback should NEVER propagate CancelError
|
|
// — the framework handles it automatically."
|
|
func TestTurnLoop_CancelError_CapturedIndependentlyOfCallback(t *testing.T) {
|
|
ctx := context.Background()
|
|
modelStarted := make(chan struct{}, 1)
|
|
checkpointID := "cancel-capture-independent-1"
|
|
store := newTestStore()
|
|
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: checkpointID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
// Custom OnAgentEvents that deliberately swallows all errors including CancelError.
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*TypedAgentEvent[*schema.Message]]) error {
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
// Deliberately ignore event.Err — do NOT propagate CancelError.
|
|
}
|
|
return nil // swallow everything
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
<-modelStarted
|
|
loop.Stop(WithImmediate())
|
|
|
|
result := loop.Wait()
|
|
|
|
// The framework should capture CancelError independently of the callback's return value.
|
|
var cancelErr *CancelError
|
|
assert.True(t, errors.As(result.ExitReason, &cancelErr),
|
|
"ExitReason should be *CancelError even when OnAgentEvents swallows it, got: %v", result.ExitReason)
|
|
|
|
// InterruptedItems should be populated.
|
|
assert.Equal(t, []string{"msg1"}, result.InterruptedItems,
|
|
"InterruptedItems should contain the items that were being processed")
|
|
|
|
// Checkpoint should be saved.
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
_, ok := store.m[checkpointID]
|
|
assert.True(t, ok, "checkpoint should be saved under the configured CheckpointID")
|
|
}
|
|
|
|
// TestTurnLoop_CancelError_CustomErrorWins_InterruptedItemsStillSet verifies that when
|
|
// the user's OnAgentEvents callback returns a custom error during a cancel, the custom
|
|
// error becomes ExitReason (not overwritten by CancelError), but InterruptedItems is still
|
|
// populated because the items were factually mid-execution when the cancel signal arrived.
|
|
func TestTurnLoop_CancelError_CustomErrorWins_InterruptedItemsStillSet(t *testing.T) {
|
|
ctx := context.Background()
|
|
modelStarted := make(chan struct{}, 1)
|
|
checkpointID := "cancel-custom-error-wins-1"
|
|
store := newTestStore()
|
|
customErr := fmt.Errorf("user callback encountered a problem")
|
|
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: checkpointID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
// Custom OnAgentEvents that returns a custom error instead of the CancelError.
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*TypedAgentEvent[*schema.Message]]) error {
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
}
|
|
return customErr
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
<-modelStarted
|
|
loop.Stop(WithImmediate())
|
|
|
|
result := loop.Wait()
|
|
|
|
// User's custom error should win as ExitReason.
|
|
assert.ErrorIs(t, result.ExitReason, customErr,
|
|
"ExitReason should be the user's custom error, not CancelError")
|
|
|
|
// But InterruptedItems should still be populated (items were factually in-flight).
|
|
assert.Equal(t, []string{"msg1"}, result.InterruptedItems,
|
|
"InterruptedItems should contain the items that were being processed")
|
|
|
|
// Checkpoint should be saved (cancel was captured, items were in-flight).
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
_, ok := store.m[checkpointID]
|
|
assert.True(t, ok, "checkpoint should be saved even when user returns custom error")
|
|
}
|
|
|
|
func TestTurnLoop_StopWithoutCheckpointIDDoesNotPersist(t *testing.T) {
|
|
ctx := context.Background()
|
|
modelStarted := make(chan struct{}, 1)
|
|
store := newTestStore()
|
|
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
<-modelStarted
|
|
loop.Stop(WithImmediate())
|
|
|
|
result := loop.Wait()
|
|
|
|
var cancelErr *CancelError
|
|
assert.True(t, errors.As(result.ExitReason, &cancelErr), "ExitReason should be a *CancelError")
|
|
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
assert.Empty(t, store.m, "no checkpoint should be saved when CheckpointID is not configured")
|
|
}
|
|
|
|
func TestTurnLoop_StopWhileIdle_SkipsCheckpoint(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := &deletableCheckpointStore{
|
|
turnLoopCheckpointStore: turnLoopCheckpointStore{m: make(map[string][]byte)},
|
|
}
|
|
cpID := "idle-session"
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Stop()
|
|
exit := loop.Wait()
|
|
assert.NoError(t, exit.ExitReason)
|
|
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
_, exists := store.m[cpID]
|
|
assert.False(t, exists, "no checkpoint should be saved when TurnLoop is idle")
|
|
}
|
|
|
|
func TestTurnLoop_StopBetweenTurnsAndResume(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "between-turns-session"
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("a")
|
|
loop.Push("b")
|
|
loop.Stop()
|
|
loop.Run(ctx)
|
|
|
|
exit := loop.Wait()
|
|
assert.NoError(t, exit.ExitReason)
|
|
|
|
var seen []string
|
|
var mu sync.Mutex
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
mu.Lock()
|
|
seen = append([]string{}, items...)
|
|
mu.Unlock()
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop2.Push("c")
|
|
loop2.Run(ctx)
|
|
exit2 := loop2.Wait()
|
|
assert.NoError(t, exit2.ExitReason)
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
assert.Equal(t, []string{"a", "b", "c"}, seen)
|
|
}
|
|
|
|
// TestTurnLoop_StopCancel_InterruptedItems_PersistAndRestore verifies the full
|
|
// lifecycle: Stop(WithImmediate()) cancels a running agent, the exit state
|
|
// reports the correct InterruptedItems, the checkpoint persists them, and a new
|
|
// TurnLoop restoring from that checkpoint passes the exact same items to
|
|
// GenResume's interruptedItems parameter.
|
|
func TestTurnLoop_StopCancel_InterruptedItems_PersistAndRestore(t *testing.T) {
|
|
ctx := context.Background()
|
|
modelStarted := make(chan struct{}, 1)
|
|
store := newTestStore()
|
|
cpID := "inflight-persist-session"
|
|
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Phase 1: Run, cancel mid-turn, verify exit state.
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-modelStarted
|
|
loop.Stop(WithImmediate())
|
|
exit := loop.Wait()
|
|
|
|
// ExitReason must be CancelError.
|
|
var cancelErr *CancelError
|
|
require.True(t, errors.As(exit.ExitReason, &cancelErr), "expected *CancelError, got: %v", exit.ExitReason)
|
|
|
|
// InterruptedItems must contain the consumed items.
|
|
assert.Equal(t, []string{"msg1"}, exit.InterruptedItems,
|
|
"InterruptedItems in exit state should contain the items that were mid-execution")
|
|
|
|
// Checkpoint must be saved.
|
|
assert.True(t, exit.CheckpointAttempted, "checkpoint should be attempted")
|
|
assert.NoError(t, exit.CheckpointErr, "checkpoint save should succeed")
|
|
|
|
store.mu.Lock()
|
|
_, cpExists := store.m[cpID]
|
|
store.mu.Unlock()
|
|
require.True(t, cpExists, "checkpoint should exist in store")
|
|
|
|
// Phase 2: Restore from checkpoint, verify GenResume receives the exact InterruptedItems.
|
|
slowModel.setDelay(10 * time.Millisecond)
|
|
|
|
var genResumeCalled bool
|
|
var resumeInterruptedItems []string
|
|
var resumeUnhandledItems []string
|
|
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenResume: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], interruptedItems, unhandledItems, newItems []string) (*GenResumeResult[string, *schema.Message], error) {
|
|
genResumeCalled = true
|
|
resumeInterruptedItems = append([]string{}, interruptedItems...)
|
|
resumeUnhandledItems = append([]string{}, unhandledItems...)
|
|
return &GenResumeResult[string, *schema.Message]{
|
|
Consumed: interruptedItems,
|
|
Remaining: append(append([]string{}, unhandledItems...), newItems...),
|
|
}, nil
|
|
},
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
t.Fatal("GenInput should not be called when resuming from a cancel checkpoint")
|
|
return nil, nil
|
|
},
|
|
PrepareAgent: prepareAgent(agent),
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop2.Run(ctx)
|
|
exit2 := loop2.Wait()
|
|
|
|
assert.NoError(t, exit2.ExitReason)
|
|
assert.True(t, genResumeCalled, "GenResume should be called when restoring from a cancel checkpoint")
|
|
assert.Equal(t, []string{"msg1"}, resumeInterruptedItems,
|
|
"GenResume's interruptedItems should match the original exit state's InterruptedItems")
|
|
assert.Empty(t, resumeUnhandledItems,
|
|
"unhandledItems should be empty (all items were consumed before cancel)")
|
|
}
|
|
|
|
// TestTurnLoop_PreemptThenStop_InterruptedItems_ReflectsStoppedTurn verifies that
|
|
// when a preempt interrupts turn 1 and then Stop(WithImmediate()) cancels turn 2,
|
|
// the reported InterruptedItems correspond to turn 2's consumed items (not turn 1's).
|
|
func TestTurnLoop_PreemptThenStop_InterruptedItems_ReflectsStoppedTurn(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "preempt-then-stop-session"
|
|
|
|
turnCount := int32(0)
|
|
modelStarted := make(chan struct{}, 10)
|
|
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(5 * time.Second),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 10),
|
|
}
|
|
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
n := atomic.AddInt32(&turnCount, 1)
|
|
_ = n
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: prepareAgent(agent),
|
|
})
|
|
|
|
// Turn 1: push "a", wait for agent to start, then preempt with "b".
|
|
loop.Push("a")
|
|
<-modelStarted
|
|
|
|
_, ack := loop.Push("b", WithPreemptTimeout[string, *schema.Message](AnySafePoint, 10*time.Millisecond))
|
|
// Wait for the preempt to be acknowledged.
|
|
waitOrFail(t, ack, "preempt ack timed out")
|
|
|
|
// Turn 2: the loop restarts with all buffered items. Wait for agent to start.
|
|
<-modelStarted
|
|
|
|
// Now stop with immediate cancel on turn 2.
|
|
loop.Stop(WithImmediate())
|
|
exit := loop.Wait()
|
|
|
|
// ExitReason must be CancelError.
|
|
var cancelErr *CancelError
|
|
require.True(t, errors.As(exit.ExitReason, &cancelErr), "expected *CancelError, got: %v", exit.ExitReason)
|
|
|
|
// KEY ASSERTION: InterruptedItems must reflect turn 2's consumed items (which
|
|
// includes both "a" and "b" since GenInput consumes all items), NOT turn 1's ["a"].
|
|
// Turn 2 consumed whatever GenInput received (the re-buffered items from preempt + "b").
|
|
assert.NotEmpty(t, exit.InterruptedItems, "InterruptedItems should not be empty after Stop cancel")
|
|
assert.Contains(t, exit.InterruptedItems, "b",
|
|
"InterruptedItems should contain 'b' which was part of turn 2's consumed items")
|
|
}
|
|
|
|
func TestTurnLoop_BusinessInterrupt_PersistAndResume(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "interrupt-session"
|
|
|
|
// Agent that produces a business interrupt via Interrupt() call.
|
|
interruptAgent := &turnLoopInterruptAgent{interruptInfo: "approval_needed"}
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return interruptAgent, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
exit := loop.Wait()
|
|
|
|
// 1. ExitReason is an *InterruptError (not nil, not *CancelError).
|
|
var intErr *InterruptError
|
|
require.True(t, errors.As(exit.ExitReason, &intErr), "expected *InterruptError, got: %v", exit.ExitReason)
|
|
|
|
// 2. InterruptContexts is populated.
|
|
require.NotEmpty(t, intErr.InterruptContexts)
|
|
|
|
// 3. InterruptedItems contains the items being processed.
|
|
assert.Equal(t, []string{"msg1"}, exit.InterruptedItems)
|
|
|
|
// 4. Checkpoint was persisted.
|
|
assert.True(t, exit.CheckpointAttempted)
|
|
assert.NoError(t, exit.CheckpointErr)
|
|
|
|
store.mu.Lock()
|
|
_, cpExists := store.m[cpID]
|
|
store.mu.Unlock()
|
|
assert.True(t, cpExists, "checkpoint should exist in store")
|
|
|
|
// 5. Resume: new TurnLoop with same CheckpointID gets GenResume called.
|
|
var genResumeCalled bool
|
|
var resumeInterruptedItems []string
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenResume: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], interruptedItems []string, unhandledItems []string, newItems []string) (*GenResumeResult[string, *schema.Message], error) {
|
|
genResumeCalled = true
|
|
resumeInterruptedItems = append([]string{}, interruptedItems...)
|
|
return &GenResumeResult[string, *schema.Message]{
|
|
Consumed: interruptedItems,
|
|
Remaining: append(append([]string{}, unhandledItems...), newItems...),
|
|
}, nil
|
|
},
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
// On resume, agent completes normally.
|
|
return &turnLoopMockAgent{
|
|
name: "ResumeAgent",
|
|
events: []*AgentEvent{{Output: &AgentOutput{}}},
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop2.Run(ctx)
|
|
exit2 := loop2.Wait()
|
|
assert.NoError(t, exit2.ExitReason)
|
|
assert.True(t, genResumeCalled, "GenResume should be called on checkpoint resume")
|
|
assert.Equal(t, []string{"msg1"}, resumeInterruptedItems, "interruptedItems should contain the original items")
|
|
}
|
|
|
|
// turnLoopInterruptAgent is a test agent that produces a business interrupt event.
|
|
type turnLoopInterruptAgent struct {
|
|
interruptInfo any
|
|
}
|
|
|
|
func (a *turnLoopInterruptAgent) Name(_ context.Context) string { return "InterruptAgent" }
|
|
func (a *turnLoopInterruptAgent) Description(_ context.Context) string {
|
|
return "agent that interrupts"
|
|
}
|
|
func (a *turnLoopInterruptAgent) Run(ctx context.Context, _ *AgentInput, _ ...AgentRunOption) *AsyncIterator[*AgentEvent] {
|
|
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
|
|
go func() {
|
|
defer gen.Close()
|
|
event := Interrupt(ctx, a.interruptInfo)
|
|
gen.Send(event)
|
|
}()
|
|
return iter
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointIDWithoutStore_FreshStart(t *testing.T) {
|
|
ctx := context.Background()
|
|
var genInputCalled bool
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
CheckpointID: "some-id",
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
genInputCalled = true
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
loop.Push("a")
|
|
loop.Run(ctx)
|
|
exit := loop.Wait()
|
|
assert.NoError(t, exit.ExitReason)
|
|
assert.True(t, genInputCalled)
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointNotFound_FreshStart(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
var genInputCalled bool
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: "nonexistent-id",
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
genInputCalled = true
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
loop.Push("a")
|
|
loop.Run(ctx)
|
|
exit := loop.Wait()
|
|
assert.NoError(t, exit.ExitReason)
|
|
assert.True(t, genInputCalled)
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointEmptyData_TreatedAsNoCheckpoint(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
store.m["cp-empty"] = nil
|
|
|
|
var genInputCalled bool
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: "cp-empty",
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
genInputCalled = true
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
loop.Push("a")
|
|
loop.Run(ctx)
|
|
exit := loop.Wait()
|
|
assert.NoError(t, exit.ExitReason)
|
|
assert.True(t, genInputCalled)
|
|
}
|
|
|
|
type errorCheckpointStore struct {
|
|
getErr error
|
|
setErr error
|
|
}
|
|
|
|
func (s *errorCheckpointStore) Get(_ context.Context, _ string) ([]byte, bool, error) {
|
|
return nil, false, s.getErr
|
|
}
|
|
|
|
func (s *errorCheckpointStore) Set(_ context.Context, _ string, _ []byte) error {
|
|
return s.setErr
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointLoadError_ReturnsError(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := &errorCheckpointStore{getErr: fmt.Errorf("store unavailable")}
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: "cp-1",
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop.Push("a")
|
|
loop.Run(ctx)
|
|
exit := loop.Wait()
|
|
assert.Error(t, exit.ExitReason)
|
|
assert.Contains(t, exit.ExitReason.Error(), "store unavailable")
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointCorruptData_ReturnsError(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
store.m["cp-corrupt"] = []byte("not-valid-gob-data")
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: "cp-corrupt",
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop.Push("a")
|
|
loop.Run(ctx)
|
|
exit := loop.Wait()
|
|
assert.Error(t, exit.ExitReason)
|
|
assert.Contains(t, exit.ExitReason.Error(), "failed to unmarshal checkpoint")
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointSaveError_ReturnsError(t *testing.T) {
|
|
ctx := context.Background()
|
|
modelStarted := make(chan struct{}, 1)
|
|
saveStore := &errorCheckpointStore{setErr: fmt.Errorf("write failed")}
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: saveStore,
|
|
CheckpointID: "cp-1",
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
})
|
|
loop.Push("msg1")
|
|
<-modelStarted
|
|
loop.Stop(WithImmediate())
|
|
exit := loop.Wait()
|
|
assert.Error(t, exit.ExitReason)
|
|
assert.True(t, exit.CheckpointAttempted)
|
|
assert.Error(t, exit.CheckpointErr)
|
|
assert.Contains(t, exit.CheckpointErr.Error(), "write failed")
|
|
}
|
|
|
|
func TestTurnLoop_StaleCheckpointDeletion_OnCleanResume(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "stale-session"
|
|
|
|
loop1 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop1.Push("a")
|
|
loop1.Stop()
|
|
loop1.Run(ctx)
|
|
loop1.Wait()
|
|
|
|
store.mu.Lock()
|
|
_, exists := store.m[cpID]
|
|
store.mu.Unlock()
|
|
assert.True(t, exists, "checkpoint should exist after first loop saves it")
|
|
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
loop2.Push("b")
|
|
loop2.Run(ctx)
|
|
exit2 := loop2.Wait()
|
|
assert.NoError(t, exit2.ExitReason)
|
|
|
|
store.mu.Lock()
|
|
_, exists = store.m[cpID]
|
|
store.mu.Unlock()
|
|
assert.True(t, exists, "checkpoint should still exist because loop2 was stopped and saved a new one")
|
|
}
|
|
|
|
type deletableCheckpointStore struct {
|
|
turnLoopCheckpointStore
|
|
deleteCalled bool
|
|
deletedKey string
|
|
}
|
|
|
|
func (s *deletableCheckpointStore) Delete(_ context.Context, key string) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.deleteCalled = true
|
|
s.deletedKey = key
|
|
delete(s.m, key)
|
|
return nil
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointDeleter_CalledOnContextCancel(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := &deletableCheckpointStore{
|
|
turnLoopCheckpointStore: turnLoopCheckpointStore{m: make(map[string][]byte)},
|
|
}
|
|
cpID := "deleter-session"
|
|
|
|
loop1 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop1.Push("a")
|
|
loop1.Stop()
|
|
loop1.Run(ctx)
|
|
loop1.Wait()
|
|
|
|
store.mu.Lock()
|
|
_, exists := store.m[cpID]
|
|
store.mu.Unlock()
|
|
assert.True(t, exists, "checkpoint saved after loop1")
|
|
|
|
ctx2, cancel2 := context.WithCancel(ctx)
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
cancel2()
|
|
return nil
|
|
},
|
|
})
|
|
loop2.Push("b")
|
|
loop2.Run(ctx2)
|
|
exit2 := loop2.Wait()
|
|
assert.ErrorIs(t, exit2.ExitReason, context.Canceled)
|
|
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
assert.True(t, store.deleteCalled, "CheckPointDeleter.Delete should be called")
|
|
assert.Equal(t, cpID, store.deletedKey)
|
|
_, exists = store.m[cpID]
|
|
assert.False(t, exists, "checkpoint should be removed from store")
|
|
}
|
|
|
|
func TestTurnLoop_GenResumeNil_Error(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "resume-nil-session"
|
|
modelStarted := make(chan struct{}, 1)
|
|
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
loop1 := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
})
|
|
loop1.Push("msg1")
|
|
<-modelStarted
|
|
loop1.Stop(WithImmediate())
|
|
loop1.Wait()
|
|
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop2.Run(ctx)
|
|
exit2 := loop2.Wait()
|
|
assert.Error(t, exit2.ExitReason)
|
|
assert.Contains(t, exit2.ExitReason.Error(), "GenResume is required")
|
|
}
|
|
|
|
func TestTurnLoop_SameCheckpointID_OverwritePattern(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "overwrite-session"
|
|
|
|
loop1 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop1.Push("a")
|
|
loop1.Push("b")
|
|
loop1.Stop()
|
|
loop1.Run(ctx)
|
|
loop1.Wait()
|
|
|
|
store.mu.Lock()
|
|
data1 := append([]byte{}, store.m[cpID]...)
|
|
store.mu.Unlock()
|
|
assert.NotEmpty(t, data1)
|
|
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop2.Push("c")
|
|
loop2.Stop()
|
|
loop2.Run(ctx)
|
|
loop2.Wait()
|
|
|
|
store.mu.Lock()
|
|
data2 := append([]byte{}, store.m[cpID]...)
|
|
store.mu.Unlock()
|
|
assert.NotEmpty(t, data2)
|
|
assert.NotEqual(t, data1, data2, "checkpoint data should change because items are different")
|
|
|
|
var seen []string
|
|
var mu sync.Mutex
|
|
loop3 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
mu.Lock()
|
|
seen = append([]string{}, items...)
|
|
mu.Unlock()
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
loop3.Push("d")
|
|
loop3.Run(ctx)
|
|
exit3 := loop3.Wait()
|
|
assert.NoError(t, exit3.ExitReason)
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
assert.Equal(t, []string{"a", "b", "c", "d"}, seen, "should see loop2's unhandled items (a,b,c from loop2's checkpoint) plus new d")
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointHasRunnerStateButEmptyBytes(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "empty-runner-bytes"
|
|
|
|
cp := &turnLoopCheckpoint[string]{
|
|
HasRunnerState: true,
|
|
RunnerCheckpoint: nil,
|
|
UnhandledItems: []string{"x"},
|
|
}
|
|
data, err := marshalTurnLoopCheckpoint(cp)
|
|
assert.NoError(t, err)
|
|
store.m[cpID] = data
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop.Push("a")
|
|
loop.Run(ctx)
|
|
exit := loop.Wait()
|
|
assert.Error(t, exit.ExitReason)
|
|
assert.Contains(t, exit.ExitReason.Error(), "has runner state but bytes are empty")
|
|
}
|
|
|
|
func TestTurnLoop_GenResumeReturnsError(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "resume-err-session"
|
|
modelStarted := make(chan struct{}, 1)
|
|
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
loop1 := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
})
|
|
loop1.Push("msg1")
|
|
<-modelStarted
|
|
loop1.Stop(WithImmediate())
|
|
loop1.Wait()
|
|
|
|
genResumeErr := fmt.Errorf("resume callback failed")
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
GenResume: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], canceled, unhandled, newItems []string) (*GenResumeResult[string, *schema.Message], error) {
|
|
return nil, genResumeErr
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop2.Run(ctx)
|
|
exit2 := loop2.Wait()
|
|
assert.Error(t, exit2.ExitReason)
|
|
assert.ErrorIs(t, exit2.ExitReason, genResumeErr)
|
|
}
|
|
|
|
func TestTurnLoop_ResumeWaitsForInFlightPushBeforePlanning(t *testing.T) {
|
|
ctx := context.Background()
|
|
resumeErr := errors.New("stop after observing resume inputs")
|
|
strategyEntered := make(chan struct{})
|
|
allowStrategy := make(chan struct{})
|
|
pushDone := make(chan struct{})
|
|
genResumeCalled := make(chan struct{})
|
|
|
|
var resumeNewItems []string
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
GenResume: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], interruptedItems, unhandledItems, newItems []string) (*GenResumeResult[string, *schema.Message], error) {
|
|
resumeNewItems = append([]string{}, newItems...)
|
|
close(genResumeCalled)
|
|
return nil, resumeErr
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop.pendingResume = &turnLoopPendingResume[string]{
|
|
interrupted: []string{"interrupted"},
|
|
newItems: []string{"pre-existing"},
|
|
}
|
|
|
|
go func() {
|
|
defer close(pushDone)
|
|
ok, ack := loop.Push("during-resume", WithPushStrategy(func(ctx context.Context, tc *TurnContext[string, *schema.Message]) []PushOption[string, *schema.Message] {
|
|
close(strategyEntered)
|
|
<-allowStrategy
|
|
return nil
|
|
}))
|
|
assert.True(t, ok)
|
|
assert.Nil(t, ack)
|
|
}()
|
|
|
|
waitOrFail(t, strategyEntered, "strategy did not enter")
|
|
|
|
loop.Run(ctx)
|
|
|
|
select {
|
|
case <-genResumeCalled:
|
|
t.Fatal("GenResume should wait for in-flight PushStrategy to finish")
|
|
default:
|
|
}
|
|
|
|
close(allowStrategy)
|
|
waitOrFail(t, pushDone, "push did not finish")
|
|
|
|
exit := loop.Wait()
|
|
assert.ErrorIs(t, exit.ExitReason, resumeErr)
|
|
assert.Equal(t, []string{"pre-existing", "during-resume"}, resumeNewItems)
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointSaveError_MergesWithExistingError(t *testing.T) {
|
|
ctx := context.Background()
|
|
modelStarted := make(chan struct{}, 1)
|
|
saveStore := &errorCheckpointStore{setErr: fmt.Errorf("disk full")}
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: saveStore,
|
|
CheckpointID: "cp-merge-err",
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
})
|
|
loop.Push("msg1")
|
|
<-modelStarted
|
|
loop.Stop(WithImmediate())
|
|
exit := loop.Wait()
|
|
assert.Error(t, exit.ExitReason)
|
|
var ce *CancelError
|
|
assert.True(t, errors.As(exit.ExitReason, &ce), "ExitReason should be CancelError, not merged with checkpoint error")
|
|
assert.True(t, exit.CheckpointAttempted)
|
|
assert.Error(t, exit.CheckpointErr)
|
|
assert.Contains(t, exit.CheckpointErr.Error(), "disk full")
|
|
}
|
|
|
|
func TestTurnLoop_ResumeWithParams(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "resume-params-session"
|
|
modelStarted := make(chan struct{}, 1)
|
|
|
|
slowModel := &cancelTestChatModel{
|
|
delayNs: int64(500 * time.Millisecond),
|
|
response: &schema.Message{
|
|
Role: schema.Assistant,
|
|
Content: "Hello",
|
|
},
|
|
startedChan: modelStarted,
|
|
doneChan: make(chan struct{}, 1),
|
|
}
|
|
agent, err := NewChatModelAgent(ctx, &ChatModelAgentConfig{
|
|
Name: "TestAgent",
|
|
Description: "Test agent",
|
|
Instruction: "You are a test assistant",
|
|
Model: slowModel,
|
|
})
|
|
assert.NoError(t, err)
|
|
|
|
loop1 := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareAgent(agent),
|
|
})
|
|
loop1.Push("msg1")
|
|
<-modelStarted
|
|
loop1.Stop(WithImmediate())
|
|
exit1 := loop1.Wait()
|
|
var ce *CancelError
|
|
assert.True(t, errors.As(exit1.ExitReason, &ce))
|
|
|
|
var resumeParamsUsed *ResumeParams
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
GenResume: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], canceled, unhandled, newItems []string) (*GenResumeResult[string, *schema.Message], error) {
|
|
params := &ResumeParams{
|
|
Targets: map[string]any{"some-address": "user-data"},
|
|
}
|
|
resumeParamsUsed = params
|
|
return &GenResumeResult[string, *schema.Message]{
|
|
ResumeParams: params,
|
|
Consumed: append(append(canceled, unhandled...), newItems...),
|
|
}, nil
|
|
},
|
|
PrepareAgent: prepareAgent(agent),
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
loop2.Run(ctx)
|
|
exit2 := loop2.Wait()
|
|
assert.NotNil(t, resumeParamsUsed, "GenResume should have been called with ResumeParams")
|
|
assert.Contains(t, resumeParamsUsed.Targets, "some-address")
|
|
_ = exit2
|
|
}
|
|
|
|
func TestTurnLoop_ResumeInterruptAgain_PreservesEnableStreamingCheckpoint(t *testing.T) {
|
|
for _, enableStreaming := range []bool{true, false} {
|
|
t.Run(fmt.Sprintf("enable_streaming_%t", enableStreaming), func(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := fmt.Sprintf("streaming-resume-%t", enableStreaming)
|
|
originalMessage := "msg1"
|
|
|
|
firstAgent := &myAgent{
|
|
runFn: func(ctx context.Context, input *AgentInput, _ ...AgentRunOption) *AsyncIterator[*AgentEvent] {
|
|
assert.Equal(t, enableStreaming, input.EnableStreaming)
|
|
assert.Len(t, input.Messages, 1)
|
|
assert.Equal(t, originalMessage, input.Messages[0].Content)
|
|
|
|
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
|
|
go func() {
|
|
defer gen.Close()
|
|
gen.Send(Interrupt(ctx, "first_interrupt"))
|
|
}()
|
|
return iter
|
|
},
|
|
}
|
|
|
|
loop1 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: func(_ context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{
|
|
Messages: []Message{schema.UserMessage(items[0])},
|
|
EnableStreaming: enableStreaming,
|
|
},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: prepareAgent(firstAgent),
|
|
})
|
|
loop1.Push(originalMessage)
|
|
loop1.Run(ctx)
|
|
exit1 := loop1.Wait()
|
|
require.ErrorAs(t, exit1.ExitReason, new(*InterruptError))
|
|
require.NoError(t, exit1.CheckpointErr)
|
|
|
|
secondAgent := &myAgent{
|
|
resumeFn: func(ctx context.Context, info *ResumeInfo, _ ...AgentRunOption) *AsyncIterator[*AgentEvent] {
|
|
assert.Equal(t, enableStreaming, info.EnableStreaming)
|
|
|
|
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
|
|
go func() {
|
|
defer gen.Close()
|
|
gen.Send(Interrupt(ctx, "second_interrupt"))
|
|
}()
|
|
return iter
|
|
},
|
|
}
|
|
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
GenResume: func(_ context.Context, _ *TurnLoop[string, *schema.Message], interrupted, unhandled, newItems []string) (*GenResumeResult[string, *schema.Message], error) {
|
|
return &GenResumeResult[string, *schema.Message]{
|
|
Consumed: interrupted,
|
|
Remaining: append(append([]string{}, unhandled...), newItems...),
|
|
}, nil
|
|
},
|
|
PrepareAgent: prepareAgent(secondAgent),
|
|
})
|
|
loop2.Run(ctx)
|
|
exit2 := loop2.Wait()
|
|
require.ErrorAs(t, exit2.ExitReason, new(*InterruptError))
|
|
require.NoError(t, exit2.CheckpointErr)
|
|
|
|
// Verify the runner-level checkpoint persisted by the second interrupt
|
|
// still encodes the original streaming mode. This is the invariant the PR
|
|
// fixes: even though loop2's TypedRunner was constructed with the
|
|
// resume-path placeholder (false), runner uses resumeInfo.EnableStreaming
|
|
// from the previous checkpoint when re-saving.
|
|
store.mu.Lock()
|
|
data, ok := store.m[cpID]
|
|
store.mu.Unlock()
|
|
require.True(t, ok)
|
|
cp, err := unmarshalTurnLoopCheckpoint[string](data)
|
|
require.NoError(t, err)
|
|
require.True(t, cp.HasRunnerState)
|
|
_, _, info2, err := runnerLoadCheckPointImpl(newResumeBridgeStore(bridgeCheckpointID, cp.RunnerCheckpoint), context.Background(), bridgeCheckpointID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, enableStreaming, info2.EnableStreaming)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTurnLoop_Stop_EscalatesCancelMode(t *testing.T) {
|
|
ctx := context.Background()
|
|
agentStarted := make(chan *cancelContext, 1)
|
|
probe := &turnLoopStopModeProbeAgent{ccCh: agentStarted}
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return probe, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
cc := <-agentStarted
|
|
|
|
loop.Stop(WithGracefulTimeout(10 * time.Second))
|
|
loop.Stop(WithImmediate())
|
|
|
|
deadline := time.After(1 * time.Second)
|
|
for {
|
|
if cc.getMode() == CancelImmediate {
|
|
break
|
|
}
|
|
select {
|
|
case <-deadline:
|
|
t.Fatal("cancel mode did not escalate to CancelImmediate")
|
|
default:
|
|
}
|
|
time.Sleep(1 * time.Millisecond)
|
|
}
|
|
|
|
exit := loop.Wait()
|
|
var ce *CancelError
|
|
require.True(t, errors.As(exit.ExitReason, &ce))
|
|
assert.Equal(t, CancelImmediate, ce.Info.Mode)
|
|
}
|
|
|
|
func TestTurnLoop_DefaultOnAgentEvents_ErrorPropagation(t *testing.T) {
|
|
agentErr := errors.New("agent execution error")
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
return nil, agentErr
|
|
},
|
|
}, nil
|
|
},
|
|
// No OnAgentEvents — use default handler
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
result := loop.Wait()
|
|
// The default handler should propagate the agent error as ExitReason
|
|
assert.Error(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_OnAgentEventsError(t *testing.T) {
|
|
handlerErr := errors.New("event handler error")
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
// Drain events then return error
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
}
|
|
return handlerErr
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
result := loop.Wait()
|
|
assert.ErrorIs(t, result.ExitReason, handlerErr)
|
|
}
|
|
|
|
func TestTurnLoop_StopCallFromGenInput(t *testing.T) {
|
|
// Test that calling Stop() from within GenInput works correctly
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, loop *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
loop.Stop()
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_PushFromOnAgentEvents(t *testing.T) {
|
|
// Test that calling Push() from within OnAgentEvents works
|
|
pushCount := int32(0)
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeFirst,
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
}
|
|
count := atomic.AddInt32(&pushCount, 1)
|
|
if count == 1 {
|
|
// Push a follow-up item from the callback
|
|
_, _ = tc.Loop.Push("follow-up")
|
|
} else {
|
|
tc.Loop.Stop()
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("initial")
|
|
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.Equal(t, int32(2), atomic.LoadInt32(&pushCount))
|
|
}
|
|
|
|
// Tests for NewTurnLoop: the permissive API where Push, Stop, and Wait are
|
|
// all valid on a not-yet-running loop.
|
|
|
|
func TestNewTurnLoop_PushBeforeRun(t *testing.T) {
|
|
// Items pushed before Run are buffered and processed after Run starts.
|
|
var processedItems []string
|
|
var mu sync.Mutex
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
mu.Lock()
|
|
processedItems = append(processedItems, items...)
|
|
mu.Unlock()
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
// Push before Run — items should be buffered.
|
|
ok, _ := loop.Push("msg1")
|
|
assert.True(t, ok)
|
|
ok, _ = loop.Push("msg2")
|
|
assert.True(t, ok)
|
|
|
|
loop.Run(context.Background())
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.Contains(t, processedItems, "msg1")
|
|
assert.Contains(t, processedItems, "msg2")
|
|
}
|
|
|
|
func TestNewTurnLoop_WaitBeforeRun(t *testing.T) {
|
|
// Wait blocks until Run is called AND the loop exits.
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
waitDone := make(chan *TurnLoopExitState[string, *schema.Message], 1)
|
|
go func() {
|
|
waitDone <- loop.Wait()
|
|
}()
|
|
|
|
// Wait should not return yet since Run hasn't been called.
|
|
select {
|
|
case <-waitDone:
|
|
t.Fatal("Wait returned before Run was called")
|
|
case <-time.After(50 * time.Millisecond):
|
|
// expected
|
|
}
|
|
|
|
loop.Push("msg1")
|
|
loop.Stop()
|
|
loop.Run(context.Background())
|
|
|
|
select {
|
|
case result := <-waitDone:
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.Equal(t, []string{"msg1"}, result.UnhandledItems)
|
|
case <-time.After(1 * time.Second):
|
|
t.Fatal("Wait did not return after Run + Stop")
|
|
}
|
|
}
|
|
|
|
func TestNewTurnLoop_RunIsIdempotent(t *testing.T) {
|
|
var genInputCalls int32
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
atomic.AddInt32(&genInputCalls, 1)
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
loop.Run(context.Background())
|
|
loop.Run(context.Background())
|
|
loop.Run(context.Background())
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.True(t, atomic.LoadInt32(&genInputCalls) >= 1)
|
|
}
|
|
|
|
func TestNewTurnLoop_ConcurrentPushAndRun(t *testing.T) {
|
|
// Concurrent Push and Run should not race.
|
|
for i := 0; i < 100; i++ {
|
|
var count int32
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
atomic.AddInt32(&count, int32(len(items)))
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{name: "test"}, nil
|
|
},
|
|
})
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
_, _ = loop.Push("item")
|
|
}()
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
loop.Run(context.Background())
|
|
}()
|
|
|
|
wg.Wait()
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
|
|
processed := atomic.LoadInt32(&count)
|
|
unhandled := len(result.UnhandledItems)
|
|
assert.True(t, int(processed)+unhandled <= 1,
|
|
"total should not exceed pushed amount")
|
|
}
|
|
}
|
|
|
|
type turnCtxKey struct{}
|
|
|
|
func TestTurnLoop_RunCtx_Propagation(t *testing.T) {
|
|
// Verify that GenInputResult.RunCtx is propagated to PrepareAgent,
|
|
// the agent run, and OnAgentEvents.
|
|
|
|
const traceVal = "trace-123"
|
|
var prepareCtxVal, agentCtxVal, eventsCtxVal string
|
|
|
|
cfg := TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, loop *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
// Derive a new context with per-item trace data
|
|
runCtx := context.WithValue(ctx, turnCtxKey{}, traceVal)
|
|
return &GenInputResult[string, *schema.Message]{
|
|
RunCtx: runCtx,
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, loop *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
if v, ok := ctx.Value(turnCtxKey{}).(string); ok {
|
|
prepareCtxVal = v
|
|
}
|
|
return &turnLoopMockAgent{
|
|
name: "trace-agent",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
if v, ok := ctx.Value(turnCtxKey{}).(string); ok {
|
|
agentCtxVal = v
|
|
}
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
if v, ok := ctx.Value(turnCtxKey{}).(string); ok {
|
|
eventsCtxVal = v
|
|
}
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
tc.Loop.Stop()
|
|
return nil
|
|
},
|
|
}
|
|
|
|
loop := NewTurnLoop(cfg)
|
|
loop.Push("hello")
|
|
loop.Run(context.Background())
|
|
result := loop.Wait()
|
|
|
|
assert.Nil(t, result.ExitReason)
|
|
assert.Equal(t, traceVal, prepareCtxVal, "PrepareAgent should receive RunCtx")
|
|
assert.Equal(t, traceVal, agentCtxVal, "Agent run should receive RunCtx")
|
|
assert.Equal(t, traceVal, eventsCtxVal, "OnAgentEvents should receive RunCtx")
|
|
}
|
|
|
|
func TestTurnLoop_TurnContext_PreemptedChannel(t *testing.T) {
|
|
preemptedSeen := make(chan struct{})
|
|
agentStarted := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "slow",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
close(agentStarted)
|
|
select {
|
|
case <-tc.Preempted:
|
|
close(preemptedSeen)
|
|
case <-time.After(5 * time.Second):
|
|
t.Error("timed out waiting for Preempted channel")
|
|
}
|
|
// Drain events
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
loop.Push("msg2", WithPreemptTimeout[string, *schema.Message](AnySafePoint, time.Millisecond))
|
|
|
|
select {
|
|
case <-preemptedSeen:
|
|
// success
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("preempted channel was never observed in OnAgentEvents")
|
|
}
|
|
|
|
loop.Stop()
|
|
loop.Wait()
|
|
}
|
|
|
|
// =============================================================================
|
|
// preemptController unit tests
|
|
// =============================================================================
|
|
|
|
func requireAckClosed(t *testing.T, ack <-chan struct{}) {
|
|
t.Helper()
|
|
waitOrFail(t, ack, "ack should be closed")
|
|
}
|
|
|
|
func requireAckOpen(t *testing.T, ack <-chan struct{}) {
|
|
t.Helper()
|
|
select {
|
|
case <-ack:
|
|
t.Fatal("ack should still be open")
|
|
default:
|
|
}
|
|
}
|
|
|
|
func requirePreemptPhase(t *testing.T, c *preemptController, phase preemptTurnPhase) {
|
|
t.Helper()
|
|
require.Eventually(t, func() bool {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.turnPhase == phase
|
|
}, time.Second, time.Millisecond)
|
|
}
|
|
|
|
func TestPreemptController_BeginPushSnapshotsPlanningTurn(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
assert.True(t, snapshot.hasTargetTurn)
|
|
assert.NotZero(t, snapshot.turnID)
|
|
assert.Nil(t, snapshot.ctx)
|
|
assert.Nil(t, snapshot.tc)
|
|
}
|
|
|
|
type testContextKey struct{}
|
|
|
|
func TestPreemptController_BeginPushSnapshotsActiveTurn(t *testing.T) {
|
|
c := newPreemptController()
|
|
ctx := context.WithValue(context.Background(), testContextKey{}, "value")
|
|
tc := "turn-context"
|
|
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(ctx, tc)
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
assert.True(t, snapshot.hasTargetTurn)
|
|
assert.NotZero(t, snapshot.turnID)
|
|
assert.Equal(t, ctx, snapshot.ctx)
|
|
assert.Equal(t, tc, snapshot.tc)
|
|
}
|
|
|
|
func TestPreemptController_RequestPreemptIdleTurnAcksImmediately(t *testing.T) {
|
|
c := newPreemptController()
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
|
|
requireAckClosed(t, ack)
|
|
_, ok := c.receivePreempt()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestPreemptController_RejectsInvalidTurnPhaseTransitions(t *testing.T) {
|
|
c := newPreemptController()
|
|
|
|
assert.PanicsWithValue(t, "adk: preemptController.beginActiveTurn called while turn phase is idle; expected planning", func() {
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
})
|
|
assert.PanicsWithValue(t, "adk: preemptController.abortPlanningTurn called while turn phase is idle; expected planning", func() {
|
|
c.abortPlanningTurn()
|
|
})
|
|
assert.PanicsWithValue(t, "adk: preemptController.endActiveTurn called while turn phase is idle; expected active", func() {
|
|
c.endActiveTurn()
|
|
})
|
|
|
|
c.beginPlanningTurn()
|
|
assert.PanicsWithValue(t, "adk: preemptController.beginPlanningTurn called while turn phase is planning; expected idle", func() {
|
|
c.beginPlanningTurn()
|
|
})
|
|
c.abortPlanningTurn()
|
|
|
|
c.pending = newPreemptRequest(nil, nil, time.Now())
|
|
assert.PanicsWithValue(t, "adk: preemptController.beginPlanningTurn called with stale pending preempt request", func() {
|
|
c.beginPlanningTurn()
|
|
})
|
|
}
|
|
|
|
func TestPreemptController_RequestPreemptForPlanningTurnIsConsumedAfterActivation(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
|
|
_, ok := c.receivePreempt()
|
|
assert.False(t, ok)
|
|
requireAckOpen(t, ack)
|
|
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
req, ok := c.receivePreempt()
|
|
require.True(t, ok)
|
|
requireAckOpen(t, ack)
|
|
|
|
req.ack()
|
|
requireAckClosed(t, ack)
|
|
}
|
|
|
|
func TestPreemptController_RequestPreemptForActiveTurnIsConsumedOnce(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
|
|
req, ok := c.receivePreempt()
|
|
require.True(t, ok)
|
|
opts := req.cancelOptions(time.Now())
|
|
cfg := parseAgentCancelOptions(opts...)
|
|
assert.Equal(t, CancelAfterChatModel, cfg.Mode)
|
|
requireAckOpen(t, ack)
|
|
|
|
_, ok = c.receivePreempt()
|
|
assert.False(t, ok)
|
|
|
|
req.ack()
|
|
requireAckClosed(t, ack)
|
|
}
|
|
|
|
func TestPreemptController_AbortPlanningTurnAcksUnconsumedRequest(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
|
|
req := c.abortPlanningTurn()
|
|
require.NotNil(t, req)
|
|
req.ack()
|
|
requireAckClosed(t, ack)
|
|
|
|
_, ok := c.receivePreempt()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestPreemptController_EndActiveTurnAcksUnconsumedRequest(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
|
|
req := c.endActiveTurn()
|
|
require.NotNil(t, req)
|
|
req.ack()
|
|
requireAckClosed(t, ack)
|
|
|
|
_, ok := c.receivePreempt()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestPreemptController_EndActiveTurnDoesNotAckConsumedRequest(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
req, ok := c.receivePreempt()
|
|
require.True(t, ok)
|
|
|
|
assert.Nil(t, c.endActiveTurn())
|
|
requireAckOpen(t, ack)
|
|
|
|
req.ack()
|
|
requireAckClosed(t, ack)
|
|
}
|
|
|
|
func TestPreemptController_TargetTurnMismatchAcksImmediately(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
c.abortPlanningTurn()
|
|
c.beginPlanningTurn()
|
|
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
|
|
requireAckClosed(t, ack)
|
|
_, ok := c.receivePreempt()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestPreemptController_WaitForPushesBlocksUntilPushEnds(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPush()
|
|
|
|
waitDone := make(chan struct{})
|
|
go func() {
|
|
c.waitForPushes()
|
|
close(waitDone)
|
|
}()
|
|
|
|
select {
|
|
case <-waitDone:
|
|
t.Fatal("waitForPushes should block while a push is in flight")
|
|
default:
|
|
}
|
|
|
|
c.endPush()
|
|
waitOrFail(t, waitDone, "waitForPushes should unblock after endPush")
|
|
}
|
|
|
|
func TestPreemptController_MultipleRequestsBeforeReceiveMergeAcksAndUseMergedOpts(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
ack1 := make(chan struct{})
|
|
ack2 := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack1, WithAgentCancelMode(CancelAfterChatModel), WithAgentCancelTimeout(time.Minute))
|
|
c.requestPreempt(snapshot, ack2, WithAgentCancelMode(CancelAfterToolCalls), WithRecursive(), WithAgentCancelTimeout(time.Second))
|
|
|
|
req, ok := c.receivePreempt()
|
|
require.True(t, ok)
|
|
opts := req.cancelOptions(time.Now())
|
|
cfg := parseAgentCancelOptions(opts...)
|
|
assert.Equal(t, CancelAfterChatModel|CancelAfterToolCalls, cfg.Mode)
|
|
assert.True(t, cfg.Recursive)
|
|
require.NotNil(t, cfg.Timeout)
|
|
assert.LessOrEqual(t, *cfg.Timeout, time.Second)
|
|
|
|
requireAckOpen(t, ack1)
|
|
requireAckOpen(t, ack2)
|
|
req.ack()
|
|
requireAckClosed(t, ack1)
|
|
requireAckClosed(t, ack2)
|
|
}
|
|
|
|
func TestPreemptController_ConcurrentPreemptRequestsMergeAndAck(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
const requestCount = 50
|
|
start := make(chan struct{})
|
|
acks := make([]chan struct{}, requestCount)
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < requestCount; i++ {
|
|
acks[i] = make(chan struct{})
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
<-start
|
|
c.requestPreempt(snapshot, acks[i], WithAgentCancelMode(CancelAfterChatModel))
|
|
}(i)
|
|
}
|
|
|
|
close(start)
|
|
wg.Wait()
|
|
|
|
req, ok := c.receivePreempt()
|
|
require.True(t, ok)
|
|
req.ack()
|
|
for _, ack := range acks {
|
|
requireAckClosed(t, ack)
|
|
}
|
|
}
|
|
|
|
func TestPreemptController_CloseForLoopExitDuringDelayedPreempt(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
|
|
// Capture a snapshot while the turn is active.
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
// closeForLoopExit tears down controller state during TurnLoop cleanup.
|
|
c.closeForLoopExit()
|
|
|
|
// A delayed goroutine fires requestPreempt AFTER closeForLoopExit.
|
|
// This must not panic or deadlock; ack should be closed immediately
|
|
// because the controller is now closed.
|
|
ack := make(chan struct{})
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
}()
|
|
|
|
waitOrFail(t, done, "requestPreempt after closeForLoopExit must not deadlock")
|
|
requireAckClosed(t, ack)
|
|
}
|
|
|
|
func TestPreemptController_RequestPreemptAfterCloseForLoopExit(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
c.closeForLoopExit()
|
|
|
|
// requestPreempt on a closed controller should close ack immediately.
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterToolCalls))
|
|
requireAckClosed(t, ack)
|
|
|
|
// receivePreempt should return nothing.
|
|
_, ok := c.receivePreempt()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestPreemptController_ConcurrentBeginPushAndWaitForPushes(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
|
|
const pushCount = 100
|
|
var wg sync.WaitGroup
|
|
|
|
// Launch many goroutines doing beginPush / endPush concurrently.
|
|
for i := 0; i < pushCount; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_ = c.beginPush()
|
|
// Simulate some work.
|
|
time.Sleep(time.Microsecond)
|
|
c.endPush()
|
|
}()
|
|
}
|
|
|
|
// Meanwhile, waitForPushes should eventually return once all are done.
|
|
waitDone := make(chan struct{})
|
|
go func() {
|
|
c.waitForPushes()
|
|
close(waitDone)
|
|
}()
|
|
|
|
wg.Wait() // all pushes complete
|
|
|
|
waitOrFail(t, waitDone, "waitForPushes deadlocked with concurrent beginPush/endPush")
|
|
}
|
|
|
|
func TestPreemptController_RequestPreemptWithNilAck(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
// requestPreempt with nil ack channel must not panic.
|
|
assert.NotPanics(t, func() {
|
|
c.requestPreempt(snapshot, nil, WithAgentCancelMode(CancelAfterChatModel))
|
|
})
|
|
|
|
// The request should still be stored and consumable.
|
|
req, ok := c.receivePreempt()
|
|
require.True(t, ok)
|
|
// ack() with nil channels in the list must not panic.
|
|
assert.NotPanics(t, func() {
|
|
req.ack()
|
|
})
|
|
}
|
|
|
|
func TestPreemptController_MergeImmediateOverridesTimeout(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
// First request: graceful with timeout.
|
|
ack1 := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack1,
|
|
WithAgentCancelMode(CancelAfterToolCalls),
|
|
WithAgentCancelTimeout(10*time.Second))
|
|
|
|
// Second request: CancelImmediate (no timeout).
|
|
ack2 := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack2, WithAgentCancelMode(CancelImmediate))
|
|
|
|
req, ok := c.receivePreempt()
|
|
require.True(t, ok)
|
|
|
|
opts := req.cancelOptions(time.Now())
|
|
cfg := parseAgentCancelOptions(opts...)
|
|
|
|
// CancelImmediate should win and timeout should be nil.
|
|
assert.Equal(t, CancelImmediate, cfg.Mode)
|
|
assert.Nil(t, cfg.Timeout, "CancelImmediate merge should clear timeout")
|
|
|
|
req.ack()
|
|
requireAckClosed(t, ack1)
|
|
requireAckClosed(t, ack2)
|
|
}
|
|
|
|
func TestPreemptController_DelayedPreemptTargetGoneBetweenTurns(t *testing.T) {
|
|
c := newPreemptController()
|
|
|
|
// Turn 1: planning → active → end
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc1")
|
|
oldSnapshot := c.beginPush()
|
|
c.endPush()
|
|
req := c.endActiveTurn()
|
|
assert.Nil(t, req) // no pending request
|
|
|
|
// Turn 2: start a new turn
|
|
c.beginPlanningTurn()
|
|
c.beginActiveTurn(context.Background(), "tc2")
|
|
|
|
// A delayed preempt from Turn 1 fires with stale snapshot.
|
|
// It should resolve as no-op (ack immediately) because turnID doesn't match.
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(oldSnapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
requireAckClosed(t, ack)
|
|
|
|
// The new turn should have no pending preempt.
|
|
_, ok := c.receivePreempt()
|
|
assert.False(t, ok, "stale preempt must not affect new turn")
|
|
|
|
c.endActiveTurn()
|
|
}
|
|
|
|
func TestPreemptController_EndPushWithoutBeginPushPanics(t *testing.T) {
|
|
c := newPreemptController()
|
|
|
|
// endPush without a matching beginPush should panic with the new invariant.
|
|
assert.PanicsWithValue(t,
|
|
"adk: preemptController.endPush called without matching beginPush",
|
|
func() {
|
|
c.endPush()
|
|
},
|
|
)
|
|
}
|
|
|
|
func TestPreemptController_BeginActiveTurnNotifiesExistingPending(t *testing.T) {
|
|
c := newPreemptController()
|
|
c.beginPlanningTurn()
|
|
snapshot := c.beginPush()
|
|
c.endPush()
|
|
|
|
// Send a preempt during planning phase.
|
|
ack := make(chan struct{})
|
|
c.requestPreempt(snapshot, ack, WithAgentCancelMode(CancelAfterChatModel))
|
|
|
|
// During planning, receivePreempt returns nothing.
|
|
_, ok := c.receivePreempt()
|
|
assert.False(t, ok)
|
|
|
|
// beginActiveTurn should notify the watcher via the notify channel.
|
|
c.beginActiveTurn(context.Background(), "tc")
|
|
|
|
// The notify channel should have a message.
|
|
select {
|
|
case <-c.notify:
|
|
// Expected: watcher notification was sent.
|
|
case <-time.After(1 * time.Second):
|
|
t.Fatal("beginActiveTurn should notify watcher when there is a pending request")
|
|
}
|
|
|
|
// Now receivePreempt should return the pending request.
|
|
req, ok := c.receivePreempt()
|
|
require.True(t, ok)
|
|
req.ack()
|
|
requireAckClosed(t, ack)
|
|
}
|
|
|
|
// =============================================================================
|
|
// Integration tests for race-prone preempt scenarios
|
|
// =============================================================================
|
|
|
|
func TestTurnLoop_ConcurrentPreemptsDuringTurn(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentStartedOnce := sync.Once{}
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
agentStartedOnce.Do(func() {
|
|
close(agentStarted)
|
|
})
|
|
<-ctx.Done()
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
var genInputCount int32
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
atomic.AddInt32(&genInputCount, 1)
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("seed")
|
|
|
|
waitOrFail(t, agentStarted, "agent did not start")
|
|
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 10; i++ {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
ok, ack := loop.Push(fmt.Sprintf("urgent-%d", i), WithPreemptTimeout[string, *schema.Message](AnySafePoint, 10*time.Millisecond))
|
|
if ok && ack != nil {
|
|
select {
|
|
case <-ack:
|
|
case <-time.After(5 * time.Second):
|
|
t.Error("ack channel not closed within timeout")
|
|
}
|
|
}
|
|
}(i)
|
|
}
|
|
|
|
// Stop the loop concurrently. The run loop may be blocked on
|
|
// buffer.Receive after processing all preempts; Stop unblocks it
|
|
// and triggers closeForLoopExit which closes any orphaned ack channels.
|
|
go func() {
|
|
time.Sleep(500 * time.Millisecond)
|
|
loop.Stop(WithImmediate())
|
|
}()
|
|
|
|
wg.Wait()
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.True(t, atomic.LoadInt32(&genInputCount) >= 2, "should have had at least the initial turn + one preempted turn")
|
|
}
|
|
|
|
func TestTurnLoop_PreemptBetweenTurnsAcksImmediately(t *testing.T) {
|
|
var cancelCount int32
|
|
var turnCount int32
|
|
firstTurnDone := make(chan struct{})
|
|
secondTurnDone := make(chan struct{})
|
|
firstTurnOnce := sync.Once{}
|
|
secondTurnOnce := sync.Once{}
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "fast",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
return &AgentOutput{}, nil
|
|
},
|
|
onCancel: func(cc *cancelContext) {
|
|
atomic.AddInt32(&cancelCount, 1)
|
|
},
|
|
}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
atomic.AddInt32(&turnCount, 1)
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
switch atomic.LoadInt32(&turnCount) {
|
|
case 1:
|
|
firstTurnOnce.Do(func() { close(firstTurnDone) })
|
|
case 2:
|
|
secondTurnOnce.Do(func() { close(secondTurnDone) })
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("first")
|
|
waitOrFail(t, firstTurnDone, "first turn did not complete")
|
|
requirePreemptPhase(t, loop.preemptCtrl, preemptTurnIdle)
|
|
|
|
ok, ack := loop.Push("between-turns", WithPreempt[string, *schema.Message](AnySafePoint))
|
|
require.True(t, ok)
|
|
require.NotNil(t, ack)
|
|
requireAckClosed(t, ack)
|
|
assert.Equal(t, int32(0), atomic.LoadInt32(&cancelCount), "between-turn preempt must not submit cancel")
|
|
|
|
waitOrFail(t, secondTurnDone, "between-turn item was not processed")
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.Equal(t, int32(2), atomic.LoadInt32(&turnCount))
|
|
}
|
|
|
|
func TestTurnLoop_PushStrategy_DuringTurnTransition(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentStartedOnce := sync.Once{}
|
|
allowFinish := make(chan struct{})
|
|
strategyEntered := make(chan struct{})
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
agentStartedOnce.Do(func() {
|
|
close(agentStarted)
|
|
})
|
|
select {
|
|
case <-allowFinish:
|
|
return &AgentOutput{}, nil
|
|
case <-ctx.Done():
|
|
return &AgentOutput{}, nil
|
|
}
|
|
},
|
|
}
|
|
|
|
var genInputCount int32
|
|
secondTurnDone := make(chan struct{})
|
|
secondTurnOnce := sync.Once{}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
count := atomic.AddInt32(&genInputCount, 1)
|
|
if count >= 2 {
|
|
secondTurnOnce.Do(func() {
|
|
close(secondTurnDone)
|
|
})
|
|
}
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("first")
|
|
|
|
waitOrFail(t, agentStarted, "agent did not start")
|
|
|
|
strategyBlocker := make(chan struct{})
|
|
var strategyTCNotNil int32
|
|
|
|
go func() {
|
|
loop.Push("strategic-item", WithPushStrategy(func(ctx context.Context, tc *TurnContext[string, *schema.Message]) []PushOption[string, *schema.Message] {
|
|
if tc != nil {
|
|
atomic.StoreInt32(&strategyTCNotNil, 1)
|
|
}
|
|
close(strategyEntered)
|
|
<-strategyBlocker
|
|
return []PushOption[string, *schema.Message]{WithPreempt[string, *schema.Message](AnySafePoint)}
|
|
}))
|
|
}()
|
|
|
|
waitOrFail(t, strategyEntered, "strategy did not enter")
|
|
|
|
close(allowFinish)
|
|
|
|
select {
|
|
case <-secondTurnDone:
|
|
t.Fatal("second turn should not be planned before strategy Push finishes")
|
|
default:
|
|
}
|
|
|
|
close(strategyBlocker)
|
|
|
|
waitOrFail(t, secondTurnDone, "second turn should eventually run after strategy resolves")
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.True(t, atomic.LoadInt32(&genInputCount) >= 2)
|
|
assert.Equal(t, int32(1), atomic.LoadInt32(&strategyTCNotNil))
|
|
}
|
|
|
|
func TestTurnLoop_ConcurrentPreemptAndStop(t *testing.T) {
|
|
for iter := 0; iter < 20; iter++ {
|
|
t.Run(fmt.Sprintf("iter_%d", iter), func(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
agentStarted := make(chan struct{})
|
|
agentStartedOnce := sync.Once{}
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
agentStartedOnce.Do(func() {
|
|
close(agentStarted)
|
|
})
|
|
<-ctx.Done()
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return agent, nil
|
|
},
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("seed")
|
|
|
|
select {
|
|
case <-agentStarted:
|
|
case <-time.After(1 * time.Second):
|
|
t.Fatal("agent did not start")
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
_, ack := loop.Push("preempt-item", WithPreempt[string, *schema.Message](AnySafePoint))
|
|
if ack != nil {
|
|
<-ack
|
|
}
|
|
}()
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
loop.Stop(WithImmediate())
|
|
}()
|
|
|
|
wg.Wait()
|
|
loop.Wait()
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTurnLoop_ConcurrentPushStrategyAndStop(t *testing.T) {
|
|
for iter := 0; iter < 20; iter++ {
|
|
t.Run(fmt.Sprintf("iter_%d", iter), func(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
agentStarted := make(chan struct{})
|
|
agentStartedOnce := sync.Once{}
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
agentStartedOnce.Do(func() {
|
|
close(agentStarted)
|
|
})
|
|
<-ctx.Done()
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return agent, nil
|
|
},
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("seed")
|
|
|
|
select {
|
|
case <-agentStarted:
|
|
case <-time.After(1 * time.Second):
|
|
t.Fatal("agent did not start")
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
_, ack := loop.Push("strategic-item", WithPushStrategy(func(ctx context.Context, tc *TurnContext[string, *schema.Message]) []PushOption[string, *schema.Message] {
|
|
return []PushOption[string, *schema.Message]{WithPreempt[string, *schema.Message](AnySafePoint)}
|
|
}))
|
|
if ack != nil {
|
|
<-ack
|
|
}
|
|
}()
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
loop.Stop(WithImmediate())
|
|
}()
|
|
|
|
wg.Wait()
|
|
loop.Wait()
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTurnLoop_TurnContext_StoppedChannel(t *testing.T) {
|
|
stoppedSeen := make(chan struct{})
|
|
agentStarted := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "slow",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
close(agentStarted)
|
|
select {
|
|
case <-tc.Stopped:
|
|
close(stoppedSeen)
|
|
case <-time.After(5 * time.Second):
|
|
t.Error("timed out waiting for Stopped channel")
|
|
}
|
|
// Drain events
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
loop.Stop(WithImmediate())
|
|
|
|
select {
|
|
case <-stoppedSeen:
|
|
// success
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("stopped channel was never observed in OnAgentEvents")
|
|
}
|
|
|
|
loop.Wait()
|
|
}
|
|
|
|
func TestTurnLoop_TurnContext_BothPreemptedAndStopped(t *testing.T) {
|
|
t.Run("PreemptThenStop_OnlyPreemptContributes", func(t *testing.T) {
|
|
preemptedSeen := make(chan struct{})
|
|
agentStarted := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "slow",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*TypedAgentEvent[*schema.Message]]) error {
|
|
close(agentStarted)
|
|
select {
|
|
case <-tc.Preempted:
|
|
close(preemptedSeen)
|
|
case <-time.After(5 * time.Second):
|
|
t.Error("timed out waiting for Preempted")
|
|
}
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
loop.Push("msg2", WithPreemptTimeout[string, *schema.Message](AnySafePoint, time.Millisecond))
|
|
|
|
select {
|
|
case <-preemptedSeen:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("Preempted channel was never closed")
|
|
}
|
|
|
|
loop.Stop(WithImmediate())
|
|
loop.Wait()
|
|
})
|
|
|
|
t.Run("StopThenPreempt_OnlyStopContributes", func(t *testing.T) {
|
|
stoppedSeen := make(chan struct{})
|
|
agentStarted := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "slow",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*TypedAgentEvent[*schema.Message]]) error {
|
|
close(agentStarted)
|
|
select {
|
|
case <-tc.Stopped:
|
|
close(stoppedSeen)
|
|
case <-time.After(5 * time.Second):
|
|
t.Error("timed out waiting for Stopped")
|
|
}
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
loop.Stop(WithImmediate())
|
|
|
|
select {
|
|
case <-stoppedSeen:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("Stopped channel was never closed")
|
|
}
|
|
|
|
loop.Push("msg2", WithPreemptTimeout[string, *schema.Message](AnySafePoint, time.Millisecond))
|
|
loop.Wait()
|
|
})
|
|
}
|
|
|
|
func TestTurnLoop_PushStrategy_DuringTurn(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentStartedOnce := sync.Once{}
|
|
agentCancelled := make(chan struct{})
|
|
agentCancelledOnce := sync.Once{}
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
agentStartedOnce.Do(func() {
|
|
close(agentStarted)
|
|
})
|
|
<-ctx.Done()
|
|
agentCancelledOnce.Do(func() {
|
|
close(agentCancelled)
|
|
})
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
genInputCalls := int32(0)
|
|
secondGenInputCalled := make(chan struct{})
|
|
secondGenInputOnce := sync.Once{}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
count := atomic.AddInt32(&genInputCalls, 1)
|
|
if count >= 2 {
|
|
secondGenInputOnce.Do(func() {
|
|
close(secondGenInputCalled)
|
|
})
|
|
}
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: []string{items[0]},
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("first")
|
|
|
|
waitOrFail(t, agentStarted, "agent did not start")
|
|
|
|
// Strategy inspects TurnContext during a running turn and decides to preempt.
|
|
var strategyCalled int32
|
|
var strategyTC *TurnContext[string, *schema.Message]
|
|
loop.Push("urgent", WithPushStrategy(func(ctx context.Context, tc *TurnContext[string, *schema.Message]) []PushOption[string, *schema.Message] {
|
|
atomic.AddInt32(&strategyCalled, 1)
|
|
strategyTC = tc
|
|
return []PushOption[string, *schema.Message]{WithPreempt[string, *schema.Message](AnySafePoint)}
|
|
}))
|
|
|
|
waitOrFail(t, agentCancelled, "agent was not cancelled by strategy-returned preempt")
|
|
|
|
waitOrFail(t, secondGenInputCalled, "second GenInput was not called after preempt")
|
|
|
|
loop.Stop(WithImmediate())
|
|
loop.Wait()
|
|
|
|
assert.Equal(t, int32(1), atomic.LoadInt32(&strategyCalled))
|
|
assert.NotNil(t, strategyTC, "strategy should receive non-nil TurnContext during a turn")
|
|
assert.Equal(t, []string{"first"}, strategyTC.Consumed)
|
|
}
|
|
|
|
func TestTurnLoop_PushStrategy_BetweenTurns(t *testing.T) {
|
|
// Push with strategy before Run() — TurnContext should be nil.
|
|
var strategyCalled int32
|
|
var strategyTCWasNil bool
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
agentDone := make(chan struct{})
|
|
agentDoneOnce := sync.Once{}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
Remaining: nil,
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
}
|
|
agentDoneOnce.Do(func() {
|
|
close(agentDone)
|
|
})
|
|
return nil
|
|
},
|
|
})
|
|
|
|
// Push with strategy — no turn is active yet, so tc should be nil.
|
|
loop.Push("item", WithPushStrategy(func(ctx context.Context, tc *TurnContext[string, *schema.Message]) []PushOption[string, *schema.Message] {
|
|
atomic.AddInt32(&strategyCalled, 1)
|
|
strategyTCWasNil = tc == nil
|
|
return nil // plain push, no preempt
|
|
}))
|
|
|
|
waitOrFail(t, agentDone, "agent did not complete")
|
|
|
|
loop.Stop()
|
|
loop.Wait()
|
|
|
|
assert.Equal(t, int32(1), atomic.LoadInt32(&strategyCalled))
|
|
assert.True(t, strategyTCWasNil, "strategy should receive nil TurnContext between turns")
|
|
}
|
|
|
|
func TestTurnLoop_PushStrategy_OverridesOtherOptions(t *testing.T) {
|
|
// Push with both WithPreempt and WithPushStrategy — only strategy's result applies.
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
agentDone := make(chan struct{})
|
|
agentDoneOnce := sync.Once{}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
Remaining: nil,
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
}
|
|
agentDoneOnce.Do(func() {
|
|
close(agentDone)
|
|
})
|
|
return nil
|
|
},
|
|
})
|
|
|
|
// Strategy returns nil (no preempt), even though WithPreempt is also passed.
|
|
// The strategy should override — so the agent should NOT be preempted.
|
|
ok, ack := loop.Push("item", WithPreempt[string, *schema.Message](AnySafePoint), WithPushStrategy(func(ctx context.Context, tc *TurnContext[string, *schema.Message]) []PushOption[string, *schema.Message] {
|
|
return nil // no preempt
|
|
}))
|
|
assert.True(t, ok)
|
|
assert.Nil(t, ack, "ack should be nil since strategy returned no preempt")
|
|
|
|
waitOrFail(t, agentDone, "agent did not complete normally")
|
|
|
|
loop.Stop()
|
|
loop.Wait()
|
|
}
|
|
|
|
func TestTurnLoop_PushStrategy_NestedStrategyStripped(t *testing.T) {
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
agentDone := make(chan struct{})
|
|
agentDoneOnce := sync.Once{}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
Remaining: nil,
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
_, ok := events.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
}
|
|
agentDoneOnce.Do(func() {
|
|
close(agentDone)
|
|
})
|
|
return nil
|
|
},
|
|
})
|
|
|
|
// Strategy returns another WithPushStrategy — the nested one should be stripped.
|
|
innerCalled := int32(0)
|
|
ok, ack := loop.Push("item", WithPushStrategy(func(ctx context.Context, tc *TurnContext[string, *schema.Message]) []PushOption[string, *schema.Message] {
|
|
return []PushOption[string, *schema.Message]{
|
|
WithPushStrategy(func(ctx context.Context, tc *TurnContext[string, *schema.Message]) []PushOption[string, *schema.Message] {
|
|
atomic.AddInt32(&innerCalled, 1)
|
|
return []PushOption[string, *schema.Message]{WithPreempt[string, *schema.Message](AnySafePoint)}
|
|
}),
|
|
}
|
|
}))
|
|
assert.True(t, ok)
|
|
assert.Nil(t, ack, "ack should be nil since nested strategy was stripped (no preempt)")
|
|
|
|
waitOrFail(t, agentDone, "agent did not complete normally")
|
|
|
|
loop.Stop()
|
|
loop.Wait()
|
|
|
|
assert.Equal(t, int32(0), atomic.LoadInt32(&innerCalled), "nested strategy should not be called")
|
|
}
|
|
|
|
func TestTurnLoop_PushStrategy_ConsumedInspection(t *testing.T) {
|
|
// Strategy preempts only when current turn is processing "low-priority" items.
|
|
agentStarted := make(chan struct{})
|
|
agentStartedOnce := sync.Once{}
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
agentStartedOnce.Do(func() {
|
|
close(agentStarted)
|
|
})
|
|
<-ctx.Done()
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
genInputCalls := int32(0)
|
|
secondGenInputItems := make(chan []string, 1)
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
count := atomic.AddInt32(&genInputCalls, 1)
|
|
if count >= 2 {
|
|
select {
|
|
case secondGenInputItems <- append([]string{}, items...):
|
|
default:
|
|
}
|
|
}
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: []string{items[0]},
|
|
Remaining: items[1:],
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("low-priority-task")
|
|
|
|
waitOrFail(t, agentStarted, "agent did not start")
|
|
|
|
// Strategy checks Consumed and preempts because current turn has "low-priority" items.
|
|
loop.Push("urgent-task", WithPushStrategy(func(ctx context.Context, tc *TurnContext[string, *schema.Message]) []PushOption[string, *schema.Message] {
|
|
if tc != nil && len(tc.Consumed) > 0 && tc.Consumed[0] == "low-priority-task" {
|
|
return []PushOption[string, *schema.Message]{WithPreempt[string, *schema.Message](AnySafePoint)}
|
|
}
|
|
return nil
|
|
}))
|
|
|
|
select {
|
|
case items := <-secondGenInputItems:
|
|
assert.Contains(t, items, "urgent-task")
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("second GenInput was not called after strategy-driven preempt")
|
|
}
|
|
|
|
loop.Stop(WithImmediate())
|
|
loop.Wait()
|
|
}
|
|
|
|
func TestTurnLoop_PushAfterStop_BufferedAsLateItems(t *testing.T) {
|
|
ctx := context.Background()
|
|
processed := make(chan string, 10)
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
processed <- tc.Consumed[0]
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-processed
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
|
|
// Push after stop — should be buffered as late items
|
|
ok1, _ := loop.Push("late1")
|
|
ok2, _ := loop.Push("late2")
|
|
ok3, _ := loop.Push("late3")
|
|
assert.False(t, ok1)
|
|
assert.False(t, ok2)
|
|
assert.False(t, ok3)
|
|
|
|
late := result.TakeLateItems()
|
|
assert.Equal(t, []string{"late1", "late2", "late3"}, late)
|
|
}
|
|
|
|
func TestTurnLoop_TakeLateItems_Idempotent(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop.Push("a")
|
|
loop.Stop()
|
|
loop.Run(ctx)
|
|
result := loop.Wait()
|
|
|
|
loop.Push("late1")
|
|
|
|
first := result.TakeLateItems()
|
|
second := result.TakeLateItems()
|
|
third := result.TakeLateItems()
|
|
|
|
assert.Equal(t, []string{"late1"}, first)
|
|
assert.Equal(t, first, second, "subsequent calls should return the same slice")
|
|
assert.Equal(t, first, third, "subsequent calls should return the same slice")
|
|
}
|
|
|
|
func TestTurnLoop_PushAfterTakeLateItems_Panics(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop.Push("a")
|
|
loop.Stop()
|
|
loop.Run(ctx)
|
|
result := loop.Wait()
|
|
|
|
result.TakeLateItems()
|
|
|
|
assert.PanicsWithValue(t, "TurnLoop: Push called after TakeLateItems", func() {
|
|
loop.Push("too-late")
|
|
})
|
|
}
|
|
|
|
func TestTurnLoop_TakeLateItems_NeverCalled_NoImpact(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop.Push("a")
|
|
loop.Push("b")
|
|
loop.Stop()
|
|
loop.Run(ctx)
|
|
result := loop.Wait()
|
|
|
|
// Don't call TakeLateItems — verify UnhandledItems works normally
|
|
assert.Contains(t, result.UnhandledItems, "b")
|
|
assert.Nil(t, result.ExitReason)
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointErr_SeparateFromExitReason(t *testing.T) {
|
|
ctx := context.Background()
|
|
saveStore := &errorCheckpointStore{setErr: fmt.Errorf("storage unavailable")}
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: saveStore,
|
|
CheckpointID: "cp-separate-err",
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop.Push("a")
|
|
loop.Stop()
|
|
loop.Run(ctx)
|
|
result := loop.Wait()
|
|
|
|
// ExitReason should be nil (clean stop), checkpoint error should be separate
|
|
assert.Nil(t, result.ExitReason)
|
|
assert.True(t, result.CheckpointAttempted)
|
|
assert.Error(t, result.CheckpointErr)
|
|
assert.Contains(t, result.CheckpointErr.Error(), "storage unavailable")
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointAttempted_FalseWhenNoStore(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop.Push("a")
|
|
loop.Stop()
|
|
loop.Run(ctx)
|
|
result := loop.Wait()
|
|
|
|
assert.False(t, result.CheckpointAttempted)
|
|
assert.Nil(t, result.CheckpointErr)
|
|
}
|
|
|
|
func TestTurnLoop_CheckpointAttempted_FalseOnErrorExit(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
genInputErr := errors.New("gen input failed")
|
|
|
|
firstTurnDone := make(chan struct{})
|
|
var callCount int32
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: "cp-err-exit",
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
n := atomic.AddInt32(&callCount, 1)
|
|
if n > 1 {
|
|
return nil, genInputErr
|
|
}
|
|
return &GenInputResult[string, *schema.Message]{Input: &AgentInput{}, Consumed: items}, nil
|
|
},
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
close(firstTurnDone)
|
|
return nil
|
|
},
|
|
})
|
|
loop.Push("msg1")
|
|
<-firstTurnDone
|
|
loop.Push("msg2")
|
|
result := loop.Wait()
|
|
|
|
// Loop exited from error, not Stop() — checkpoint should not be saved
|
|
assert.ErrorIs(t, result.ExitReason, genInputErr)
|
|
assert.False(t, result.CheckpointAttempted)
|
|
assert.Nil(t, result.CheckpointErr)
|
|
}
|
|
|
|
func TestTurnLoop_StopConcurrentWithCallbackError_NoCheckpoint(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "stop-concurrent-err"
|
|
|
|
prepareErr := errors.New("prepare agent failed")
|
|
firstTurnDone := make(chan struct{})
|
|
stopCalled := make(chan struct{})
|
|
var prepareCount int32
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
n := atomic.AddInt32(&prepareCount, 1)
|
|
if n > 1 {
|
|
// Wait until Stop() has been called so stopCtrl.isCommitted() is true.
|
|
<-stopCalled
|
|
return nil, prepareErr
|
|
}
|
|
return &turnLoopMockAgent{name: "test"}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
close(firstTurnDone)
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-firstTurnDone
|
|
loop.Push("msg2")
|
|
|
|
// Call Stop() and signal PrepareAgent to proceed with error
|
|
go func() {
|
|
loop.Stop()
|
|
close(stopCalled)
|
|
}()
|
|
|
|
result := loop.Wait()
|
|
|
|
// The loop may exit via Stop (clean) or via PrepareAgent error.
|
|
// If it exited via PrepareAgent error with Stop also called:
|
|
// checkpoint should NOT be saved.
|
|
if result.ExitReason != nil && !errors.As(result.ExitReason, new(*CancelError)) {
|
|
assert.ErrorIs(t, result.ExitReason, prepareErr)
|
|
assert.False(t, result.CheckpointAttempted, "should not checkpoint when exit is caused by callback error")
|
|
}
|
|
// If Stop won the race, that's fine — checkpoint may or may not be saved
|
|
// depending on idle state. The test is about the error path.
|
|
}
|
|
|
|
func TestTurnLoop_DeleteWithoutCheckPointDeleter_NoOp(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "no-deleter"
|
|
|
|
// First loop: save a checkpoint
|
|
loop1 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop1.Push("a")
|
|
loop1.Stop()
|
|
loop1.Run(ctx)
|
|
loop1.Wait()
|
|
|
|
store.mu.Lock()
|
|
_, exists := store.m[cpID]
|
|
store.mu.Unlock()
|
|
assert.True(t, exists, "checkpoint should be saved")
|
|
|
|
// Second loop: exit via context cancel — should try to delete but store
|
|
// doesn't implement CheckPointDeleter, so checkpoint persists (no-op)
|
|
ctx2, cancel2 := context.WithCancel(ctx)
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareTestAgent,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
cancel2()
|
|
return nil
|
|
},
|
|
})
|
|
loop2.Push("b")
|
|
loop2.Run(ctx2)
|
|
loop2.Wait()
|
|
|
|
// Without CheckPointDeleter, the stale checkpoint should NOT be deleted
|
|
store.mu.Lock()
|
|
v, exists := store.m[cpID]
|
|
store.mu.Unlock()
|
|
assert.True(t, exists, "checkpoint should still exist without CheckPointDeleter")
|
|
assert.NotNil(t, v, "checkpoint should not be set to nil")
|
|
}
|
|
|
|
func TestTurnLoop_StopWithSkipCheckpoint(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
cpID := "skip-cp-session"
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("a")
|
|
loop.Push("b")
|
|
loop.Stop(WithSkipCheckpoint())
|
|
loop.Run(ctx)
|
|
|
|
exit := loop.Wait()
|
|
assert.NoError(t, exit.ExitReason)
|
|
assert.False(t, exit.CheckpointAttempted, "checkpoint should be skipped when WithSkipCheckpoint is used")
|
|
|
|
store.mu.Lock()
|
|
_, exists := store.m[cpID]
|
|
store.mu.Unlock()
|
|
assert.False(t, exists, "no checkpoint should be saved when WithSkipCheckpoint is used")
|
|
}
|
|
|
|
func TestTurnLoop_StopWithSkipCheckpoint_DeletesStaleCheckpoint(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := &deletableCheckpointStore{
|
|
turnLoopCheckpointStore: turnLoopCheckpointStore{m: make(map[string][]byte)},
|
|
}
|
|
cpID := "skip-stale-session"
|
|
|
|
loop1 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop1.Push("a")
|
|
loop1.Stop()
|
|
loop1.Run(ctx)
|
|
exit1 := loop1.Wait()
|
|
assert.True(t, exit1.CheckpointAttempted)
|
|
|
|
store.mu.Lock()
|
|
_, exists := store.m[cpID]
|
|
store.mu.Unlock()
|
|
assert.True(t, exists, "first loop should save checkpoint")
|
|
|
|
loop2 := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
loop2.Push("b")
|
|
loop2.Stop(WithSkipCheckpoint())
|
|
loop2.Run(ctx)
|
|
exit2 := loop2.Wait()
|
|
assert.False(t, exit2.CheckpointAttempted, "second loop should skip checkpoint")
|
|
|
|
store.mu.Lock()
|
|
deleteCalled := store.deleteCalled
|
|
store.mu.Unlock()
|
|
assert.True(t, deleteCalled, "stale checkpoint should be deleted when SkipCheckpoint is used")
|
|
}
|
|
|
|
func TestTurnLoop_StopWithStopCause(t *testing.T) {
|
|
ctx := context.Background()
|
|
cause := "user session timeout"
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Push("a")
|
|
loop.Stop(WithStopCause(cause))
|
|
|
|
exit := loop.Wait()
|
|
assert.Equal(t, cause, exit.StopCause)
|
|
}
|
|
|
|
func TestTurnLoop_StopCause_EmptyWhenNoStop(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAll,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Stop()
|
|
exit := loop.Wait()
|
|
assert.Empty(t, exit.StopCause)
|
|
}
|
|
|
|
func TestTurnLoop_StopCause_InTurnContext(t *testing.T) {
|
|
cause := "business shutdown"
|
|
gotCause := make(chan string, 1)
|
|
agentStarted := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "slow",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
close(agentStarted)
|
|
select {
|
|
case <-tc.Stopped:
|
|
gotCause <- tc.StopCause()
|
|
case <-time.After(5 * time.Second):
|
|
t.Error("timed out waiting for Stopped channel")
|
|
}
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
loop.Stop(WithImmediate(), WithStopCause(cause))
|
|
|
|
select {
|
|
case c := <-gotCause:
|
|
assert.Equal(t, cause, c)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("timed out waiting for StopCause in TurnContext")
|
|
}
|
|
|
|
exit := loop.Wait()
|
|
assert.Equal(t, cause, exit.StopCause)
|
|
}
|
|
|
|
func TestTurnLoop_StopCause_FirstNonEmptyWins(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "slow",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
close(agentStarted)
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
loop.Stop(WithGraceful(), WithStopCause("first cause"))
|
|
loop.Stop(WithStopCause("second cause"))
|
|
|
|
exit := loop.Wait()
|
|
assert.Equal(t, "first cause", exit.StopCause, "first non-empty StopCause should win")
|
|
}
|
|
|
|
func TestTurnLoop_StopBeforeRun_PushThenStop(t *testing.T) {
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
t.Fatal("GenInput should not be called when Stop is called before Run")
|
|
return nil, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
t.Fatal("PrepareAgent should not be called when Stop is called before Run")
|
|
return nil, nil
|
|
},
|
|
})
|
|
|
|
ok, _ := loop.Push("item1")
|
|
assert.True(t, ok)
|
|
ok, _ = loop.Push("item2")
|
|
assert.True(t, ok)
|
|
|
|
loop.Stop()
|
|
loop.Run(context.Background())
|
|
result := loop.Wait()
|
|
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.Equal(t, []string{"item1", "item2"}, result.UnhandledItems)
|
|
assert.Empty(t, result.InterruptedItems)
|
|
assert.Empty(t, result.TakeLateItems())
|
|
}
|
|
|
|
func TestTurnLoop_StopBeforeRun_StopThenPush(t *testing.T) {
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
t.Fatal("GenInput should not be called when Stop is called before Run")
|
|
return nil, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
t.Fatal("PrepareAgent should not be called when Stop is called before Run")
|
|
return nil, nil
|
|
},
|
|
})
|
|
|
|
loop.Stop()
|
|
|
|
ok, _ := loop.Push("item1")
|
|
assert.False(t, ok)
|
|
ok, _ = loop.Push("item2")
|
|
assert.False(t, ok)
|
|
|
|
loop.Run(context.Background())
|
|
result := loop.Wait()
|
|
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.Empty(t, result.UnhandledItems)
|
|
assert.Empty(t, result.InterruptedItems)
|
|
assert.Equal(t, []string{"item1", "item2"}, result.TakeLateItems())
|
|
}
|
|
|
|
func TestTurnLoop_SkipCheckpoint_Sticky(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
|
|
store := newTestStore()
|
|
cpID := "sticky-skip-session"
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: cpID,
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "slow",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
close(agentStarted)
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
loop.Stop(WithGraceful(), WithSkipCheckpoint())
|
|
loop.Stop()
|
|
|
|
exit := loop.Wait()
|
|
assert.False(t, exit.CheckpointAttempted, "SkipCheckpoint should be sticky across multiple Stop calls")
|
|
|
|
store.mu.Lock()
|
|
_, exists := store.m[cpID]
|
|
store.mu.Unlock()
|
|
assert.False(t, exists, "no checkpoint should be saved when SkipCheckpoint was set in any Stop call")
|
|
}
|
|
|
|
func TestWithGracefulTimeout_NonPositive_Panics(t *testing.T) {
|
|
assert.PanicsWithValue(t, "adk: WithGracefulTimeout: gracePeriod must be positive",
|
|
func() { WithGracefulTimeout(0) })
|
|
assert.PanicsWithValue(t, "adk: WithGracefulTimeout: gracePeriod must be positive",
|
|
func() { WithGracefulTimeout(-1 * time.Second) })
|
|
}
|
|
|
|
func TestWithPreempt_ZeroSafePoint_Panics(t *testing.T) {
|
|
assert.PanicsWithValue(t, "adk: SafePoint must not be zero; use AfterToolCalls, AfterChatModel, or AnySafePoint",
|
|
func() { WithPreempt[string, *schema.Message](SafePoint(0)) })
|
|
}
|
|
|
|
func TestWithPreemptTimeout_ZeroSafePoint_Panics(t *testing.T) {
|
|
assert.PanicsWithValue(t, "adk: SafePoint must not be zero; use AfterToolCalls, AfterChatModel, or AnySafePoint",
|
|
func() { WithPreemptTimeout[string, *schema.Message](SafePoint(0), time.Second) })
|
|
}
|
|
|
|
func TestSafePoint_ToCancelMode(t *testing.T) {
|
|
assert.Equal(t, CancelAfterToolCalls, AfterToolCalls.toCancelMode())
|
|
assert.Equal(t, CancelAfterChatModel, AfterChatModel.toCancelMode())
|
|
assert.Equal(t, CancelAfterToolCalls|CancelAfterChatModel, AnySafePoint.toCancelMode())
|
|
}
|
|
|
|
func TestNewTurnLoop_NilGenInput_Panics(t *testing.T) {
|
|
assert.PanicsWithValue(t, "adk: NewTurnLoop: GenInput is required", func() {
|
|
NewTurnLoop(TurnLoopConfig[string, *schema.Message]{PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return nil, nil
|
|
}})
|
|
})
|
|
}
|
|
|
|
func TestNewTurnLoop_NilPrepareAgent_Panics(t *testing.T) {
|
|
assert.PanicsWithValue(t, "adk: NewTurnLoop: PrepareAgent is required", func() {
|
|
NewTurnLoop(TurnLoopConfig[string, *schema.Message]{GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return nil, nil
|
|
}})
|
|
})
|
|
}
|
|
|
|
func TestDeriveAgentToolCancelContext_NilParent_ReturnsNil(t *testing.T) {
|
|
var cc *cancelContext
|
|
assert.Nil(t, cc.deriveAgentToolCancelContext(context.Background()))
|
|
}
|
|
|
|
func TestUntilIdleFor(t *testing.T) {
|
|
t.Run("FiresAfterIdleDuration", func(t *testing.T) {
|
|
turnDone := make(chan struct{})
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(turnDone)
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-turnDone
|
|
|
|
loop.Stop(UntilIdleFor(50 * time.Millisecond))
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
loop.Wait()
|
|
close(done)
|
|
}()
|
|
|
|
select {
|
|
case <-done:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("loop did not exit after idle timeout")
|
|
}
|
|
})
|
|
|
|
t.Run("ResetsOnPush", func(t *testing.T) {
|
|
turnCount := int32(0)
|
|
turnDone := make(chan struct{}, 10)
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
atomic.AddInt32(&turnCount, 1)
|
|
turnDone <- struct{}{}
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-turnDone
|
|
|
|
loop.Stop(UntilIdleFor(200 * time.Millisecond))
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
loop.Push("msg2")
|
|
<-turnDone
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
loop.Wait()
|
|
close(done)
|
|
}()
|
|
|
|
select {
|
|
case <-done:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("loop did not exit after idle timeout")
|
|
}
|
|
|
|
assert.Equal(t, int32(2), atomic.LoadInt32(&turnCount))
|
|
})
|
|
|
|
t.Run("EscalatedByStopWithImmediate", func(t *testing.T) {
|
|
agentStarted := make(chan *cancelContext, 1)
|
|
probe := &turnLoopStopModeProbeAgent{ccCh: agentStarted}
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return probe, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
cc := <-agentStarted
|
|
|
|
loop.Stop(UntilIdleFor(10 * time.Minute))
|
|
loop.Stop(WithImmediate())
|
|
|
|
deadline := time.After(2 * time.Second)
|
|
for {
|
|
if cc.getMode() == CancelImmediate {
|
|
break
|
|
}
|
|
select {
|
|
case <-deadline:
|
|
t.Fatal("cancel mode did not escalate to CancelImmediate")
|
|
default:
|
|
}
|
|
time.Sleep(1 * time.Millisecond)
|
|
}
|
|
|
|
exit := loop.Wait()
|
|
var ce *CancelError
|
|
require.True(t, errors.As(exit.ExitReason, &ce))
|
|
assert.Equal(t, CancelImmediate, ce.Info.Mode)
|
|
})
|
|
|
|
t.Run("EscalatedByStopWithGraceful", func(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentDone := make(chan struct{})
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(agentStarted)
|
|
<-ctx.Done()
|
|
close(agentDone)
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
|
|
loop.Stop(UntilIdleFor(10 * time.Minute))
|
|
loop.Stop(WithGracefulTimeout(50 * time.Millisecond))
|
|
|
|
select {
|
|
case <-agentDone:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("agent was not cancelled")
|
|
}
|
|
|
|
exit := loop.Wait()
|
|
assert.Error(t, exit.ExitReason)
|
|
})
|
|
}
|
|
|
|
// TestUntilIdleFor_DoesNotCancelRunningAgent verifies that Stop(UntilIdleFor)
|
|
// records an idle stop policy but does NOT create a pending cancel request for
|
|
// the running agent.
|
|
func TestUntilIdleFor_DoesNotCancelRunningAgent(t *testing.T) {
|
|
t.Run("BeforeRun", func(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentCtxCanceled := int32(0)
|
|
agentDone := make(chan struct{})
|
|
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(agentStarted)
|
|
// Block until context is canceled or a short timeout.
|
|
select {
|
|
case <-ctx.Done():
|
|
atomic.StoreInt32(&agentCtxCanceled, 1)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
close(agentDone)
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
// Call Stop(UntilIdleFor) BEFORE Run.
|
|
loop.Stop(UntilIdleFor(50 * time.Millisecond))
|
|
loop.Run(context.Background())
|
|
|
|
<-agentStarted
|
|
<-agentDone
|
|
|
|
exit := loop.Wait()
|
|
assert.Nil(t, exit.ExitReason, "UntilIdleFor should not produce a CancelError")
|
|
assert.Equal(t, int32(0), atomic.LoadInt32(&agentCtxCanceled),
|
|
"agent context should not have been canceled by UntilIdleFor")
|
|
})
|
|
|
|
t.Run("DuringRun", func(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentCtxCanceled := int32(0)
|
|
agentDone := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(agentStarted)
|
|
select {
|
|
case <-ctx.Done():
|
|
atomic.StoreInt32(&agentCtxCanceled, 1)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
close(agentDone)
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
|
|
// Call Stop(UntilIdleFor) while the agent is running.
|
|
loop.Stop(UntilIdleFor(50 * time.Millisecond))
|
|
<-agentDone
|
|
|
|
exit := loop.Wait()
|
|
assert.Nil(t, exit.ExitReason, "UntilIdleFor should not produce a CancelError")
|
|
assert.Equal(t, int32(0), atomic.LoadInt32(&agentCtxCanceled),
|
|
"agent context should not have been canceled by UntilIdleFor")
|
|
})
|
|
|
|
// Cancel opts paired with UntilIdleFor in the same call are silently
|
|
// dropped. The agent must run to completion even when WithImmediate is
|
|
// combined with UntilIdleFor.
|
|
t.Run("CancelOptsDroppedInSameCall", func(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentCtxCanceled := int32(0)
|
|
agentDone := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage(items[0])}},
|
|
Consumed: items,
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(agentStarted)
|
|
select {
|
|
case <-ctx.Done():
|
|
atomic.StoreInt32(&agentCtxCanceled, 1)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
close(agentDone)
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
|
|
// WithImmediate in the same call as UntilIdleFor must be ignored.
|
|
loop.Stop(UntilIdleFor(50*time.Millisecond), WithImmediate())
|
|
<-agentDone
|
|
|
|
exit := loop.Wait()
|
|
assert.Nil(t, exit.ExitReason, "cancel opts should be dropped when combined with UntilIdleFor")
|
|
assert.Equal(t, int32(0), atomic.LoadInt32(&agentCtxCanceled),
|
|
"agent context should not have been canceled")
|
|
})
|
|
}
|
|
|
|
func TestUntilIdleFor_ContextCancelDuringIdleWait(t *testing.T) {
|
|
turnDone := make(chan struct{})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(turnDone)
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-turnDone
|
|
|
|
// Start idle timer, then cancel the parent context while idle.
|
|
loop.Stop(UntilIdleFor(10 * time.Minute))
|
|
time.Sleep(20 * time.Millisecond)
|
|
cancel()
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
loop.Wait()
|
|
close(done)
|
|
}()
|
|
|
|
waitOrFail(t, done, "loop should exit when context is canceled during idle wait")
|
|
|
|
exit := loop.Wait()
|
|
assert.ErrorIs(t, exit.ExitReason, context.Canceled)
|
|
}
|
|
|
|
func TestCancelRequestState_ImmediateDominatesSafePointModes(t *testing.T) {
|
|
now := time.Now()
|
|
state := newCancelRequestState([]AgentCancelOption{
|
|
WithAgentCancelMode(CancelAfterChatModel),
|
|
WithAgentCancelTimeout(time.Minute),
|
|
}, now)
|
|
|
|
state.merge([]AgentCancelOption{WithAgentCancelMode(CancelImmediate)}, now)
|
|
|
|
cfg := parseAgentCancelOptions(state.cancelOptions(now)...)
|
|
assert.Equal(t, CancelImmediate, cfg.Mode)
|
|
assert.Nil(t, cfg.Timeout)
|
|
}
|
|
|
|
func TestCancelRequestState_NilMergeDoesNotCreateCancelIntent(t *testing.T) {
|
|
now := time.Now()
|
|
state := newCancelRequestState([]AgentCancelOption{WithAgentCancelMode(CancelAfterChatModel)}, now)
|
|
|
|
state.merge(nil, now)
|
|
|
|
cfg := parseAgentCancelOptions(state.cancelOptions(now)...)
|
|
assert.Equal(t, CancelAfterChatModel, cfg.Mode)
|
|
}
|
|
|
|
func TestCancelRequestState_EmptyMergeMeansExplicitImmediate(t *testing.T) {
|
|
now := time.Now()
|
|
state := newCancelRequestState([]AgentCancelOption{WithAgentCancelMode(CancelAfterChatModel)}, now)
|
|
|
|
state.merge([]AgentCancelOption{}, now)
|
|
|
|
cfg := parseAgentCancelOptions(state.cancelOptions(now)...)
|
|
assert.Equal(t, CancelImmediate, cfg.Mode)
|
|
}
|
|
|
|
func TestCancelRequestState_SafePointModesJoin(t *testing.T) {
|
|
now := time.Now()
|
|
state := newCancelRequestState([]AgentCancelOption{WithAgentCancelMode(CancelAfterChatModel)}, now)
|
|
|
|
state.merge([]AgentCancelOption{WithAgentCancelMode(CancelAfterToolCalls)}, now)
|
|
|
|
cfg := parseAgentCancelOptions(state.cancelOptions(now)...)
|
|
assert.Equal(t, CancelAfterChatModel|CancelAfterToolCalls, cfg.Mode)
|
|
}
|
|
|
|
func TestCancelRequestState_RecursiveIsMonotonic(t *testing.T) {
|
|
now := time.Now()
|
|
state := newCancelRequestState([]AgentCancelOption{WithAgentCancelMode(CancelAfterChatModel)}, now)
|
|
|
|
state.merge([]AgentCancelOption{WithAgentCancelMode(CancelAfterToolCalls), WithRecursive()}, now)
|
|
state.merge([]AgentCancelOption{WithAgentCancelMode(CancelAfterChatModel)}, now)
|
|
|
|
cfg := parseAgentCancelOptions(state.cancelOptions(now)...)
|
|
assert.True(t, cfg.Recursive)
|
|
}
|
|
|
|
func TestCancelRequestState_TimeoutUsesEarliestDeadline(t *testing.T) {
|
|
now := time.Now()
|
|
state := newCancelRequestState([]AgentCancelOption{
|
|
WithAgentCancelMode(CancelAfterChatModel),
|
|
WithAgentCancelTimeout(10 * time.Second),
|
|
}, now)
|
|
|
|
state.merge([]AgentCancelOption{
|
|
WithAgentCancelMode(CancelAfterToolCalls),
|
|
WithAgentCancelTimeout(time.Second),
|
|
}, now.Add(100*time.Millisecond))
|
|
|
|
cfg := parseAgentCancelOptions(state.cancelOptions(now.Add(100 * time.Millisecond))...)
|
|
require.NotNil(t, cfg.Timeout)
|
|
assert.LessOrEqual(t, *cfg.Timeout, time.Second)
|
|
}
|
|
|
|
func TestCancelRequestState_ExpiredTimeoutConvertsToImmediate(t *testing.T) {
|
|
now := time.Now()
|
|
state := newCancelRequestState([]AgentCancelOption{
|
|
WithAgentCancelMode(CancelAfterChatModel),
|
|
WithAgentCancelTimeout(time.Nanosecond),
|
|
}, now)
|
|
|
|
cfg := parseAgentCancelOptions(state.cancelOptions(now.Add(time.Second))...)
|
|
assert.Equal(t, CancelImmediate, cfg.Mode)
|
|
assert.Nil(t, cfg.Timeout)
|
|
}
|
|
|
|
func TestStopController_BareStopCommitsWithoutCancelRequest(t *testing.T) {
|
|
c := newStopController()
|
|
|
|
decision := c.requestStop(&stopConfig{})
|
|
|
|
assert.True(t, decision.commit)
|
|
assert.True(t, c.isCommitted())
|
|
c.beginActiveTurn()
|
|
_, ok := c.receiveCancel()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestStopController_UntilIdleForDoesNotCreateCancelRequest(t *testing.T) {
|
|
c := newStopController()
|
|
|
|
decision := c.requestStop(&stopConfig{idleFor: time.Second})
|
|
|
|
assert.False(t, decision.commit)
|
|
assert.True(t, decision.wakeIdle)
|
|
assert.Equal(t, time.Second, c.idleDuration())
|
|
assert.False(t, c.isCommitted())
|
|
c.beginActiveTurn()
|
|
_, ok := c.receiveCancel()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestStopController_CancelOptsDroppedWhenCombinedWithUntilIdleFor(t *testing.T) {
|
|
c := newStopController()
|
|
|
|
decision := c.requestStop(&stopConfig{
|
|
idleFor: time.Second,
|
|
agentCancelOpts: []AgentCancelOption{WithRecursive()},
|
|
})
|
|
|
|
assert.False(t, decision.commit)
|
|
c.beginActiveTurn()
|
|
_, ok := c.receiveCancel()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestStopController_ImmediateStopCreatesPendingCancelForActiveTurn(t *testing.T) {
|
|
c := newStopController()
|
|
c.beginActiveTurn()
|
|
|
|
decision := c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{WithRecursive()}})
|
|
|
|
assert.True(t, decision.commit)
|
|
req, ok := c.receiveCancel()
|
|
require.True(t, ok)
|
|
cfg := parseAgentCancelOptions(req.cancelOptions(time.Now())...)
|
|
assert.Equal(t, CancelImmediate, cfg.Mode)
|
|
assert.True(t, cfg.Recursive)
|
|
}
|
|
|
|
func TestStopController_StopBeforeWatcherStartsConsumedAfterBeginActiveTurn(t *testing.T) {
|
|
c := newStopController()
|
|
|
|
decision := c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{WithRecursive()}})
|
|
assert.True(t, decision.commit)
|
|
|
|
c.beginActiveTurn()
|
|
req, ok := c.receiveCancel()
|
|
require.True(t, ok)
|
|
cfg := parseAgentCancelOptions(req.cancelOptions(time.Now())...)
|
|
assert.Equal(t, CancelImmediate, cfg.Mode)
|
|
}
|
|
|
|
func TestStopController_EndActiveTurnDropsUnconsumedCancel(t *testing.T) {
|
|
c := newStopController()
|
|
c.beginActiveTurn()
|
|
c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{WithRecursive()}})
|
|
|
|
req := c.endActiveTurn()
|
|
|
|
require.NotNil(t, req)
|
|
_, ok := c.receiveCancel()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestStopController_RepeatedStopsMergeWithoutDeescalation(t *testing.T) {
|
|
c := newStopController()
|
|
c.beginActiveTurn()
|
|
|
|
c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{WithRecursive()}})
|
|
c.requestStop(&stopConfig{})
|
|
c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{
|
|
WithAgentCancelMode(CancelAfterChatModel | CancelAfterToolCalls),
|
|
WithRecursive(),
|
|
}})
|
|
|
|
req, ok := c.receiveCancel()
|
|
require.True(t, ok)
|
|
cfg := parseAgentCancelOptions(req.cancelOptions(time.Now())...)
|
|
assert.Equal(t, CancelImmediate, cfg.Mode)
|
|
assert.True(t, cfg.Recursive)
|
|
}
|
|
|
|
func TestStopController_RepeatedStopsUseSharedCancelMergeState(t *testing.T) {
|
|
c := newStopController()
|
|
c.beginActiveTurn()
|
|
|
|
c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{WithAgentCancelMode(CancelAfterChatModel)}})
|
|
c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{WithAgentCancelMode(CancelAfterToolCalls)}})
|
|
|
|
req, ok := c.receiveCancel()
|
|
require.True(t, ok)
|
|
cfg := parseAgentCancelOptions(req.cancelOptions(time.Now())...)
|
|
assert.Equal(t, CancelAfterChatModel|CancelAfterToolCalls, cfg.Mode)
|
|
}
|
|
|
|
func TestStopController_StopCauseFirstNonEmptyWins(t *testing.T) {
|
|
c := newStopController()
|
|
|
|
c.requestStop(&stopConfig{})
|
|
c.requestStop(&stopConfig{stopCause: "first"})
|
|
c.requestStop(&stopConfig{stopCause: "second"})
|
|
|
|
assert.Equal(t, "first", c.cause())
|
|
}
|
|
|
|
func TestStopController_SkipCheckpointSticky(t *testing.T) {
|
|
c := newStopController()
|
|
|
|
c.requestStop(&stopConfig{skipCheckpoint: true})
|
|
c.requestStop(&stopConfig{})
|
|
|
|
assert.True(t, c.skipCheckpointEnabled())
|
|
}
|
|
|
|
func TestStopController_ConcurrentStopRequestsRaceSafe(t *testing.T) {
|
|
c := newStopController()
|
|
c.beginActiveTurn()
|
|
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 20; i++ {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
switch i % 5 {
|
|
case 0:
|
|
c.requestStop(&stopConfig{})
|
|
case 1:
|
|
c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{WithRecursive()}})
|
|
case 2:
|
|
c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{
|
|
WithAgentCancelMode(CancelAfterChatModel),
|
|
WithAgentCancelTimeout(time.Second),
|
|
WithRecursive(),
|
|
}})
|
|
case 3:
|
|
c.requestStop(&stopConfig{idleFor: time.Second})
|
|
case 4:
|
|
c.requestStop(&stopConfig{skipCheckpoint: true, stopCause: "cause"})
|
|
}
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
|
|
assert.True(t, c.isCommitted())
|
|
assert.True(t, c.skipCheckpointEnabled())
|
|
}
|
|
|
|
func TestStopController_CloseForLoopExitClearsPendingCancel(t *testing.T) {
|
|
c := newStopController()
|
|
c.beginActiveTurn()
|
|
c.requestStop(&stopConfig{agentCancelOpts: []AgentCancelOption{WithRecursive()}})
|
|
|
|
c.closeForLoopExit()
|
|
|
|
_, ok := c.receiveCancel()
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestAttack_UntilIdleFor_ConcurrentPushDuringIdleTimer(t *testing.T) {
|
|
turnCount := int32(0)
|
|
turnDone := make(chan struct{}, 10)
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
atomic.AddInt32(&turnCount, 1)
|
|
turnDone <- struct{}{}
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-turnDone
|
|
|
|
loop.Stop(UntilIdleFor(200 * time.Millisecond))
|
|
|
|
for i := 0; i < 5; i++ {
|
|
time.Sleep(50 * time.Millisecond)
|
|
loop.Push("concurrent-" + string(rune('a'+i)))
|
|
<-turnDone
|
|
}
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
loop.Wait()
|
|
close(done)
|
|
}()
|
|
|
|
waitOrFail(t, done, "loop did not exit after idle timeout — Push did not reset timer correctly")
|
|
|
|
finalCount := atomic.LoadInt32(&turnCount)
|
|
assert.Equal(t, int32(6), finalCount, "all 6 pushes should have been processed")
|
|
}
|
|
|
|
func TestAttack_UntilIdleFor_MultipleStopCallsFirstWins(t *testing.T) {
|
|
turnDone := make(chan struct{})
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(turnDone)
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-turnDone
|
|
|
|
loop.Stop(UntilIdleFor(100 * time.Millisecond))
|
|
loop.Stop(UntilIdleFor(10 * time.Minute))
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
loop.Wait()
|
|
close(done)
|
|
}()
|
|
|
|
waitOrFail(t, done, "second UntilIdleFor should have been ignored; loop should have exited with 100ms timer")
|
|
}
|
|
|
|
func TestAttack_BareStopOverridesUntilIdleFor(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
agentDone := make(chan struct{})
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(agentStarted)
|
|
<-agentDone
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
|
|
loop.Stop(UntilIdleFor(10 * time.Minute))
|
|
|
|
loop.Stop()
|
|
close(agentDone)
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
loop.Wait()
|
|
close(done)
|
|
}()
|
|
|
|
waitOrFail(t, done, "bare Stop should override UntilIdleFor and cause immediate shutdown")
|
|
|
|
exit := loop.Wait()
|
|
assert.NoError(t, exit.ExitReason, "bare Stop should exit cleanly")
|
|
}
|
|
|
|
func TestAttack_BareStopDoesNotDeescalateExistingCancelIntent(t *testing.T) {
|
|
agentStarted := make(chan *cancelContext, 1)
|
|
probe := &turnLoopStopModeProbeAgent{ccCh: agentStarted}
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return probe, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
cc := <-agentStarted
|
|
|
|
loop.Stop(WithImmediate())
|
|
|
|
time.Sleep(20 * time.Millisecond)
|
|
|
|
loop.Stop()
|
|
|
|
time.Sleep(20 * time.Millisecond)
|
|
mode := cc.getMode()
|
|
assert.Equal(t, CancelImmediate, mode, "bare Stop after WithImmediate must not de-escalate cancel mode")
|
|
|
|
exit := loop.Wait()
|
|
var ce *CancelError
|
|
require.True(t, errors.As(exit.ExitReason, &ce))
|
|
assert.Equal(t, CancelImmediate, ce.Info.Mode)
|
|
}
|
|
|
|
func TestAttack_InterruptedItems_EmptyWhenAgentFinishesNormally(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(agentStarted)
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
time.Sleep(50 * time.Millisecond)
|
|
loop.Stop()
|
|
|
|
exit := loop.Wait()
|
|
assert.NoError(t, exit.ExitReason)
|
|
assert.Empty(t, exit.InterruptedItems, "InterruptedItems must be empty when agent finished normally")
|
|
}
|
|
|
|
func TestAttack_TurnBuffer_WakeupDoesNotLoseItems(t *testing.T) {
|
|
tb := newTurnBuffer[string]()
|
|
|
|
tb.Send("a")
|
|
tb.Send("b")
|
|
tb.Wakeup()
|
|
tb.Send("c")
|
|
|
|
var got []string
|
|
for i := 0; i < 3; i++ {
|
|
val, ok := tb.Receive()
|
|
require.True(t, ok)
|
|
got = append(got, val)
|
|
}
|
|
|
|
assert.Equal(t, []string{"a", "b", "c"}, got, "Wakeup must not cause items to be lost")
|
|
}
|
|
|
|
func TestAttack_TurnBuffer_ClearWakeupPreventsSpuriousReturn(t *testing.T) {
|
|
tb := newTurnBuffer[string]()
|
|
|
|
tb.Wakeup()
|
|
tb.ClearWakeup()
|
|
|
|
received := make(chan string, 1)
|
|
go func() {
|
|
val, ok := tb.Receive()
|
|
if ok {
|
|
received <- val
|
|
}
|
|
}()
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
tb.Send("real")
|
|
|
|
select {
|
|
case val := <-received:
|
|
assert.Equal(t, "real", val, "ClearWakeup should prevent spurious empty return")
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("Receive blocked forever despite Send")
|
|
}
|
|
}
|
|
|
|
func TestAttack_StopBeforeRun_UntilIdleFor_ExitsImmediately(t *testing.T) {
|
|
loop := NewTurnLoop(TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: prepareTestAgent,
|
|
})
|
|
|
|
loop.Stop(UntilIdleFor(10 * time.Minute))
|
|
loop.Stop()
|
|
|
|
loop.Run(context.Background())
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
loop.Wait()
|
|
close(done)
|
|
}()
|
|
|
|
waitOrFail(t, done, "loop should exit immediately when Stop() called before Run()")
|
|
}
|
|
|
|
func TestAttack_PushAfterStop_UntilIdleFor_RoutedToLateItems(t *testing.T) {
|
|
turnDone := make(chan struct{})
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(turnDone)
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-turnDone
|
|
|
|
loop.Stop(UntilIdleFor(50 * time.Millisecond))
|
|
exit := loop.Wait()
|
|
assert.NoError(t, exit.ExitReason)
|
|
|
|
ok, _ := loop.Push("after-stop")
|
|
assert.False(t, ok, "Push after loop exited should return false")
|
|
|
|
late := exit.TakeLateItems()
|
|
assert.Equal(t, []string{"after-stop"}, late)
|
|
}
|
|
|
|
func TestAttack_ConcurrentStopEscalation_RaceDetector(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(agentStarted)
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 10; i++ {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
switch i % 4 {
|
|
case 0:
|
|
loop.Stop()
|
|
case 1:
|
|
loop.Stop(WithImmediate())
|
|
case 2:
|
|
loop.Stop(WithGracefulTimeout(100 * time.Millisecond))
|
|
case 3:
|
|
loop.Stop(UntilIdleFor(50 * time.Millisecond))
|
|
}
|
|
}(i)
|
|
}
|
|
|
|
wg.Wait()
|
|
exit := loop.Wait()
|
|
t.Log("ExitReason:", exit.ExitReason)
|
|
}
|
|
|
|
func TestAttack_SkipCheckpoint_Sticky(t *testing.T) {
|
|
agentStarted := make(chan struct{})
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
close(agentStarted)
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
},
|
|
}, nil
|
|
},
|
|
Store: newTestStore(),
|
|
CheckpointID: "test-sticky",
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
<-agentStarted
|
|
|
|
loop.Stop(WithSkipCheckpoint())
|
|
loop.Stop(WithImmediate())
|
|
|
|
exit := loop.Wait()
|
|
assert.False(t, exit.CheckpointAttempted, "SkipCheckpoint is sticky; checkpoint should be skipped")
|
|
}
|
|
|
|
// turnLoopNestedProbeAgent simulates an agent with a nested sub-agent
|
|
// by deriving a child cancelContext. This allows tests to verify that
|
|
// TurnLoop's Stop/Push options correctly propagate recursive cancellation.
|
|
//
|
|
// IMPORTANT: child.markDone() is NOT called by the probe. The test MUST
|
|
// call it (e.g. via t.Cleanup) after verifying propagation to avoid a
|
|
// race between markDone closing child.doneChan and the deriveAgentToolCancelContext
|
|
// goroutines propagating the cancel signal.
|
|
type turnLoopNestedProbeAgent struct {
|
|
parentCCCh chan *cancelContext
|
|
childCCCh chan *cancelContext
|
|
}
|
|
|
|
func (a *turnLoopNestedProbeAgent) Name(_ context.Context) string { return "nested-probe" }
|
|
func (a *turnLoopNestedProbeAgent) Description(_ context.Context) string { return "nested-probe" }
|
|
func (a *turnLoopNestedProbeAgent) Run(ctx context.Context, _ *AgentInput, opts ...AgentRunOption) *AsyncIterator[*AgentEvent] {
|
|
iter, gen := NewAsyncIteratorPair[*AgentEvent]()
|
|
o := getCommonOptions(nil, opts...)
|
|
cc := o.cancelCtx
|
|
|
|
child := cc.deriveAgentToolCancelContext(ctx)
|
|
a.parentCCCh <- cc
|
|
a.childCCCh <- child
|
|
|
|
go func() {
|
|
defer gen.Close()
|
|
<-cc.cancelChan
|
|
for {
|
|
if cc.getMode() == CancelImmediate {
|
|
gen.Send(&AgentEvent{Err: cc.createCancelError()})
|
|
return
|
|
}
|
|
time.Sleep(1 * time.Millisecond)
|
|
}
|
|
}()
|
|
return iter
|
|
}
|
|
|
|
func TestTurnLoop_Stop_WithImmediate_RecursivePropagation(t *testing.T) {
|
|
parentCCCh := make(chan *cancelContext, 1)
|
|
childCCCh := make(chan *cancelContext, 1)
|
|
probe := &turnLoopNestedProbeAgent{parentCCCh: parentCCCh, childCCCh: childCCCh}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return probe, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
cc := <-parentCCCh
|
|
child := <-childCCCh
|
|
t.Cleanup(func() { child.markDone() })
|
|
|
|
loop.Stop(WithImmediate())
|
|
|
|
// Child should receive the cancel signal via recursive propagation.
|
|
select {
|
|
case <-child.cancelChan:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("child did not receive cancel via recursive propagation")
|
|
}
|
|
|
|
// Child should also receive the immediate cancel signal.
|
|
select {
|
|
case <-child.immediateChan:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("child did not receive immediate cancel via recursive propagation")
|
|
}
|
|
|
|
assert.True(t, cc.isRecursive(), "WithImmediate should set recursive on parent")
|
|
assert.True(t, child.shouldCancel(), "child should be cancelled")
|
|
assert.True(t, child.isImmediateCancelled(), "child should have received immediate cancel")
|
|
|
|
exit := loop.Wait()
|
|
var ce *CancelError
|
|
require.True(t, errors.As(exit.ExitReason, &ce))
|
|
assert.Equal(t, CancelImmediate, ce.Info.Mode)
|
|
}
|
|
|
|
func TestTurnLoop_Push_WithPreemptTimeout_RecursivePropagation(t *testing.T) {
|
|
parentCCCh := make(chan *cancelContext, 2)
|
|
childCCCh := make(chan *cancelContext, 2)
|
|
probe := &turnLoopNestedProbeAgent{parentCCCh: parentCCCh, childCCCh: childCCCh}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return probe, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("first")
|
|
cc := <-parentCCCh
|
|
child := <-childCCCh
|
|
t.Cleanup(func() { child.markDone() })
|
|
|
|
// Preempt with a very short timeout so it escalates to CancelImmediate quickly.
|
|
loop.Push("urgent", WithPreemptTimeout[string, *schema.Message](AfterChatModel, 10*time.Millisecond))
|
|
|
|
// After timeout escalation, child should receive the immediate cancel
|
|
// via recursive propagation.
|
|
select {
|
|
case <-child.immediateChan:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("child did not receive immediate cancel after preempt timeout escalation")
|
|
}
|
|
|
|
assert.True(t, cc.isRecursive(), "WithPreemptTimeout should set recursive on parent")
|
|
assert.True(t, child.isImmediateCancelled(), "child should have received immediate cancel")
|
|
|
|
loop.Stop(WithImmediate())
|
|
loop.Wait()
|
|
}
|
|
|
|
func TestUntilIdleFor_NonPositive_Panics(t *testing.T) {
|
|
assert.PanicsWithValue(t, "adk: UntilIdleFor: duration must be positive",
|
|
func() { UntilIdleFor(0) })
|
|
assert.PanicsWithValue(t, "adk: UntilIdleFor: duration must be positive",
|
|
func() { UntilIdleFor(-1 * time.Second) })
|
|
}
|
|
|
|
func TestSaveTurnLoopCheckpoint_NilStore(t *testing.T) {
|
|
l := &TurnLoop[string, *schema.Message]{config: TurnLoopConfig[string, *schema.Message]{Store: nil}}
|
|
err := l.saveTurnLoopCheckpoint(context.Background(), "cp-1", &turnLoopCheckpoint[string]{})
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "checkpoint store is nil")
|
|
}
|
|
|
|
func TestSetupBridgeStore_NilStore_Resume(t *testing.T) {
|
|
l := &TurnLoop[string, *schema.Message]{config: TurnLoopConfig[string, *schema.Message]{Store: nil}}
|
|
spec := &turnRunSpec[string, *schema.Message]{isResume: true}
|
|
_, _, err := l.setupBridgeStore(spec, nil)
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "checkpoint store is nil")
|
|
}
|
|
|
|
// TestTurnLoop_Preempt_LoopStalledAfterSecondPreemptPush covers a liveness
|
|
// regression where a preempted turn was followed by another preemptive Push and
|
|
// the loop stopped making progress before processing the later item.
|
|
func TestTurnLoop_Preempt_LoopStalledAfterSecondPreemptPush(t *testing.T) {
|
|
// turnCount tracks how many turns have been fully processed.
|
|
var turnCount int32
|
|
|
|
// Channels to synchronize the test with each turn's lifecycle.
|
|
firstAgentStarted := make(chan struct{})
|
|
secondTurnDone := make(chan struct{})
|
|
thirdTurnDone := make(chan struct{})
|
|
|
|
var firstAgentStartedOnce, secondTurnDoneOnce, thirdTurnDoneOnce sync.Once
|
|
|
|
agent := &turnLoopCancellableMockAgent{
|
|
name: "test",
|
|
runFunc: func(ctx context.Context, input *AgentInput) (*AgentOutput, error) {
|
|
turn := atomic.AddInt32(&turnCount, 1)
|
|
switch turn {
|
|
case 1:
|
|
// First turn: signal started, then block until preempted.
|
|
firstAgentStartedOnce.Do(func() { close(firstAgentStarted) })
|
|
<-ctx.Done()
|
|
case 2, 3:
|
|
// Subsequent turns: complete immediately.
|
|
}
|
|
return &AgentOutput{}, nil
|
|
},
|
|
}
|
|
|
|
loop := newAndRunTurnLoop(context.Background(), TurnLoopConfig[string, *schema.Message]{
|
|
PrepareAgent: prepareAgent(agent),
|
|
GenInput: genInputConsumeFirst,
|
|
OnAgentEvents: func(ctx context.Context, tc *TurnContext[string, *schema.Message], events *AsyncIterator[*AgentEvent]) error {
|
|
for {
|
|
if _, ok := events.Next(); !ok {
|
|
break
|
|
}
|
|
}
|
|
turn := atomic.LoadInt32(&turnCount)
|
|
switch turn {
|
|
case 2:
|
|
secondTurnDoneOnce.Do(func() { close(secondTurnDone) })
|
|
case 3:
|
|
thirdTurnDoneOnce.Do(func() { close(thirdTurnDone) })
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
|
|
// Step 1: Push item A (no preempt). Wait for agent to start.
|
|
loop.Push("A")
|
|
waitOrFail(t, firstAgentStarted, "agent did not start for item A")
|
|
|
|
// Step 2: Push item B with preempt. This cancels the first turn.
|
|
loop.Push("B", WithPreempt[string, *schema.Message](AnySafePoint))
|
|
|
|
// Wait for the second turn (item B) to complete successfully.
|
|
waitOrFail(t, secondTurnDone, "second turn (item B) did not complete")
|
|
|
|
// Step 3: Push item C with preempt. This is the scenario that triggers
|
|
// the bug — the loop should process item C but instead gets stuck.
|
|
loop.Push("C", WithPreempt[string, *schema.Message](AnySafePoint))
|
|
|
|
// The loop should process item C. If the bug is present, this will timeout.
|
|
waitOrFail(t, thirdTurnDone, "third turn (item C) was never processed — loop is stuck between turns")
|
|
|
|
loop.Stop()
|
|
result := loop.Wait()
|
|
assert.NoError(t, result.ExitReason)
|
|
assert.Equal(t, int32(3), atomic.LoadInt32(&turnCount), "expected 3 turns to be processed")
|
|
}
|
|
|
|
func TestAttack_BusinessInterrupt_NoStore_ExitsWithoutPanic(t *testing.T) {
|
|
ctx := context.Background()
|
|
interruptAgent := &turnLoopInterruptAgent{interruptInfo: "no_store_test"}
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
GenInput: genInputConsumeAllWithMsg,
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return interruptAgent, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
exit := loop.Wait()
|
|
|
|
var intErr *InterruptError
|
|
require.True(t, errors.As(exit.ExitReason, &intErr), "expected *InterruptError, got: %v", exit.ExitReason)
|
|
assert.Equal(t, []string{"msg1"}, exit.InterruptedItems)
|
|
assert.False(t, exit.CheckpointAttempted, "no store → no checkpoint attempt")
|
|
}
|
|
|
|
func TestAttack_BusinessInterrupt_EmptyConsumed_NoCheckpoint(t *testing.T) {
|
|
ctx := context.Background()
|
|
store := newTestStore()
|
|
interruptAgent := &turnLoopInterruptAgent{interruptInfo: "idle_test"}
|
|
|
|
loop := newAndRunTurnLoop(ctx, TurnLoopConfig[string, *schema.Message]{
|
|
Store: store,
|
|
CheckpointID: "idle-cp",
|
|
GenInput: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], items []string) (*GenInputResult[string, *schema.Message], error) {
|
|
return &GenInputResult[string, *schema.Message]{
|
|
Input: &AgentInput{Messages: []Message{schema.UserMessage("x")}},
|
|
Consumed: []string{},
|
|
}, nil
|
|
},
|
|
PrepareAgent: func(ctx context.Context, _ *TurnLoop[string, *schema.Message], consumed []string) (Agent, error) {
|
|
return interruptAgent, nil
|
|
},
|
|
})
|
|
|
|
loop.Push("msg1")
|
|
exit := loop.Wait()
|
|
|
|
var intErr *InterruptError
|
|
require.True(t, errors.As(exit.ExitReason, &intErr), "expected *InterruptError, got: %v", exit.ExitReason)
|
|
assert.Empty(t, exit.InterruptedItems, "consumed was empty → InterruptedItems should be empty")
|
|
}
|