976 lines
32 KiB
Go
976 lines
32 KiB
Go
/*
|
|
* Copyright 2026 CloudWeGo Authors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package adk
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/cloudwego/eino/components/model"
|
|
"github.com/cloudwego/eino/compose"
|
|
"github.com/cloudwego/eino/schema"
|
|
)
|
|
|
|
func init() {
|
|
schema.RegisterName[*CancelError]("_eino_adk_cancel_error")
|
|
schema.RegisterName[*AgentCancelInfo]("_eino_adk_agent_cancel_info")
|
|
schema.RegisterName[*StreamCanceledError]("_eino_adk_stream_cancelled_error")
|
|
}
|
|
|
|
// CancelMode specifies when an agent should be canceled.
|
|
// Modes can be combined with bitwise OR to cancel at multiple safe-points.
|
|
// For example, CancelAfterChatModel | CancelAfterToolCalls cancels the agent
|
|
// after whichever safe-point is reached first.
|
|
type CancelMode int
|
|
|
|
const (
|
|
// CancelImmediate cancels the agent as soon as the signal is received,
|
|
// without waiting for a ChatModel or ToolCalls safe-point.
|
|
// By default, only the root agent is interrupted; descendant agents inside
|
|
// AgentTools are torn down via context cancellation as a side effect.
|
|
// Use WithRecursive to propagate explicit immediate-cancel signals to
|
|
// descendants for clean teardown with grace period.
|
|
CancelImmediate CancelMode = 0
|
|
// CancelAfterChatModel cancels after the root agent's next chat model call
|
|
// completes. By default, only the root agent checks this safe-point;
|
|
// nested sub-agents inside AgentTools are unaware of the cancel.
|
|
// Use WithRecursive to propagate the cancel to all descendants — whichever
|
|
// ChatModel finishes first triggers the cancel.
|
|
CancelAfterChatModel CancelMode = 1 << iota
|
|
// CancelAfterToolCalls cancels after the root agent's next set of tool calls
|
|
// completes. By default, only the root agent checks this safe-point.
|
|
// Use WithRecursive to propagate to all descendants.
|
|
CancelAfterToolCalls
|
|
)
|
|
|
|
// CancelHandle represents a cancel operation that can be waited on.
|
|
type CancelHandle struct {
|
|
wait func() error
|
|
}
|
|
|
|
// Wait blocks until the cancel request reaches a terminal outcome.
|
|
//
|
|
// It reports the result of the cancel operation itself, not the agent's final
|
|
// business error:
|
|
// - nil: cancellation succeeded, including the case where a business interrupt
|
|
// was absorbed into CancelError while cancellation was active
|
|
// - ErrCancelTimeout: the requested safe-point cancellation timed out and was
|
|
// escalated to immediate cancellation
|
|
// - ErrExecutionEnded: the execution ended before cancellation took effect,
|
|
// meaning the stream drained to completion without any interrupt
|
|
func (h *CancelHandle) Wait() error {
|
|
return h.wait()
|
|
}
|
|
|
|
// AgentCancelFunc is called to request cancellation of a running agent.
|
|
// It returns after the cancel request is committed; use the returned handle's
|
|
// Wait to block for completion and outcome.
|
|
//
|
|
// The returned bool reports whether this call contributed to the CancelError
|
|
// for the current execution. "Contributed" means this call's cancel options
|
|
// were included before cancellation was finalized. It is false when cancellation
|
|
// was already finalized (handled or execution completed).
|
|
type AgentCancelFunc func(...AgentCancelOption) (*CancelHandle, bool)
|
|
|
|
type agentCancelConfig struct {
|
|
Mode CancelMode
|
|
Recursive bool
|
|
Timeout *time.Duration
|
|
}
|
|
|
|
// AgentCancelOption configures cancel behavior.
|
|
type AgentCancelOption func(*agentCancelConfig)
|
|
|
|
// WithAgentCancelMode sets the cancel mode for the agent cancel operation.
|
|
func WithAgentCancelMode(mode CancelMode) AgentCancelOption {
|
|
return func(config *agentCancelConfig) {
|
|
config.Mode = mode
|
|
}
|
|
}
|
|
|
|
// WithAgentCancelTimeout sets a timeout for the cancel operation.
|
|
// This only applies to safe-point modes (CancelAfterChatModel, CancelAfterToolCalls):
|
|
// if the safe-point hasn't fired within this duration, the cancel escalates to
|
|
// CancelImmediate. The escalated cancel still saves a checkpoint, so the execution
|
|
// can be resumed via Runner.Resume or Runner.ResumeWithParams.
|
|
// For CancelImmediate this timeout is ignored — the cancel fires immediately.
|
|
func WithAgentCancelTimeout(timeout time.Duration) AgentCancelOption {
|
|
return func(config *agentCancelConfig) {
|
|
config.Timeout = &timeout
|
|
}
|
|
}
|
|
|
|
// WithRecursive opts into recursive cancel propagation. By default, cancel
|
|
// modes only affect the root agent; descendant agents inside AgentTools are
|
|
// not notified. WithRecursive makes the cancel propagate to all descendants:
|
|
// - CancelAfterChatModel / CancelAfterToolCalls: descendants check their own safe-points.
|
|
// - CancelImmediate: descendants receive explicit immediate-cancel signals for
|
|
// clean teardown; the root uses a grace period to collect child interrupts.
|
|
//
|
|
// With recursive cancellation, each descendant agent also triggers cancellation
|
|
// and cascades its interrupt information upward. The root agent ultimately
|
|
// produces a complete checkpoint that includes descendant checkpoints, enabling
|
|
// resumption from the exact point where each descendant was interrupted.
|
|
//
|
|
// Once any cancel call includes WithRecursive, the flag stays set for the
|
|
// entire cancel lifecycle (monotonic escalation).
|
|
func WithRecursive() AgentCancelOption {
|
|
return func(config *agentCancelConfig) {
|
|
config.Recursive = true
|
|
}
|
|
}
|
|
|
|
// AgentCancelInfo contains information about a cancel operation.
|
|
type AgentCancelInfo struct {
|
|
Mode CancelMode
|
|
Escalated bool
|
|
Timeout bool
|
|
}
|
|
|
|
// CancelError is sent via AgentEvent.Err when an agent is canceled.
|
|
// Use errors.As to match and extract *CancelError from event errors.
|
|
//
|
|
// Interrupt absorption: when a cancel is active (shouldCancel() == true), ANY
|
|
// interrupt — whether from a cancel safe-point node or from business logic
|
|
// (e.g. tool.Interrupt in a tool) — is converted to a CancelError. The
|
|
// cancel "absorbs" the business interrupt. This is intentional:
|
|
//
|
|
// - In concurrent execution (parallel workflows, concurrent tool calls),
|
|
// cancel-induced and business interrupts can arrive as a single composite
|
|
// signal that cannot be split apart.
|
|
// - Even in sequential execution, treating business interrupts as CancelError
|
|
// during active cancel gives consistent semantics.
|
|
// - The business interrupt is NOT lost — the checkpoint preserves the full
|
|
// interrupt hierarchy. On resume (Runner.Resume or Runner.ResumeWithParams),
|
|
// the agent re-executes the interrupting code path and the business
|
|
// interrupt re-fires naturally.
|
|
type CancelError struct {
|
|
Info *AgentCancelInfo
|
|
|
|
// InterruptContexts provides the interrupt contexts needed for targeted
|
|
// resumption via Runner.ResumeWithParams. Each context represents a step
|
|
// in the agent hierarchy that was interrupted. This is a slice because
|
|
// composite agents (e.g. parallel workflows) may interrupt at multiple
|
|
// points simultaneously, matching the shape of AgentAction.Interrupted.InterruptContexts.
|
|
// Use each InterruptCtx.ID as a key in ResumeParams.Targets.
|
|
InterruptContexts []*InterruptCtx
|
|
|
|
interruptSignal *InterruptSignal // unexported — only Runner needs it for checkpoint
|
|
}
|
|
|
|
func (e *CancelError) Error() string {
|
|
return fmt.Sprintf("agent canceled: mode=%v, escalated=%v", e.Info.Mode, e.Info.Escalated)
|
|
}
|
|
|
|
// Sentinel errors for cancel outcomes.
|
|
var (
|
|
// ErrCancelTimeout is returned by CancelHandle.Wait when the cancel operation timed out.
|
|
ErrCancelTimeout = errors.New("cancel timed out")
|
|
|
|
// ErrExecutionEnded is returned by CancelHandle.Wait when the agent ended
|
|
// before the cancel took effect. "Ended" means the event stream was fully
|
|
// drained without any interrupt — normal completion or a fatal error.
|
|
//
|
|
// Note: business interrupts that occur while cancel is active are absorbed
|
|
// into CancelError (see CancelError doc), so they result in nil (cancel
|
|
// succeeded), NOT ErrExecutionEnded. Only execution that completes with
|
|
// no interrupt at all produces this error.
|
|
ErrExecutionEnded = errors.New("execution already ended")
|
|
|
|
// ErrStreamCanceled is the error sent through the stream when CancelImmediate aborts it.
|
|
// It is a *StreamCanceledError so it can be gob-serialized during checkpoint save
|
|
// (when stored as agentEventWrapper.StreamErr).
|
|
ErrStreamCanceled error = &StreamCanceledError{}
|
|
)
|
|
|
|
// StreamCanceledError is the concrete error type for ErrStreamCanceled.
|
|
// It is exported so that gob can serialize it during checkpoint save when the error
|
|
// is stored in agentEventWrapper.StreamErr.
|
|
type StreamCanceledError struct{}
|
|
|
|
func (e *StreamCanceledError) Error() string {
|
|
return "stream canceled"
|
|
}
|
|
|
|
// WithCancel creates an AgentRunOption that enables cancellation for an agent run.
|
|
// It returns the option to pass to Run/Resume and a cancel function.
|
|
// Cancel options (mode, timeout) are passed to the returned AgentCancelFunc at call time.
|
|
func WithCancel() (AgentRunOption, AgentCancelFunc) {
|
|
cc := newCancelContext()
|
|
opt := WrapImplSpecificOptFn(func(o *options) {
|
|
o.cancelCtx = cc
|
|
})
|
|
cancelFn := cc.buildCancelFunc()
|
|
return opt, cancelFn
|
|
}
|
|
|
|
// cancelContext state constants (for int32 CAS).
|
|
//
|
|
// State transition rules:
|
|
//
|
|
// stateRunning -> stateCancelling (cancel requested by AgentCancelFunc)
|
|
// stateRunning -> stateDone (execution finished without interrupt)
|
|
// stateCancelling -> stateCancelHandled (ANY interrupt absorbed as CancelError)
|
|
// stateCancelling -> stateDone (execution finished without interrupt while cancel pending)
|
|
//
|
|
// Terminal states: stateDone, stateCancelHandled.
|
|
//
|
|
// Note: We intentionally do NOT distinguish between "completed" and "errored"
|
|
// terminal states. End-users get the actual outcome from AgentEvent.
|
|
// This simplification keeps the state machine minimal — only the cancel/non-cancel
|
|
// distinction matters for the AgentCancelFunc return value.
|
|
//
|
|
// Business interrupt handling: when cancel is active (stateCancelling) and any
|
|
// interrupt arrives — cancel-induced OR business — wrapIterWithCancelCtx absorbs
|
|
// it as a CancelError and transitions to stateCancelHandled. The business interrupt
|
|
// data is preserved in the checkpoint for re-emission on resume.
|
|
const (
|
|
// stateRunning is the initial state: agent is executing normally.
|
|
stateRunning int32 = 0
|
|
// stateCancelling means AgentCancelFunc has been called and cancelChan is
|
|
// closed, but the cancel has not yet been handled by the runFunc.
|
|
stateCancelling int32 = 1
|
|
// stateDone means execution has finished through any non-cancel path:
|
|
// normal completion, business interrupt, or error. The specific outcome
|
|
// is conveyed through AgentEvent, not through the cancel state machine.
|
|
stateDone int32 = 2
|
|
// stateCancelHandled means the cancel was processed by the runFunc and a
|
|
// CancelError was emitted through the event stream. This is the success
|
|
// terminal state for cancellation.
|
|
stateCancelHandled int32 = 5
|
|
)
|
|
|
|
// interruptSent constants (for int32 CAS).
|
|
//
|
|
// Transition rules:
|
|
//
|
|
// interruptNotSent -> interruptImmediate (CancelImmediate or escalation)
|
|
const (
|
|
// interruptNotSent means no compose graph interrupt has been sent.
|
|
interruptNotSent int32 = 0
|
|
// interruptImmediate means an immediate graph interrupt was sent with
|
|
// timeout=0, forcing the graph to stop as soon as possible.
|
|
interruptImmediate int32 = 1
|
|
)
|
|
|
|
// defaultCancelImmediateGracePeriod is the bounded time a recursive
|
|
// AgentTool cancel waits before forcing the current level's graph interrupt.
|
|
// This gives deeper AgentTool/internal-agent interrupts a chance to bubble up
|
|
// as CompositeInterrupts. If this proves insufficient for deeply nested
|
|
// structures or too slow for latency-sensitive use cases, consider making it
|
|
// configurable via an AgentCancelOption.
|
|
const defaultCancelImmediateGracePeriod = 1 * time.Second
|
|
|
|
type cancelContextKey struct{}
|
|
|
|
// withCancelContext stores a cancelContext in the Go context.
|
|
func withCancelContext(ctx context.Context, cc *cancelContext) context.Context {
|
|
if cc == nil {
|
|
return ctx
|
|
}
|
|
return context.WithValue(ctx, cancelContextKey{}, cc)
|
|
}
|
|
|
|
// getCancelContext retrieves the cancelContext from the Go context, or nil.
|
|
func getCancelContext(ctx context.Context) *cancelContext {
|
|
if v := ctx.Value(cancelContextKey{}); v != nil {
|
|
return v.(*cancelContext)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type cancelContext struct {
|
|
mode int32 // atomic, CancelMode
|
|
|
|
cancelChan chan struct{} // closed when cancel is requested (all modes, not just safe-point)
|
|
immediateChan chan struct{} // closed when an immediate graph interrupt fires
|
|
doneChan chan struct{} // closed when execution completes (by any mark* method)
|
|
doneOnce sync.Once // ensures doneChan is closed exactly once
|
|
|
|
state int32 // stateRunning, stateCancelling, stateDone, stateCancelHandled
|
|
interruptSent int32 // interruptNotSent, interruptImmediate
|
|
escalated int32 // 1 if escalated from safe-point to immediate
|
|
timeoutEscalated int32 // 1 if escalation was triggered by timeout
|
|
startedMode int32 // atomic, mode when state transitioned to cancelling
|
|
deadlineUnixNano int64 // atomic, 0 means no deadline
|
|
|
|
recursive int32 // atomic; 1 if cancel should propagate into AgentTool internal agents
|
|
recursiveChan chan struct{} // closed when recursive transitions from 0 to 1
|
|
|
|
root bool // true for the original cancelContext created by WithCancel(); false for AgentTool internal agents
|
|
parent *cancelContext // non-nil for AgentTool internal agents; used to propagate AgentTool boundary markers upward
|
|
|
|
agentToolDescendant int32 // atomic; 1 once an AgentTool runs under this cancel context
|
|
|
|
cancelMu sync.Mutex
|
|
timeoutOnce sync.Once
|
|
timeoutNotify chan struct{}
|
|
|
|
mu sync.Mutex
|
|
graphInterruptFuncs []func(...compose.GraphInterruptOption)
|
|
}
|
|
|
|
func newCancelContext() *cancelContext {
|
|
return &cancelContext{
|
|
cancelChan: make(chan struct{}),
|
|
immediateChan: make(chan struct{}),
|
|
doneChan: make(chan struct{}),
|
|
timeoutNotify: make(chan struct{}, 1),
|
|
recursiveChan: make(chan struct{}),
|
|
root: true,
|
|
}
|
|
}
|
|
|
|
func (cc *cancelContext) isRoot() bool {
|
|
return cc != nil && cc.root
|
|
}
|
|
|
|
func (cc *cancelContext) isRecursive() bool {
|
|
return cc != nil && atomic.LoadInt32(&cc.recursive) == 1
|
|
}
|
|
|
|
// setRecursive(false) is a no-op; recursive is monotonically escalating:
|
|
// once set to true, it cannot be reverted.
|
|
func (cc *cancelContext) setRecursive(v bool) {
|
|
if v && atomic.CompareAndSwapInt32(&cc.recursive, 0, 1) {
|
|
close(cc.recursiveChan)
|
|
}
|
|
}
|
|
|
|
// deriveAgentToolCancelContext creates the cancelContext used by an AgentTool's
|
|
// internal agent. It receives recursive cancel propagation from the parent
|
|
// AgentTool call. The caller MUST ensure the child's markDone() is eventually
|
|
// called (e.g., via wrapIterWithCancelCtx's defer) or that ctx is canceled;
|
|
// otherwise the two propagation goroutines will leak.
|
|
func (cc *cancelContext) deriveAgentToolCancelContext(ctx context.Context) *cancelContext {
|
|
if cc == nil {
|
|
return nil
|
|
}
|
|
child := newCancelContext()
|
|
child.root = false
|
|
child.parent = cc
|
|
|
|
// Each goroutine below propagates one signal class (cancel / immediate) to
|
|
// the child. The pattern is a two-phase select:
|
|
// Phase 1: wait for the parent signal (or child/ctx completion).
|
|
// Phase 2: if the signal fired but recursive mode is not active yet,
|
|
// enter a second select waiting for either recursive escalation
|
|
// (recursiveChan) or child/ctx completion. This ensures
|
|
// non-recursive cancels leave children unaware, while a late
|
|
// escalation to recursive still propagates.
|
|
go func() {
|
|
select {
|
|
case <-cc.cancelChan:
|
|
if cc.isRecursive() {
|
|
child.setRecursive(true)
|
|
child.triggerCancel(cc.getMode())
|
|
return
|
|
}
|
|
select {
|
|
case <-cc.recursiveChan:
|
|
child.setRecursive(true)
|
|
child.triggerCancel(cc.getMode())
|
|
case <-child.doneChan:
|
|
case <-ctx.Done():
|
|
}
|
|
case <-child.doneChan:
|
|
case <-ctx.Done():
|
|
}
|
|
}()
|
|
|
|
go func() {
|
|
select {
|
|
case <-cc.immediateChan:
|
|
if cc.isRecursive() {
|
|
child.setRecursive(true)
|
|
child.triggerImmediateCancel()
|
|
return
|
|
}
|
|
select {
|
|
case <-cc.recursiveChan:
|
|
child.setRecursive(true)
|
|
child.triggerImmediateCancel()
|
|
case <-child.doneChan:
|
|
case <-ctx.Done():
|
|
}
|
|
case <-child.doneChan:
|
|
case <-ctx.Done():
|
|
}
|
|
}()
|
|
|
|
return child
|
|
}
|
|
|
|
func (cc *cancelContext) triggerCancel(mode CancelMode) {
|
|
cc.setMode(mode)
|
|
if atomic.CompareAndSwapInt32(&cc.state, stateRunning, stateCancelling) {
|
|
close(cc.cancelChan)
|
|
}
|
|
}
|
|
|
|
func (cc *cancelContext) triggerImmediateCancel() {
|
|
atomic.StoreInt32(&cc.escalated, 1)
|
|
cc.setMode(CancelImmediate)
|
|
if atomic.CompareAndSwapInt32(&cc.state, stateRunning, stateCancelling) {
|
|
close(cc.cancelChan)
|
|
}
|
|
cc.sendImmediateInterrupt()
|
|
}
|
|
|
|
func (cc *cancelContext) getMode() CancelMode {
|
|
if cc == nil {
|
|
return CancelImmediate
|
|
}
|
|
return CancelMode(atomic.LoadInt32(&cc.mode))
|
|
}
|
|
|
|
func (cc *cancelContext) setMode(mode CancelMode) {
|
|
atomic.StoreInt32(&cc.mode, int32(mode))
|
|
}
|
|
|
|
func (cc *cancelContext) getDeadlineUnixNano() int64 {
|
|
return atomic.LoadInt64(&cc.deadlineUnixNano)
|
|
}
|
|
|
|
func (cc *cancelContext) setDeadlineUnixNano(v int64) {
|
|
atomic.StoreInt64(&cc.deadlineUnixNano, v)
|
|
}
|
|
|
|
func (cc *cancelContext) wakeTimeoutController() {
|
|
select {
|
|
case cc.timeoutNotify <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// shouldCancel returns true if a cancel has been requested (cancelChan is closed).
|
|
func (cc *cancelContext) shouldCancel() bool {
|
|
if cc == nil {
|
|
return false
|
|
}
|
|
select {
|
|
case <-cc.cancelChan:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// isImmediateCancelled returns true if an immediate graph interrupt has been
|
|
// fired (CancelImmediate or timeout escalation). This is stronger than
|
|
// shouldCancel: it means the compose graph is being torn down right now and
|
|
// orphaned goroutines should not attempt to send events.
|
|
func (cc *cancelContext) isImmediateCancelled() bool {
|
|
if cc == nil {
|
|
return false
|
|
}
|
|
select {
|
|
case <-cc.immediateChan:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// sendImmediateInterrupt sends the compose graph interrupt signal via graphInterruptFuncs.
|
|
// Also closes immediateChan (used by cancelMonitoredModel to abort an in-progress stream).
|
|
// Returns false if an interrupt was already sent or if no graphInterruptFuncs have been
|
|
// registered yet (the deferred fire in setGraphInterruptFunc will handle that case).
|
|
func (cc *cancelContext) sendImmediateInterrupt() bool {
|
|
cc.mu.Lock()
|
|
|
|
if !atomic.CompareAndSwapInt32(&cc.interruptSent, interruptNotSent, interruptImmediate) {
|
|
cc.mu.Unlock()
|
|
return false
|
|
}
|
|
|
|
close(cc.immediateChan)
|
|
|
|
fns := make([]func(...compose.GraphInterruptOption), len(cc.graphInterruptFuncs))
|
|
copy(fns, cc.graphInterruptFuncs)
|
|
|
|
if cc.isRecursive() && cc.hasAgentToolDescendant() {
|
|
select {
|
|
case <-cc.doneChan:
|
|
cc.mu.Unlock()
|
|
return true
|
|
case <-time.After(defaultCancelImmediateGracePeriod):
|
|
}
|
|
}
|
|
|
|
if len(fns) == 0 {
|
|
cc.mu.Unlock()
|
|
return false
|
|
}
|
|
|
|
for _, fn := range fns {
|
|
fn(compose.WithGraphInterruptTimeout(0))
|
|
}
|
|
cc.mu.Unlock()
|
|
return true
|
|
}
|
|
|
|
// setGraphInterruptFunc appends a graph interrupt function to the list.
|
|
// If an immediate cancel was already requested, fires it retroactively.
|
|
// Multiple functions can be registered (e.g. one per parallel sub-agent).
|
|
//
|
|
// Both this method and sendImmediateInterrupt hold cc.mu across the entire
|
|
// check-and-fire sequence, ensuring each interrupt function is called exactly
|
|
// once (compose.WithGraphInterrupt returns a non-idempotent closure that panics
|
|
// on double-call).
|
|
func (cc *cancelContext) setGraphInterruptFunc(interrupt func(...compose.GraphInterruptOption)) {
|
|
cc.mu.Lock()
|
|
cc.graphInterruptFuncs = append(cc.graphInterruptFuncs, interrupt)
|
|
|
|
shouldFire := atomic.LoadInt32(&cc.interruptSent) == interruptImmediate
|
|
if shouldFire {
|
|
interrupt(compose.WithGraphInterruptTimeout(0))
|
|
}
|
|
cc.mu.Unlock()
|
|
}
|
|
|
|
// markDone marks the execution as finished through any non-cancel path
|
|
// (normal completion, business interrupt, or error).
|
|
// This is safe to call even if a cancel is in progress — it allows the
|
|
// cancel func to detect that execution finished before cancel took effect.
|
|
func (cc *cancelContext) markDone() {
|
|
if cc == nil {
|
|
return
|
|
}
|
|
if atomic.CompareAndSwapInt32(&cc.state, stateRunning, stateDone) ||
|
|
atomic.CompareAndSwapInt32(&cc.state, stateCancelling, stateDone) {
|
|
cc.doneOnce.Do(func() { close(cc.doneChan) })
|
|
}
|
|
}
|
|
|
|
func (cc *cancelContext) hasAgentToolDescendant() bool {
|
|
return cc != nil && atomic.LoadInt32(&cc.agentToolDescendant) == 1
|
|
}
|
|
|
|
func (cc *cancelContext) markAgentToolDescendant() {
|
|
for cur := cc; cur != nil; cur = cur.parent {
|
|
atomic.StoreInt32(&cur.agentToolDescendant, 1)
|
|
}
|
|
}
|
|
|
|
// markCancelHandled signals that the cancel path in the runFunc has created
|
|
// and sent a CancelError. Transitions state to stateCancelHandled so that:
|
|
// 1. The deferred markDone() becomes a no-op (CAS from cancelling fails).
|
|
// 2. buildCancelFunc sees stateCancelHandled and returns nil (cancel succeeded).
|
|
// Returns true if the transition succeeded, false if cancel was already handled
|
|
// (e.g., by a sub-agent). This prevents duplicate CancelError emission.
|
|
func (cc *cancelContext) markCancelHandled() bool {
|
|
if cc == nil {
|
|
return false
|
|
}
|
|
if atomic.CompareAndSwapInt32(&cc.state, stateCancelling, stateCancelHandled) {
|
|
cc.doneOnce.Do(func() { close(cc.doneChan) })
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// createCancelError creates a CancelError based on the current cancel state.
|
|
func (cc *cancelContext) createCancelError() *CancelError {
|
|
info := &AgentCancelInfo{}
|
|
info.Mode = cc.getMode()
|
|
if atomic.LoadInt32(&cc.escalated) == 1 {
|
|
info.Escalated = true
|
|
info.Timeout = atomic.LoadInt32(&cc.timeoutEscalated) == 1
|
|
}
|
|
return &CancelError{
|
|
Info: info,
|
|
}
|
|
}
|
|
|
|
func (cc *cancelContext) createAndMarkCancelHandled() (*CancelError, bool) {
|
|
cc.cancelMu.Lock()
|
|
defer cc.cancelMu.Unlock()
|
|
cancelErr := cc.createCancelError()
|
|
ok := cc.markCancelHandled()
|
|
return cancelErr, ok
|
|
}
|
|
|
|
// buildCancelFunc builds the AgentCancelFunc for external use.
|
|
func (cc *cancelContext) buildCancelFunc() AgentCancelFunc {
|
|
joinMode := func(a, b CancelMode) CancelMode {
|
|
if a == CancelImmediate || b == CancelImmediate {
|
|
return CancelImmediate
|
|
}
|
|
return a | b
|
|
}
|
|
|
|
parseReq := func(callOpts ...AgentCancelOption) *agentCancelConfig {
|
|
cfg := &agentCancelConfig{Mode: CancelImmediate}
|
|
for _, opt := range callOpts {
|
|
opt(cfg)
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
startTimeoutController := func() {
|
|
cc.timeoutOnce.Do(func() {
|
|
go func() {
|
|
for {
|
|
select {
|
|
case <-cc.doneChan:
|
|
return
|
|
default:
|
|
}
|
|
|
|
mode := cc.getMode()
|
|
if mode == CancelImmediate {
|
|
return
|
|
}
|
|
|
|
deadline := cc.getDeadlineUnixNano()
|
|
if deadline == 0 {
|
|
select {
|
|
case <-cc.timeoutNotify:
|
|
continue
|
|
case <-cc.doneChan:
|
|
return
|
|
}
|
|
}
|
|
|
|
now := time.Now().UnixNano()
|
|
wait := time.Duration(deadline - now)
|
|
if wait <= 0 {
|
|
atomic.StoreInt32(&cc.escalated, 1)
|
|
atomic.StoreInt32(&cc.timeoutEscalated, 1)
|
|
cc.sendImmediateInterrupt()
|
|
return
|
|
}
|
|
|
|
timer := time.NewTimer(wait)
|
|
select {
|
|
case <-timer.C:
|
|
timer.Stop()
|
|
atomic.StoreInt32(&cc.escalated, 1)
|
|
atomic.StoreInt32(&cc.timeoutEscalated, 1)
|
|
cc.sendImmediateInterrupt()
|
|
return
|
|
case <-cc.timeoutNotify:
|
|
timer.Stop()
|
|
continue
|
|
case <-cc.doneChan:
|
|
timer.Stop()
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
})
|
|
}
|
|
|
|
newHandle := func(wait func() error) *CancelHandle {
|
|
return &CancelHandle{wait: wait}
|
|
}
|
|
|
|
waitForCompletion := func() error {
|
|
<-cc.doneChan
|
|
|
|
st := atomic.LoadInt32(&cc.state)
|
|
switch st {
|
|
case stateDone:
|
|
return ErrExecutionEnded
|
|
default:
|
|
if atomic.LoadInt32(&cc.timeoutEscalated) == 1 {
|
|
return ErrCancelTimeout
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return func(callOpts ...AgentCancelOption) (*CancelHandle, bool) {
|
|
req := parseReq(callOpts...)
|
|
|
|
st := atomic.LoadInt32(&cc.state)
|
|
switch st {
|
|
case stateCancelHandled:
|
|
return newHandle(func() error { return nil }), false
|
|
case stateDone:
|
|
return newHandle(func() error { return ErrExecutionEnded }), false
|
|
}
|
|
|
|
var needImmediate, needTimeoutCtl bool
|
|
|
|
cc.cancelMu.Lock()
|
|
|
|
st = atomic.LoadInt32(&cc.state)
|
|
switch st {
|
|
case stateCancelHandled:
|
|
cc.cancelMu.Unlock()
|
|
return newHandle(func() error { return nil }), false
|
|
case stateDone:
|
|
cc.cancelMu.Unlock()
|
|
return newHandle(func() error { return ErrExecutionEnded }), false
|
|
}
|
|
|
|
curMode := cc.getMode()
|
|
if st == stateRunning {
|
|
if !atomic.CompareAndSwapInt32(&cc.state, stateRunning, stateCancelling) {
|
|
st = atomic.LoadInt32(&cc.state)
|
|
cc.cancelMu.Unlock()
|
|
if st == stateDone {
|
|
return newHandle(func() error { return ErrExecutionEnded }), false
|
|
}
|
|
return newHandle(waitForCompletion), true
|
|
}
|
|
|
|
curMode = req.Mode
|
|
cc.setMode(curMode)
|
|
atomic.StoreInt32(&cc.startedMode, int32(curMode))
|
|
cc.setRecursive(req.Recursive)
|
|
close(cc.cancelChan)
|
|
} else {
|
|
// Recursive is monotonic: once set, cannot be unset. The first
|
|
// cancel call uses the bool directly; subsequent calls only
|
|
// escalate (false → true) — setRecursive(false) is a no-op.
|
|
curMode = joinMode(curMode, req.Mode)
|
|
cc.setMode(curMode)
|
|
if req.Recursive {
|
|
cc.setRecursive(true)
|
|
}
|
|
}
|
|
|
|
if curMode == CancelImmediate {
|
|
cc.setDeadlineUnixNano(0)
|
|
needImmediate = true
|
|
} else if req.Timeout != nil && *req.Timeout > 0 {
|
|
proposed := time.Now().Add(*req.Timeout).UnixNano()
|
|
existing := cc.getDeadlineUnixNano()
|
|
if existing == 0 || proposed < existing {
|
|
cc.setDeadlineUnixNano(proposed)
|
|
cc.wakeTimeoutController()
|
|
}
|
|
needTimeoutCtl = cc.getDeadlineUnixNano() != 0
|
|
}
|
|
|
|
cc.cancelMu.Unlock()
|
|
|
|
if needImmediate {
|
|
if atomic.LoadInt32(&cc.startedMode) != int32(CancelImmediate) {
|
|
atomic.StoreInt32(&cc.escalated, 1)
|
|
}
|
|
cc.sendImmediateInterrupt()
|
|
}
|
|
if needTimeoutCtl {
|
|
startTimeoutController()
|
|
}
|
|
|
|
return newHandle(waitForCompletion), true
|
|
}
|
|
}
|
|
|
|
// wrapIterWithCancelCtx wraps an iterator with cancel lifecycle management.
|
|
// It calls markDone when the inner iterator is fully drained, ensuring the
|
|
// cancelContext's doneChan is closed and propagation goroutines can exit.
|
|
//
|
|
// For root cancelContexts (created by WithCancel, not deriveAgentToolCancelContext), it also
|
|
// converts interrupt ACTION events to CancelError when cancel is active.
|
|
// This is the single point of interrupt-to-CancelError conversion in the
|
|
// system — Runner.handleIter only enriches the resulting CancelError with
|
|
// checkpoint metadata.
|
|
//
|
|
// Interrupt absorption: ALL interrupts are converted when cancel is active,
|
|
// including business interrupts (compose.Interrupt from user code). Cancel and
|
|
// business interrupts cannot be reliably distinguished in concurrent execution
|
|
// (parallel workflows, concurrent tool calls) where they merge into a single
|
|
// composite signal. The business interrupt data is preserved in the checkpoint
|
|
// and re-fires naturally on resume.
|
|
//
|
|
// This conversion MUST happen in this wrapper (not deferred to Runner.handleIter)
|
|
// because markDone runs as a defer in this goroutine — if the interrupt event
|
|
// were passed through unconverted, markDone would transition stateCancelling→stateDone
|
|
// before the Runner goroutine could call createAndMarkCancelHandled, causing it
|
|
// to fail the CAS.
|
|
func wrapIterWithCancelCtx[M MessageType](iter *AsyncIterator[*TypedAgentEvent[M]], cancelCtx *cancelContext) *AsyncIterator[*TypedAgentEvent[M]] {
|
|
if cancelCtx == nil {
|
|
return iter
|
|
}
|
|
it, gen := NewAsyncIteratorPair[*TypedAgentEvent[M]]()
|
|
go func() {
|
|
defer cancelCtx.markDone()
|
|
defer gen.Close()
|
|
for {
|
|
event, ok := iter.Next()
|
|
if !ok {
|
|
break
|
|
}
|
|
|
|
if cancelCtx.isRoot() && event.Action != nil && event.Action.internalInterrupted != nil {
|
|
if cancelCtx.shouldCancel() {
|
|
cancelErr, ok := cancelCtx.createAndMarkCancelHandled()
|
|
if ok {
|
|
cancelErr.interruptSignal = event.Action.internalInterrupted
|
|
gen.Send(&TypedAgentEvent[M]{Err: cancelErr})
|
|
}
|
|
return
|
|
}
|
|
}
|
|
|
|
gen.Send(event)
|
|
}
|
|
}()
|
|
return it
|
|
}
|
|
|
|
// typedCancelMonitoredModel wraps a model with cancel monitoring.
|
|
// Generate: pure delegate to the inner model (CancelAfterChatModel is handled
|
|
// by a dedicated node after the ChatModel in the compose graph).
|
|
// Stream: pipes chunks through a goroutine that selects on immediateChan for
|
|
// CancelImmediate abort.
|
|
type typedCancelMonitoredModel[M MessageType] struct {
|
|
inner model.BaseModel[M]
|
|
cancelContext *cancelContext
|
|
}
|
|
|
|
type recvResult[T any] struct {
|
|
data T
|
|
err error
|
|
}
|
|
|
|
func (m *typedCancelMonitoredModel[M]) Generate(ctx context.Context, input []M, opts ...model.Option) (M, error) {
|
|
return m.inner.Generate(ctx, input, opts...)
|
|
}
|
|
|
|
func (m *typedCancelMonitoredModel[M]) Stream(ctx context.Context, input []M, opts ...model.Option) (*schema.StreamReader[M], error) {
|
|
stream, err := m.inner.Stream(ctx, input, opts...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
wrapped := wrapStreamWithCancelMonitoring(stream, m.cancelContext)
|
|
return wrapped, nil
|
|
}
|
|
|
|
// wrapStreamWithCancelMonitoring wraps a stream with cancel monitoring.
|
|
// When immediateChan fires (CancelImmediate or timeout escalation), the output
|
|
// stream is terminated with ErrStreamCanceled.
|
|
func wrapStreamWithCancelMonitoring[T any](stream *schema.StreamReader[T], cc *cancelContext) *schema.StreamReader[T] {
|
|
if cc == nil {
|
|
return stream
|
|
}
|
|
|
|
// Already canceled — terminate immediately
|
|
select {
|
|
case <-cc.immediateChan:
|
|
stream.Close()
|
|
r, w := schema.Pipe[T](1)
|
|
var zero T
|
|
w.Send(zero, ErrStreamCanceled)
|
|
w.Close()
|
|
return r
|
|
default:
|
|
}
|
|
|
|
reader, writer := schema.Pipe[T](1)
|
|
|
|
go func() {
|
|
done := make(chan struct{})
|
|
defer close(done)
|
|
defer writer.Close()
|
|
defer stream.Close()
|
|
|
|
ch := make(chan recvResult[T])
|
|
go func() {
|
|
defer close(ch)
|
|
for {
|
|
chunk, recvErr := stream.Recv()
|
|
select {
|
|
case ch <- recvResult[T]{chunk, recvErr}:
|
|
case <-done:
|
|
return
|
|
}
|
|
if recvErr != nil {
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
for {
|
|
select {
|
|
case <-cc.immediateChan:
|
|
var zero T
|
|
writer.Send(zero, ErrStreamCanceled)
|
|
return
|
|
|
|
case r, ok := <-ch:
|
|
if !ok {
|
|
return
|
|
}
|
|
if r.err != nil {
|
|
if r.err == io.EOF {
|
|
return
|
|
}
|
|
var zero T
|
|
writer.Send(zero, r.err)
|
|
return
|
|
}
|
|
if closed := writer.Send(r.data, nil); closed {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
return reader
|
|
}
|
|
|
|
// cancelMonitoredToolHandler wraps streamable tool calls with cancel monitoring.
|
|
// When CancelImmediate fires, the tool output stream is terminated with ErrStreamCanceled.
|
|
// This handler reads the cancelContext from the Go context via getCancelContext.
|
|
type cancelMonitoredToolHandler struct{}
|
|
|
|
func (h *cancelMonitoredToolHandler) WrapStreamableToolCall(next compose.StreamableToolEndpoint) compose.StreamableToolEndpoint {
|
|
return func(ctx context.Context, input *compose.ToolInput) (*compose.StreamToolOutput, error) {
|
|
output, err := next(ctx, input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
cc := getCancelContext(ctx)
|
|
if cc == nil {
|
|
return output, nil
|
|
}
|
|
|
|
wrapped := wrapStreamWithCancelMonitoring(output.Result, cc)
|
|
return &compose.StreamToolOutput{Result: wrapped}, nil
|
|
}
|
|
}
|
|
|
|
func (h *cancelMonitoredToolHandler) WrapEnhancedStreamableToolCall(next compose.EnhancedStreamableToolEndpoint) compose.EnhancedStreamableToolEndpoint {
|
|
return func(ctx context.Context, input *compose.ToolInput) (*compose.EnhancedStreamableToolOutput, error) {
|
|
output, err := next(ctx, input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
cc := getCancelContext(ctx)
|
|
if cc == nil {
|
|
return output, nil
|
|
}
|
|
|
|
wrapped := wrapStreamWithCancelMonitoring(output.Result, cc)
|
|
return &compose.EnhancedStreamableToolOutput{Result: wrapped}, nil
|
|
}
|
|
}
|