Add opt-in retries for agent tool calls

agent/builtin.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"time"
 
 	"go-micro.dev/v6/ai"
 	codecBytes "go-micro.dev/v6/codec/bytes"
@@ -121,6 +122,7 @@ func (a *agentImpl) toolHandler() ai.ToolHandler {
 	// so the result runs plan → step → loop → approve → checkpoint → base.
 	h := a.baseHandler()
 	h = a.toolTimeoutWrap(h)
+	h = a.toolRetryWrap(h)
 	h = a.checkpointToolWrap(h)
 	h = a.approveWrap(h)
 	h = a.loopWrap(h)
@@ -164,6 +166,99 @@ func (a *agentImpl) toolTimeoutWrap(next ai.ToolHandler) ai.ToolHandler {
 	}
 }
 
+// toolRetryWrap retries transient tool failures with bounded backoff. It is
+// opt-in because tools can have side effects; guardrail refusals and caller
+// cancellation are never retried.
+func (a *agentImpl) toolRetryWrap(next ai.ToolHandler) ai.ToolHandler {
+	return func(ctx context.Context, call ai.ToolCall) ai.ToolResult {
+		maxAttempts := a.opts.ToolMaxAttempts
+		if maxAttempts <= 0 {
+			maxAttempts = 1
+		}
+
+		var res ai.ToolResult
+		for attempt := 1; attempt <= maxAttempts; attempt++ {
+			if err := ctx.Err(); err != nil {
+				return errResult(call.ID, err.Error())
+			}
+			res = next(ctx, call)
+			if !retryableToolResult(res) || attempt == maxAttempts || ctx.Err() != nil {
+				return annotateToolAttempts(res, attempt)
+			}
+
+			t := time.NewTimer(toolRetryBackoff(attempt, a.opts.ToolRetryBackoff))
+			select {
+			case <-ctx.Done():
+				if !t.Stop() {
+					<-t.C
+				}
+				return errResult(call.ID, ctx.Err().Error())
+			case <-t.C:
+			}
+		}
+		return annotateToolAttempts(res, maxAttempts)
+	}
+}
+
+func retryableToolResult(res ai.ToolResult) bool {
+	if res.Refused != "" {
+		return false
+	}
+	msg := toolErrorMessage(res)
+	if msg == "" {
+		return false
+	}
+	return ai.IsTransientError(fmt.Errorf("%s", msg))
+}
+
+func toolErrorMessage(res ai.ToolResult) string {
+	if m, ok := res.Value.(map[string]string); ok {
+		return m["error"]
+	}
+	if m, ok := res.Value.(map[string]any); ok {
+		if v, ok := m["error"].(string); ok {
+			return v
+		}
+	}
+	var decoded map[string]string
+	if err := json.Unmarshal([]byte(res.Content), &decoded); err == nil {
+		return decoded["error"]
+	}
+	return ""
+}
+
+func annotateToolAttempts(res ai.ToolResult, attempts int) ai.ToolResult {
+	if attempts <= 1 {
+		return res
+	}
+	res.Attempts = attempts
+	if m, ok := res.Value.(map[string]string); ok {
+		cp := map[string]any{}
+		for k, v := range m {
+			cp[k] = v
+		}
+		cp["attempts"] = attempts
+		res.Value = cp
+		if b, err := json.Marshal(cp); err == nil {
+			res.Content = string(b)
+		}
+	}
+	return res
+}
+
+func toolRetryBackoff(attempt int, base time.Duration) time.Duration {
+	if base <= 0 {
+		base = 200 * time.Millisecond
+	}
+	if shift := attempt - 1; shift > 0 {
+		base <<= shift
+	}
+	if base > 30*time.Second {
+		return 30 * time.Second
+	}
+	return base
+}
+
 // baseHandler executes a tool call: a developer custom tool, the built-in
 // delegate, or an RPC to the service. It is the innermost handler.
 func (a *agentImpl) baseHandler() ai.ToolHandler {
@@ -338,6 +433,7 @@ func (a *agentImpl) handleDelegate(ctx context.Context, call ai.ToolCall) ai.Too
 		ModelCallTimeout(a.opts.ModelTimeout),
 		ModelRetry(a.opts.ModelMaxAttempts, a.opts.ModelRetryBackoff),
 		ToolCallTimeout(a.opts.ToolTimeout),
+		ToolRetry(a.opts.ToolMaxAttempts, a.opts.ToolRetryBackoff),
 		TraceProvider(a.opts.TraceProvider),
 	)
 	// Record lineage so the sub-agent's tool calls carry this run as parent.

agent/options.go

@@ -60,6 +60,11 @@ type Options struct {
 	// applied before custom tools, delegate, and service RPC calls so context
 	// deadlines propagate consistently through the agent loop.
 	ToolTimeout time.Duration
+	// ToolMaxAttempts bounds tool execution attempts including the first call.
+	// Default 1; retries are opt-in because tools can have side effects.
+	ToolMaxAttempts int
+	// ToolRetryBackoff is the base delay between transient tool failures.
+	ToolRetryBackoff time.Duration
 
 	// Memory is the agent's conversation memory. Nil = the default
 	// store-backed memory (durable across restarts).
@@ -121,6 +126,8 @@ func newOptions(opts ...Option) Options {
 		ModelMaxAttempts:  1, // retries opt-in via ModelRetry (see field doc)
 		ModelRetryBackoff: 100 * time.Millisecond,
 		ToolTimeout:       30 * time.Second,
+		ToolMaxAttempts:   1,
+		ToolRetryBackoff:  100 * time.Millisecond,
 		// On by default and lenient: identical repeated calls are a
 		// no-progress loop, never useful. Set LoopLimit(0) to disable.
 		LoopLimit: 3,
@@ -229,6 +236,16 @@ func ModelRetry(maxAttempts int, backoff time.Duration) Option {
 	}
 }
 
+// ToolRetry sets the tool retry budget and backoff for transient failures.
+// Attempts include the first call. Retries are opt-in because tools may have
+// side effects; keep handlers idempotent before enabling this.
+func ToolRetry(maxAttempts int, backoff time.Duration) Option {
+	return func(o *Options) {
+		o.ToolMaxAttempts = maxAttempts
+		o.ToolRetryBackoff = backoff
+	}
+}
+
 // WithA2A makes Run serve the agent over the A2A protocol on addr (e.g.
 // ":4000"), so other agents can reach it directly by URL without a
 // separate gateway. The agent stays a normal go-micro service as well;

agent/resilience_test.go

@@ -184,3 +184,48 @@ type testStatusError struct {
 func (e testStatusError) Error() string { return "provider status error" }
 
 func (e testStatusError) StatusCode() int { return e.code }
+
+func TestToolRetryRetriesTransientToolErrorsThenSucceeds(t *testing.T) {
+	attempts := 0
+	a := newTestAgent(
+		Name("tool-retry-success"),
+		ToolRetry(3, time.Millisecond),
+		WithTool("flaky", "flaky tool", nil, func(context.Context, map[string]any) (string, error) {
+			attempts++
+			if attempts < 3 {
+				return "", context.DeadlineExceeded
+			}
+			return "ok", nil
+		}),
+	)
+
+	content := toolContent(a.toolHandler(), "flaky", nil)
+	if content != "ok" {
+		t.Fatalf("tool result = %q, want ok", content)
+	}
+	if attempts != 3 {
+		t.Fatalf("attempts = %d, want 3", attempts)
+	}
+}
+
+func TestToolRetryDoesNotRetryGuardrailRefusals(t *testing.T) {
+	attempts := 0
+	a := newTestAgent(
+		Name("tool-retry-refusal"),
+		MaxSteps(1),
+		ToolRetry(3, time.Millisecond),
+		WithTool("counted", "counted tool", nil, func(context.Context, map[string]any) (string, error) {
+			attempts++
+			return "ok", nil
+		}),
+	)
+	h := a.toolHandler()
+	_ = toolContent(h, "counted", nil)
+	content := toolContent(h, "counted", nil)
+	if !strings.Contains(content, "step limit reached") {
+		t.Fatalf("tool result = %q, want step-limit refusal", content)
+	}
+	if attempts != 1 {
+		t.Fatalf("attempts = %d, want only the allowed tool call to execute", attempts)
+	}
+}

ai/model.go

@@ -93,9 +93,10 @@ func (c ToolCall) Scan(v any) error {
 
 // ToolResult represents the result of a tool execution
 type ToolResult struct {
-	ID      string // Tool call ID (for correlation)
-	Value   any    // Structured result (optional)
-	Content string // Tool execution result (JSON string), shown to the model
+	ID       string // Tool call ID (for correlation)
+	Value    any    // Structured result (optional)
+	Content  string // Tool execution result (JSON string), shown to the model
+	Attempts int    `json:"attempts,omitempty"` // Tool execution attempts, set when retried.
 	// Refused names the reason a guardrail blocked the call before it ran
 	// ("max_steps", "loop", "approval"); empty when the call executed. A
 	// tool wrapper can switch on it to build reliability tooling — react to

micro.go

@@ -116,6 +116,24 @@ func AgentLoopLimit(n int) AgentOption { return agent.LoopLimit(n) }
 // each action the agent takes.
 func AgentApproveTool(fn ApproveFunc) AgentOption { return agent.ApproveTool(fn) }
 
+// AgentModelCallTimeout sets the timeout for each provider Generate call.
+func AgentModelCallTimeout(d time.Duration) AgentOption { return agent.ModelCallTimeout(d) }
+
+// AgentModelRetry sets the provider retry budget and backoff for transient failures.
+func AgentModelRetry(maxAttempts int, backoff time.Duration) AgentOption {
+	return agent.ModelRetry(maxAttempts, backoff)
+}
+
+// AgentToolCallTimeout sets the timeout for each agent tool execution.
+func AgentToolCallTimeout(d time.Duration) AgentOption { return agent.ToolCallTimeout(d) }
+
+// AgentToolRetry sets the tool retry budget and backoff for transient failures.
+// Attempts include the first call. Retries are opt-in because tools can have
+// side effects; keep handlers idempotent before enabling this.
+func AgentToolRetry(maxAttempts int, backoff time.Duration) AgentOption {
+	return agent.ToolRetry(maxAttempts, backoff)
+}
+
 // Memory is an agent's pluggable conversation memory.
 type Memory = agent.Memory