harness: label provider conformance phases

* atlascloud: env-selectable chat model; run conformance on a stronger model

The daily provider-conformance harness fails 4/5 harnesses on Atlas Cloud —
its default chat model answers agent/tool-use conformance prompts
conversationally instead of performing the task. Atlas is currently the only
provider with a key configured, so the whole live run is red.

Make the Atlas Cloud provider honor an ATLASCLOUD_MODEL env override (falling
back to the existing default), and set it in the harness workflow to a
stronger tool-use model (Qwen3, overridable via an Actions variable). No
change to the default for normal use.

* atlascloud: use minimaxai/minimax-m3 for conformance model

---------

Co-authored-by: Claude <noreply@anthropic.com>

.github/workflows/harness.yml

@@ -67,6 +67,11 @@ jobs:
           MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
           TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
           ATLASCLOUD_API_KEY: ${{ secrets.ATLASCLOUD_API_KEY }}
+          # Atlas Cloud's default chat model was failing the agent/tool-use
+          # conformance harnesses; run it against a stronger tool-use model.
+          # Override with an Actions variable ATLASCLOUD_MODEL if the exact
+          # catalog id differs (Atlas uses org/model ids).
+          ATLASCLOUD_MODEL: ${{ vars.ATLASCLOUD_MODEL || 'minimaxai/minimax-m3' }}
         run: |
           PROVIDERS="${{ github.event.inputs.providers || 'anthropic,openai,gemini,groq,mistral,together,atlascloud' }}"
           HARNESSES="${{ github.event.inputs.harnesses || 'agent,universe,agent-flow,plan-delegate,a2a-stream-fallback' }}"

agent/builtin.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"time"
 
 	"go-micro.dev/v6/ai"
 	codecBytes "go-micro.dev/v6/codec/bytes"
@@ -121,6 +122,7 @@ func (a *agentImpl) toolHandler() ai.ToolHandler {
 	// so the result runs plan → step → loop → approve → checkpoint → base.
 	h := a.baseHandler()
 	h = a.toolTimeoutWrap(h)
+	h = a.toolRetryWrap(h)
 	h = a.checkpointToolWrap(h)
 	h = a.approveWrap(h)
 	h = a.loopWrap(h)
@@ -164,6 +166,99 @@ func (a *agentImpl) toolTimeoutWrap(next ai.ToolHandler) ai.ToolHandler {
 	}
 }
 
+// toolRetryWrap retries transient tool failures with bounded backoff. It is
+// opt-in because tools can have side effects; guardrail refusals and caller
+// cancellation are never retried.
+func (a *agentImpl) toolRetryWrap(next ai.ToolHandler) ai.ToolHandler {
+	return func(ctx context.Context, call ai.ToolCall) ai.ToolResult {
+		maxAttempts := a.opts.ToolMaxAttempts
+		if maxAttempts <= 0 {
+			maxAttempts = 1
+		}
+
+		var res ai.ToolResult
+		for attempt := 1; attempt <= maxAttempts; attempt++ {
+			if err := ctx.Err(); err != nil {
+				return errResult(call.ID, err.Error())
+			}
+			res = next(ctx, call)
+			if !retryableToolResult(res) || attempt == maxAttempts || ctx.Err() != nil {
+				return annotateToolAttempts(res, attempt)
+			}
+
+			t := time.NewTimer(toolRetryBackoff(attempt, a.opts.ToolRetryBackoff))
+			select {
+			case <-ctx.Done():
+				if !t.Stop() {
+					<-t.C
+				}
+				return errResult(call.ID, ctx.Err().Error())
+			case <-t.C:
+			}
+		}
+		return annotateToolAttempts(res, maxAttempts)
+	}
+}
+
+func retryableToolResult(res ai.ToolResult) bool {
+	if res.Refused != "" {
+		return false
+	}
+	msg := toolErrorMessage(res)
+	if msg == "" {
+		return false
+	}
+	return ai.IsTransientError(fmt.Errorf("%s", msg))
+}
+
+func toolErrorMessage(res ai.ToolResult) string {
+	if m, ok := res.Value.(map[string]string); ok {
+		return m["error"]
+	}
+	if m, ok := res.Value.(map[string]any); ok {
+		if v, ok := m["error"].(string); ok {
+			return v
+		}
+	}
+	var decoded map[string]string
+	if err := json.Unmarshal([]byte(res.Content), &decoded); err == nil {
+		return decoded["error"]
+	}
+	return ""
+}
+
+func annotateToolAttempts(res ai.ToolResult, attempts int) ai.ToolResult {
+	if attempts <= 1 {
+		return res
+	}
+	res.Attempts = attempts
+	if m, ok := res.Value.(map[string]string); ok {
+		cp := map[string]any{}
+		for k, v := range m {
+			cp[k] = v
+		}
+		cp["attempts"] = attempts
+		res.Value = cp
+		if b, err := json.Marshal(cp); err == nil {
+			res.Content = string(b)
+		}
+	}
+	return res
+}
+
+func toolRetryBackoff(attempt int, base time.Duration) time.Duration {
+	if base <= 0 {
+		base = 200 * time.Millisecond
+	}
+	if shift := attempt - 1; shift > 0 {
+		base <<= shift
+	}
+	if base > 30*time.Second {
+		return 30 * time.Second
+	}
+	return base
+}
+
 // baseHandler executes a tool call: a developer custom tool, the built-in
 // delegate, or an RPC to the service. It is the innermost handler.
 func (a *agentImpl) baseHandler() ai.ToolHandler {
@@ -338,6 +433,7 @@ func (a *agentImpl) handleDelegate(ctx context.Context, call ai.ToolCall) ai.Too
 		ModelCallTimeout(a.opts.ModelTimeout),
 		ModelRetry(a.opts.ModelMaxAttempts, a.opts.ModelRetryBackoff),
 		ToolCallTimeout(a.opts.ToolTimeout),
+		ToolRetry(a.opts.ToolMaxAttempts, a.opts.ToolRetryBackoff),
 		TraceProvider(a.opts.TraceProvider),
 	)
 	// Record lineage so the sub-agent's tool calls carry this run as parent.

agent/options.go

@@ -60,6 +60,11 @@ type Options struct {
 	// applied before custom tools, delegate, and service RPC calls so context
 	// deadlines propagate consistently through the agent loop.
 	ToolTimeout time.Duration
+	// ToolMaxAttempts bounds tool execution attempts including the first call.
+	// Default 1; retries are opt-in because tools can have side effects.
+	ToolMaxAttempts int
+	// ToolRetryBackoff is the base delay between transient tool failures.
+	ToolRetryBackoff time.Duration
 
 	// Memory is the agent's conversation memory. Nil = the default
 	// store-backed memory (durable across restarts).
@@ -121,6 +126,8 @@ func newOptions(opts ...Option) Options {
 		ModelMaxAttempts:  1, // retries opt-in via ModelRetry (see field doc)
 		ModelRetryBackoff: 100 * time.Millisecond,
 		ToolTimeout:       30 * time.Second,
+		ToolMaxAttempts:   1,
+		ToolRetryBackoff:  100 * time.Millisecond,
 		// On by default and lenient: identical repeated calls are a
 		// no-progress loop, never useful. Set LoopLimit(0) to disable.
 		LoopLimit: 3,
@@ -229,6 +236,16 @@ func ModelRetry(maxAttempts int, backoff time.Duration) Option {
 	}
 }
 
+// ToolRetry sets the tool retry budget and backoff for transient failures.
+// Attempts include the first call. Retries are opt-in because tools may have
+// side effects; keep handlers idempotent before enabling this.
+func ToolRetry(maxAttempts int, backoff time.Duration) Option {
+	return func(o *Options) {
+		o.ToolMaxAttempts = maxAttempts
+		o.ToolRetryBackoff = backoff
+	}
+}
+
 // WithA2A makes Run serve the agent over the A2A protocol on addr (e.g.
 // ":4000"), so other agents can reach it directly by URL without a
 // separate gateway. The agent stays a normal go-micro service as well;

agent/resilience_test.go

@@ -184,3 +184,48 @@ type testStatusError struct {
 func (e testStatusError) Error() string { return "provider status error" }
 
 func (e testStatusError) StatusCode() int { return e.code }
+
+func TestToolRetryRetriesTransientToolErrorsThenSucceeds(t *testing.T) {
+	attempts := 0
+	a := newTestAgent(
+		Name("tool-retry-success"),
+		ToolRetry(3, time.Millisecond),
+		WithTool("flaky", "flaky tool", nil, func(context.Context, map[string]any) (string, error) {
+			attempts++
+			if attempts < 3 {
+				return "", context.DeadlineExceeded
+			}
+			return "ok", nil
+		}),
+	)
+
+	content := toolContent(a.toolHandler(), "flaky", nil)
+	if content != "ok" {
+		t.Fatalf("tool result = %q, want ok", content)
+	}
+	if attempts != 3 {
+		t.Fatalf("attempts = %d, want 3", attempts)
+	}
+}
+
+func TestToolRetryDoesNotRetryGuardrailRefusals(t *testing.T) {
+	attempts := 0
+	a := newTestAgent(
+		Name("tool-retry-refusal"),
+		MaxSteps(1),
+		ToolRetry(3, time.Millisecond),
+		WithTool("counted", "counted tool", nil, func(context.Context, map[string]any) (string, error) {
+			attempts++
+			return "ok", nil
+		}),
+	)
+	h := a.toolHandler()
+	_ = toolContent(h, "counted", nil)
+	content := toolContent(h, "counted", nil)
+	if !strings.Contains(content, "step limit reached") {
+		t.Fatalf("tool result = %q, want step-limit refusal", content)
+	}
+	if attempts != 1 {
+		t.Fatalf("attempts = %d, want only the allowed tool call to execute", attempts)
+	}
+}

ai/atlascloud/atlascloud.go

@@ -27,6 +27,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -56,7 +57,14 @@ func NewProvider(opts ...ai.Option) *Provider {
 	options := ai.NewOptions(opts...)
 
 	if options.Model == "" {
-		options.Model = "deepseek-ai/DeepSeek-V3-0324"
+		// Allow the chat model to be selected via the ATLASCLOUD_MODEL env var
+		// (e.g. to run CI conformance against a stronger tool-use model) without
+		// a code change; fall back to a sensible default otherwise.
+		if m := os.Getenv("ATLASCLOUD_MODEL"); m != "" {
+			options.Model = m
+		} else {
+			options.Model = "deepseek-ai/DeepSeek-V3-0324"
+		}
 	}
 	if options.BaseURL == "" {
 		options.BaseURL = "https://api.atlascloud.ai"

ai/model.go

@@ -93,9 +93,10 @@ func (c ToolCall) Scan(v any) error {
 
 // ToolResult represents the result of a tool execution
 type ToolResult struct {
-	ID      string // Tool call ID (for correlation)
-	Value   any    // Structured result (optional)
-	Content string // Tool execution result (JSON string), shown to the model
+	ID       string // Tool call ID (for correlation)
+	Value    any    // Structured result (optional)
+	Content  string // Tool execution result (JSON string), shown to the model
+	Attempts int    `json:"attempts,omitempty"` // Tool execution attempts, set when retried.
 	// Refused names the reason a guardrail blocked the call before it ran
 	// ("max_steps", "loop", "approval"); empty when the call executed. A
 	// tool wrapper can switch on it to build reliability tooling — react to

internal/docs/PRIORITIES.md

@@ -21,9 +21,8 @@ changes, architectural rewrites. Those go to the human.
 
 ## Work queue (ranked)
 
-1. **Add durable checkpoint/resume for agent runs** ([#3524](https://github.com/micro/go-micro/issues/3524)) — the 0→hero reference app has now shipped, and the next highest-value lifecycle gap is making long-running agent work survive restarts the way flows already do. This is the clearest bridge from services → agents → workflows: flows can already checkpoint deterministic orchestration, but the dynamic agent loop still needs a CI-verifiable resume contract so scheduled, looping agents can be operated rather than merely invoked.
-2. **Emit OpenTelemetry spans for agent run timelines** ([#3525](https://github.com/micro/go-micro/issues/3525)) — recent work made runs inspectable and correlated trace metadata through scheduled dispatch; the next step is to turn that RunInfo foundation into standard OTel spans for agent runs, model calls, and tool calls. This keeps `micro runs` useful while making the harness observable in the systems developers already run.
-3. **Add retry and timeout resilience to agent tool execution** ([#3526](https://github.com/micro/go-micro/issues/3526)) — flow retry/backoff and cancellation safety have shipped, but the agent loop still needs the same failure semantics around tool/model calls: bounded retries, deadline propagation, cancellation, and visible retry/timeout outcomes. This belongs high because operability is the difference between an agent demo and a dependable service.
+1. **CI-verify cross-provider agent conformance** ([#3537](https://github.com/micro/go-micro/issues/3537)) — retry/backoff resilience and durable checkpoint/resume have shipped, so the highest-value remaining Now-phase gap is proving the same services → agents → workflows scenario works across every supported model provider. This keeps the harness honest where provider drift most directly breaks developer trust: model calls, tool calls, skip/pass/fail reporting, and CI-safe execution when credentials are absent.
+2. **Emit OpenTelemetry spans for agent run timelines** ([#3525](https://github.com/micro/go-micro/issues/3525)) — recent work made runs inspectable, correlated trace metadata through scheduled dispatch, verified restart resume, and added opt-in tool retries; the next step is to turn that RunInfo foundation into standard OTel spans for agent runs, model calls, tool calls, checkpoint/resume, and failures. This keeps `micro runs` useful while making the harness observable in the systems developers already run.
 
 _Seeded by Claude Code from the roadmap + open issues; thereafter maintained by the
 architecture-review pass._

internal/harness/provider-conformance/README.md

@@ -20,7 +20,11 @@ agent test and the harnesses in `internal/harness`:
   preserving run metadata.
 
 The command also emits the registered provider capability matrix so the run shows
-which providers advertise model, image, video, and streaming support.
+which providers advertise model, image, video, and streaming support. Console output
+and summary artifacts label each harness with the phase it is proving (for example,
+model call + tool call, workflow event + tool call, or streaming fallback + tool
+call), so provider failures identify the failed lifecycle phase instead of only the
+provider name.
 
 ## Local usage
 

internal/harness/provider-conformance/main.go

@@ -36,6 +36,14 @@ import (
 
 const defaultHarnesses = "agent,universe,agent-flow,plan-delegate,a2a-stream-fallback"
 
+var harnessPhases = map[string]string{
+	"agent":               "model call + tool call",
+	"universe":            "service discovery + tool call",
+	"agent-flow":          "workflow event + tool call",
+	"plan-delegate":       "plan persistence + delegation + tool call",
+	"a2a-stream-fallback": "streaming fallback + tool call",
+}
+
 var providerEnv = map[string]string{
 	"anthropic":  "ANTHROPIC_API_KEY",
 	"openai":     "OPENAI_API_KEY",
@@ -92,15 +100,16 @@ func main() {
 		}
 
 		for _, harness := range harnesses {
-			fmt.Printf("\n==> %s / %s\n", provider, harness)
+			phase := harnessPhase(harness)
+			fmt.Printf("\n==> %s / %s (%s)\n", provider, harness, phase)
 			if err := runHarness(provider, harness, *timeoutFlag); err != nil {
-				fmt.Printf("FAIL %s / %s: %v\n", provider, harness, err)
+				fmt.Printf("FAIL %s / %s (%s): %v\n", provider, harness, phase, err)
 				failed++
-				results = append(results, conformanceResult{Provider: provider, Harness: harness, Status: statusFailed, Error: err.Error()})
+				results = append(results, conformanceResult{Provider: provider, Harness: harness, Phase: phase, Status: statusFailed, Error: err.Error()})
 				continue
 			}
 			ran++
-			results = append(results, conformanceResult{Provider: provider, Harness: harness, Status: statusPassed})
+			results = append(results, conformanceResult{Provider: provider, Harness: harness, Phase: phase, Status: statusPassed})
 		}
 	}
 
@@ -140,6 +149,7 @@ const (
 type conformanceResult struct {
 	Provider string `json:"provider"`
 	Harness  string `json:"harness,omitempty"`
+	Phase    string `json:"phase,omitempty"`
 	Status   string `json:"status"`
 	Error    string `json:"error,omitempty"`
 }
@@ -176,14 +186,18 @@ func writeSummaryMarkdown(path string, summary conformanceSummary) error {
 	b.WriteString("## Capability matrix\n\n")
 	b.WriteString(capabilityMarkdown(summary.Capabilities))
 	b.WriteString("\n## Harness results\n\n")
-	b.WriteString("| Provider | Harness | Status | Detail |\n")
-	b.WriteString("| --- | --- | --- | --- |\n")
+	b.WriteString("| Provider | Harness | Phase | Status | Detail |\n")
+	b.WriteString("| --- | --- | --- | --- | --- |\n")
 	for _, result := range summary.Results {
 		harness := result.Harness
 		if harness == "" {
 			harness = "—"
 		}
-		fmt.Fprintf(&b, "| %s | %s | %s | %s |\n", result.Provider, harness, result.Status, markdownCell(result.Error))
+		phase := result.Phase
+		if phase == "" {
+			phase = "—"
+		}
+		fmt.Fprintf(&b, "| %s | %s | %s | %s | %s |\n", result.Provider, harness, markdownCell(phase), result.Status, markdownCell(result.Error))
 	}
 	return os.WriteFile(path, []byte(b.String()), 0o644)
 }
@@ -209,6 +223,13 @@ func markdownList(values []string) string {
 	return strings.Join(escaped, ", ")
 }
 
+func harnessPhase(harness string) string {
+	if phase, ok := harnessPhases[harness]; ok {
+		return phase
+	}
+	return "harness"
+}
+
 func markdownCell(s string) string {
 	if s == "" {
 		return "—"

internal/harness/provider-conformance/main_test.go

@@ -99,7 +99,7 @@ func TestWriteSummaryMarkdown(t *testing.T) {
 	summary := conformanceSummary{
 		Capabilities: []ai.CapabilityRow{{Provider: "mock", Capabilities: ai.Capabilities{Model: true}}},
 		Results: []conformanceResult{
-			{Provider: "mock", Harness: "agent-flow", Status: statusPassed},
+			{Provider: "mock", Harness: "agent-flow", Phase: harnessPhase("agent-flow"), Status: statusPassed},
 			{Provider: "live", Status: statusSkipped, Error: "missing | key"},
 		},
 		Passed:  1,
@@ -120,8 +120,8 @@ func TestWriteSummaryMarkdown(t *testing.T) {
 		"Providers: —.",
 		"Harnesses: —.",
 		"| mock | ✅ | — | — | — |",
-		"| mock | agent-flow | passed | — |",
-		"| live | — | skipped | missing \\| key |",
+		"| mock | agent-flow | workflow event + tool call | passed | — |",
+		"| live | — | — | skipped | missing \\| key |",
 	} {
 		if !strings.Contains(got, want) {
 			t.Fatalf("summary markdown = %q, want %q", got, want)
@@ -129,6 +129,15 @@ func TestWriteSummaryMarkdown(t *testing.T) {
 	}
 }
 
+func TestHarnessPhaseLabelsKnownHarnesses(t *testing.T) {
+	if got := harnessPhase("a2a-stream-fallback"); got != "streaming fallback + tool call" {
+		t.Fatalf("harnessPhase() = %q, want streaming fallback phase", got)
+	}
+	if got := harnessPhase("custom"); got != "harness" {
+		t.Fatalf("harnessPhase(custom) = %q, want fallback phase", got)
+	}
+}
+
 func TestMarkdownListEscapesBackticks(t *testing.T) {
 	got := markdownList([]string{"agent", "bad`name"})
 	want := "`agent`, `bad\\`name`"

micro.go

@@ -116,6 +116,24 @@ func AgentLoopLimit(n int) AgentOption { return agent.LoopLimit(n) }
 // each action the agent takes.
 func AgentApproveTool(fn ApproveFunc) AgentOption { return agent.ApproveTool(fn) }
 
+// AgentModelCallTimeout sets the timeout for each provider Generate call.
+func AgentModelCallTimeout(d time.Duration) AgentOption { return agent.ModelCallTimeout(d) }
+
+// AgentModelRetry sets the provider retry budget and backoff for transient failures.
+func AgentModelRetry(maxAttempts int, backoff time.Duration) AgentOption {
+	return agent.ModelRetry(maxAttempts, backoff)
+}
+
+// AgentToolCallTimeout sets the timeout for each agent tool execution.
+func AgentToolCallTimeout(d time.Duration) AgentOption { return agent.ToolCallTimeout(d) }
+
+// AgentToolRetry sets the tool retry budget and backoff for transient failures.
+// Attempts include the first call. Retries are opt-in because tools can have
+// side effects; keep handlers idempotent before enabling this.
+func AgentToolRetry(maxAttempts int, backoff time.Duration) AgentOption {
+	return agent.ToolRetry(maxAttempts, backoff)
+}
+
 // Memory is an agent's pluggable conversation memory.
 type Memory = agent.Memory