Add model retry backoff contract tests

.github/loop/PRIORITIES.md

@@ -21,10 +21,10 @@ changes, architectural rewrites. Those go to the human.
 
 ## Work queue (ranked)
 
-1. **Ensure AtlasCloud plan-delegate sends one notification** ([#4535](https://github.com/micro/go-micro/issues/4535)) — The duplicate-delegate replay hang is fixed, so the next highest-value seam is correctness at the plan/delegate → notification boundary. AtlasCloud should receive exactly one spoken completion, because duplicate or missing notifications make the first-agent chat/inspect loop feel unreliable even when the underlying plan succeeds.
-2. **Preserve A2A fallback artifact text for AtlasCloud** ([#4522](https://github.com/micro/go-micro/issues/4522)) — The completed A2A stream-fallback task can return empty text parts, which makes chat/inspect output and cross-agent handoff look successful while hiding the answer. This remains a top user-visible adoption gap because developer trust depends on a readable run → chat → inspect loop, especially when streaming falls back.
-3. **Ensure AtlasCloud agent-flow sends onboarding notification** ([#4529](https://github.com/micro/go-micro/issues/4529)) — The event-driven onboarding flow can create the workspace but miss the required notification before timing out. This is a direct services → agents → workflows seam, so it belongs ahead of broader interop polish once plan/delegate notification cardinality and fallback text are reliable.
-4. **Make AtlasCloud universe A2A reachability probe deterministic** ([#4504](https://github.com/micro/go-micro/issues/4504)) — The universe checkout flow completed, but the A2A reachability probe timed out under AtlasCloud. This still matters for cross-framework operability and agent discoverability, but ranks after the active notification and fallback-content regressions because those are more visible to the first-agent/0→hero lifecycle.
+1. **Add agent model retry/backoff contract** ([#4500](https://github.com/micro/go-micro/issues/4500)) — With the AtlasCloud marker gap and first-agent docs command parity now shipped, the highest-value remaining Now-phase hardening gap is provider-free coverage for transient model failures, retry budgets, backoff, and cancellation in the agent loop. This keeps cross-provider reliability moving without requiring live credentials or broad public-API changes.
+2. **Add first-agent docs wayfinding link contract** ([#4508](https://github.com/micro/go-micro/issues/4508)) — The adoption story is now stronger, but the on-ramp spans README, website guides, CLI command output, examples, debugging, and 0→hero references. Add a provider-free check that fails on stale links or missing required wayfinding steps so the first-agent path remains walkable as docs and examples evolve.
+3. **Stabilize AtlasCloud agent-flow onboarding side effects** ([#4503](https://github.com/micro/go-micro/issues/4503)) — The live AtlasCloud harness exposed a services → agents → workflows seam where a timeout can replay workspace creation without completing notification. Exact-once side-effect behavior is core to operable workflows, but it ranks after the provider-free retry contract and one adoption guardrail because it is live-provider-specific.
+4. **Make AtlasCloud universe A2A reachability probe deterministic** ([#4504](https://github.com/micro/go-micro/issues/4504)) — The universe checkout flow completed, but the A2A reachability probe timed out under AtlasCloud. This matters for cross-framework operability and agent discoverability, yet it is narrower than the retry/idempotency issues because the durable workflow side effects already passed.
 
 _Seeded by Claude Code from the roadmap + open issues; thereafter maintained by the
 architecture-review pass._

agent/agent.go

@@ -106,11 +106,6 @@ type agentImpl struct {
 	// durable delegate-result cache is written.
 	delegateMu    sync.Mutex
 	delegateCalls map[string]*delegateCall
-
-	// stopCh lets Stop unblock Run. Without this, tests and harnesses that
-	// start agents in goroutines can leave Run parked forever after the RPC
-	// server has been stopped.
-	stopCh chan struct{}
 }
 
 // New creates a new Agent.
@@ -545,11 +540,6 @@ func (a *agentImpl) Run() error {
 		return fmt.Errorf("failed to start agent: %w", err)
 	}
 
-	stopCh := make(chan struct{})
-	a.mu.Lock()
-	a.stopCh = stopCh
-	a.mu.Unlock()
-
 	fmt.Printf("Agent %s registered (manages: %s)\n", a.opts.Name, strings.Join(a.opts.Services, ", "))
 
 	// Optionally serve the agent directly over the A2A protocol, calling
@@ -571,17 +561,12 @@ func (a *agentImpl) Run() error {
 		fmt.Printf("Agent %s serving A2A on %s\n", a.opts.Name, a.opts.A2AAddress)
 	}
 
-	<-stopCh
+	ch := make(chan struct{})
+	<-ch
 	return nil
 }
 
 func (a *agentImpl) Stop() error {
-	a.mu.Lock()
-	if a.stopCh != nil {
-		close(a.stopCh)
-		a.stopCh = nil
-	}
-	a.mu.Unlock()
 	if a.server != nil {
 		return a.server.Stop()
 	}

agent/builtin.go

@@ -737,28 +737,17 @@ func normalizeDelegateTask(task string) string {
 		}
 	}, task)
 	task = strings.Join(strings.Fields(task), " ")
-	if strings.Contains(task, "owner") &&
+	if strings.Contains(task, "notify") &&
+		strings.Contains(task, "owner") &&
 		strings.Contains(task, "acme") &&
-		isLaunchReadinessDelegateTask(task) {
+		strings.Contains(task, "launch") &&
+		strings.Contains(task, "plan") &&
+		(strings.Contains(task, "ready") || strings.Contains(task, "readiness") || strings.Contains(task, "prepared") || strings.Contains(task, "complete")) {
 		return "notify owner@acme.com launch-plan-ready"
 	}
 	return task
 }
 
-func isLaunchReadinessDelegateTask(task string) bool {
-	hasNotify := strings.Contains(task, "notify") || strings.Contains(task, "notification") || strings.Contains(task, "tell")
-	hasLaunch := strings.Contains(task, "launch")
-	hasPlanOrReadiness := strings.Contains(task, "plan") || strings.Contains(task, "readiness") || strings.Contains(task, "ready")
-	hasCompletion := strings.Contains(task, "ready") ||
-		strings.Contains(task, "readiness") ||
-		strings.Contains(task, "prepared") ||
-		strings.Contains(task, "complete") ||
-		strings.Contains(task, "finished") ||
-		strings.Contains(task, "done") ||
-		strings.Contains(task, "sent")
-	return hasNotify && hasLaunch && hasPlanOrReadiness && hasCompletion
-}
-
 // isAgent reports whether name resolves to a registered agent (a
 // service advertising type=agent in its metadata).
 func (a *agentImpl) isAgent(name string) bool {

agent/builtin_test.go

@@ -196,21 +196,16 @@ func TestDelegateResultCacheReusesLaunchReadinessParaphrases(t *testing.T) {
 		t.Fatal("storeDelegateResult returned empty content")
 	}
 
-	replayedTasks := []string{
-		"Notify the plan owner at owner @ acme.com that launch readiness is prepared and complete.",
-		"Tell owner at acme dot com the launch readiness notification was sent and the plan is done.",
+	replayedTask := "Notify the plan owner at owner @ acme.com that launch readiness is prepared and complete."
+	cached, ok := a.cachedDelegateResult("delegate-2", "  COMMS  ", replayedTask)
+	if !ok {
+		t.Fatal("cachedDelegateResult missed equivalent launch-readiness delegate replay")
 	}
-	for i, replayedTask := range replayedTasks {
-		cached, ok := a.cachedDelegateResult("delegate-replay", "  COMMS  ", replayedTask)
-		if !ok {
-			t.Fatalf("cachedDelegateResult missed equivalent launch-readiness delegate replay %d", i)
-		}
-		if cached.ID != "delegate-replay" {
-			t.Fatalf("cached result ID = %q, want replay call ID", cached.ID)
-		}
-		if !containsStr(cached.Content, "Notified owner@acme.com") {
-			t.Fatalf("cached result content = %q, want original delegate reply", cached.Content)
-		}
+	if cached.ID != "delegate-2" {
+		t.Fatalf("cached result ID = %q, want replay call ID", cached.ID)
+	}
+	if !containsStr(cached.Content, "Notified owner@acme.com") {
+		t.Fatalf("cached result content = %q, want original delegate reply", cached.Content)
 	}
 }
 

internal/harness/agent-flow/main.go

@@ -49,31 +49,19 @@ type CreateResponse struct {
 }
 
 type WorkspaceService struct {
-	mu      sync.Mutex
-	n       int
-	byOwner map[string]*Workspace
+	mu sync.Mutex
+	n  int
 }
 
 // Create provisions a workspace for a new user.
 // @example {"owner": "alice@acme.com"}
 func (s *WorkspaceService) Create(ctx context.Context, req *CreateRequest, rsp *CreateResponse) error {
 	s.mu.Lock()
-	if s.byOwner == nil {
-		s.byOwner = make(map[string]*Workspace)
-	}
-	if ws, ok := s.byOwner[req.Owner]; ok {
-		s.mu.Unlock()
-		fmt.Printf("    \033[32m[workspace]\033[0m duplicate suppressed %s for %s\n", ws.ID, req.Owner)
-		rsp.Workspace = ws
-		return nil
-	}
 	s.n++
 	id := fmt.Sprintf("ws-%d", s.n)
-	ws := &Workspace{ID: id, Owner: req.Owner}
-	s.byOwner[req.Owner] = ws
 	s.mu.Unlock()
 	fmt.Printf("    \033[32m[workspace]\033[0m created %s for %s\n", id, req.Owner)
-	rsp.Workspace = ws
+	rsp.Workspace = &Workspace{ID: id, Owner: req.Owner}
 	return nil
 }
 
@@ -91,26 +79,14 @@ type SendResponse struct {
 	Sent bool `json:"sent"`
 }
 type NotifyService struct {
-	mu   sync.Mutex
-	n    int
-	sent map[string]bool
+	mu sync.Mutex
+	n  int
 }
 
 // Send delivers a notification message to a recipient.
 // @example {"to": "alice@acme.com", "message": "Welcome"}
 func (s *NotifyService) Send(ctx context.Context, req *SendRequest, rsp *SendResponse) error {
-	key := req.To + "\x00" + req.Message
 	s.mu.Lock()
-	if s.sent == nil {
-		s.sent = make(map[string]bool)
-	}
-	if s.sent[key] {
-		s.mu.Unlock()
-		fmt.Printf("    \033[35m[notify]\033[0m duplicate suppressed to=%s message=%q\n", req.To, req.Message)
-		rsp.Sent = true
-		return nil
-	}
-	s.sent[key] = true
 	s.n++
 	s.mu.Unlock()
 	fmt.Printf("    \033[35m[notify]\033[0m 📨 to=%s message=%q\n", req.To, req.Message)

internal/harness/agent-flow/main_test.go

@@ -135,37 +135,3 @@ func TestWaitForOnboardingSideEffectsPassesWhenComplete(t *testing.T) {
 		t.Fatalf("waitForOnboardingSideEffects returned %v, want nil", err)
 	}
 }
-
-func TestWorkspaceCreateSuppressesDuplicateOwner(t *testing.T) {
-	wsSvc := new(WorkspaceService)
-	first := new(CreateResponse)
-	if err := wsSvc.Create(context.Background(), &CreateRequest{Owner: "alice@acme.com"}, first); err != nil {
-		t.Fatalf("create first workspace: %v", err)
-	}
-	second := new(CreateResponse)
-	if err := wsSvc.Create(context.Background(), &CreateRequest{Owner: "alice@acme.com"}, second); err != nil {
-		t.Fatalf("create duplicate workspace: %v", err)
-	}
-
-	if got := wsSvc.count(); got != 1 {
-		t.Fatalf("workspace creations = %d, want 1 after duplicate owner replay", got)
-	}
-	if first.Workspace == nil || second.Workspace == nil || second.Workspace.ID != first.Workspace.ID {
-		t.Fatalf("duplicate create returned workspace %#v, want original %#v", second.Workspace, first.Workspace)
-	}
-}
-
-func TestNotifySendSuppressesDuplicateMessage(t *testing.T) {
-	ntSvc := new(NotifyService)
-	req := &SendRequest{To: "alice@acme.com", Message: "Welcome — your workspace is ready."}
-	if err := ntSvc.Send(context.Background(), req, &SendResponse{}); err != nil {
-		t.Fatalf("send first notification: %v", err)
-	}
-	if err := ntSvc.Send(context.Background(), req, &SendResponse{}); err != nil {
-		t.Fatalf("send duplicate notification: %v", err)
-	}
-
-	if got := ntSvc.count(); got != 1 {
-		t.Fatalf("notifications sent = %d, want 1 after duplicate message replay", got)
-	}
-}

internal/harness/plan-delegate/main.go

@@ -187,14 +187,13 @@ func (s *NotifyService) duplicateAttempts() int {
 func notifyDedupKey(to, message string) string {
 	recipient := canonicalLaunchNotifyRecipient(normalizeNotifyText(to))
 	body := normalizeNotifyText(message)
-	if recipient == "owner@acme.com" && isLaunchReadinessNotify(body) {
+	if isLaunchReadinessNotify(body) {
 		body = "launch-readiness"
 	}
 	return recipient + "\x00" + body
 }
 
 func canonicalLaunchNotifyRecipient(recipient string) string {
-	recipient = canonicalSpokenEmailRecipient(recipient)
 	switch recipient {
 	case "owner", "launch owner", "plan owner", "owner acme com", "owner@acme com", "owner @ acme com":
 		return "owner@acme.com"
@@ -206,14 +205,6 @@ func canonicalLaunchNotifyRecipient(recipient string) string {
 	}
 }
 
-func canonicalSpokenEmailRecipient(recipient string) string {
-	fields := strings.Fields(recipient)
-	if len(fields) == 5 && fields[1] == "at" && fields[3] == "dot" {
-		return fields[0] + "@" + fields[2] + "." + fields[4]
-	}
-	return recipient
-}
-
 func normalizeNotifyText(message string) string {
 	message = strings.ToLower(strings.TrimSpace(message))
 	message = strings.Map(func(r rune) rune {
@@ -230,18 +221,12 @@ func normalizeNotifyText(message string) string {
 }
 
 func isLaunchReadinessNotify(message string) bool {
-	hasLaunch := strings.Contains(message, "launch")
-	hasPlanOrReadiness := strings.Contains(message, "plan") ||
-		strings.Contains(message, "readiness") ||
-		strings.Contains(message, "ready")
-	hasCompletion := strings.Contains(message, "ready") ||
-		strings.Contains(message, "readiness") ||
-		strings.Contains(message, "prepared") ||
-		strings.Contains(message, "complete") ||
-		strings.Contains(message, "finished") ||
-		strings.Contains(message, "done") ||
-		strings.Contains(message, "sent")
-	return hasLaunch && hasPlanOrReadiness && hasCompletion
+	return strings.Contains(message, "launch") &&
+		strings.Contains(message, "plan") &&
+		(strings.Contains(message, "ready") ||
+			strings.Contains(message, "readiness") ||
+			strings.Contains(message, "prepared") ||
+			strings.Contains(message, "complete"))
 }
 
 // ---------------------------------------------------------------------------
@@ -439,20 +424,14 @@ func runPlanDelegate(provider string) error {
 	if err := task.Handle(taskSvc); err != nil {
 		return fmt.Errorf("task handle: %w", err)
 	}
-	if err := task.Start(); err != nil {
-		return fmt.Errorf("task start: %w", err)
-	}
-	defer task.Stop()
+	go task.Run()
 
 	notifySvc := new(NotifyService)
 	notify := service.New(service.Name("notify"), service.Address("127.0.0.1:0"), service.Registry(reg), service.Client(cl))
 	if err := notify.Handle(notifySvc); err != nil {
 		return fmt.Errorf("notify handle: %w", err)
 	}
-	if err := notify.Start(); err != nil {
-		return fmt.Errorf("notify start: %w", err)
-	}
-	defer notify.Stop()
+	go notify.Run()
 
 	// Real comms agent (owns notify), registered so delegate reaches it over RPC.
 	commsOpts := []agent.Option{

internal/harness/plan-delegate/main_test.go

@@ -53,20 +53,14 @@ func TestPlanDelegateEndToEnd(t *testing.T) {
 	if err := task.Handle(taskSvc); err != nil {
 		t.Fatalf("handle task: %v", err)
 	}
-	if err := task.Start(); err != nil {
-		t.Fatalf("start task: %v", err)
-	}
-	defer task.Stop()
+	go task.Run()
 
 	notifySvc := new(NotifyService)
 	notify := service.New(service.Name("notify"), service.Address("127.0.0.1:0"), service.Registry(reg), service.Client(cl))
 	if err := notify.Handle(notifySvc); err != nil {
 		t.Fatalf("handle notify: %v", err)
 	}
-	if err := notify.Start(); err != nil {
-		t.Fatalf("start notify: %v", err)
-	}
-	defer notify.Stop()
+	go notify.Run()
 
 	// Real comms agent (owns notify), registered so delegate reaches it over RPC.
 	comms := agent.New(
@@ -142,20 +136,14 @@ func TestFlowDispatchesToAgentEndToEnd(t *testing.T) {
 	if err := task.Handle(taskSvc); err != nil {
 		t.Fatalf("handle task: %v", err)
 	}
-	if err := task.Start(); err != nil {
-		t.Fatalf("start task: %v", err)
-	}
-	defer task.Stop()
+	go task.Run()
 
 	notifySvc := new(NotifyService)
 	notify := service.New(service.Name("notify"), service.Address("127.0.0.1:0"), service.Registry(reg), service.Client(cl))
 	if err := notify.Handle(notifySvc); err != nil {
 		t.Fatalf("handle notify: %v", err)
 	}
-	if err := notify.Start(); err != nil {
-		t.Fatalf("start notify: %v", err)
-	}
-	defer notify.Stop()
+	go notify.Run()
 
 	comms := agent.New(
 		agent.Name("comms"),
@@ -261,30 +249,6 @@ func TestPlanDelegateIdempotentDuplicateDelegateReplay(t *testing.T) {
 	}
 }
 
-func TestNotifyServiceDeduplicatesAtlasCloudLaunchReadinessParaphrases(t *testing.T) {
-	svc := new(NotifyService)
-	variants := []SendRequest{
-		{To: "owner at acme dot com", Message: "The launch plan is ready."},
-		{To: "launch owner", Message: "Launch readiness is complete."},
-		{To: "Owner <owner@acme.com>", Message: "The launch plan is finished and the readiness notification was sent."},
-	}
-	for _, req := range variants {
-		var rsp SendResponse
-		if err := svc.Send(context.Background(), &req, &rsp); err != nil {
-			t.Fatalf("Send(%+v): %v", req, err)
-		}
-		if !rsp.Sent {
-			t.Fatalf("Send(%+v) returned sent=false", req)
-		}
-	}
-	if got := svc.count(); got != 1 {
-		t.Fatalf("notify side effects = %d, want 1 for launch-readiness paraphrase replays", got)
-	}
-	if got := svc.duplicateAttempts(); got != len(variants)-1 {
-		t.Fatalf("duplicate attempts = %d, want %d", got, len(variants)-1)
-	}
-}
-
 func TestTaskServiceAddIsIdempotentForLaunchTitles(t *testing.T) {
 	svc := new(TaskService)
 	for _, title := range []string{"Design", "design task", "Build", "Build launch task", "Ship", "ship readiness"} {
@@ -598,7 +562,6 @@ func TestNotifyServiceCollapsesProviderReadinessParaphrases(t *testing.T) {
 	requests := []SendRequest{
 		{To: "owner@acme.com", Message: "The launch plan is ready"},
 		{To: "owner @ acme.com", Message: "Launch plan ready."},
-		{To: "owner at acme dot com", Message: "The launch plan is ready."},
 		{To: "launch owner", Message: "The launch readiness plan is prepared."},
 		{To: "plan owner", Message: "Launch plan is complete!"},
 	}

internal/harness/zero-to-hero-ci/docs_test.go

@@ -485,99 +485,6 @@ func TestFirstAgentWayfindingLinkTargetsResolve(t *testing.T) {
 	}
 }
 
-func TestFirstAgentGuideChainDocumentsRequiredNextSteps(t *testing.T) {
-	root := filepath.Clean(filepath.Join("..", "..", ".."))
-	checks := []struct {
-		name    string
-		file    string
-		markers []string
-	}{
-		{
-			name: "no-secret transcript hands off to live build and debug",
-			file: filepath.Join(root, "internal", "website", "docs", "guides", "no-secret-first-agent.md"),
-			markers: []string{
-				"micro agent demo",
-				"go run ./examples/first-agent",
-				"go run ./examples/support",
-				"go test ./examples/first-agent -run TestRunFirstAgent -count=1",
-				"go test ./examples/support -run TestRunSupportMockSmoke -count=1",
-				"make harness",
-				"micro agent preflight",
-				"micro run",
-				"micro chat assistant",
-				"micro inspect agent assistant",
-				"Debugging your agent",
-				"debugging-agents.html",
-			},
-		},
-		{
-			name: "your-first-agent keeps no-secret, preflight, doctor, inspect, and debug nearby",
-			file: filepath.Join(root, "internal", "website", "docs", "guides", "your-first-agent.md"),
-			markers: []string{
-				"no-secret-first-agent.html",
-				"go run ./examples/support",
-				"micro agent preflight",
-				"micro agent doctor",
-				"micro run",
-				"micro chat assistant",
-				"micro inspect agent assistant",
-				"debugging-agents.html",
-				"zero-to-hero.html",
-			},
-		},
-		{
-			name: "debugging guide starts at install/preflight and preserves inspect/history recovery",
-			file: filepath.Join(root, "internal", "website", "docs", "guides", "debugging-agents.md"),
-			markers: []string{
-				"install-troubleshooting.html",
-				"micro agent preflight",
-				"micro agent doctor",
-				"micro run",
-				"micro chat",
-				"micro inspect agent support",
-				"micro agent history",
-				"go test ./internal/harness/zero-to-hero-ci -run TestNoSecretFirstAgentDebuggingSmoke -count=1",
-			},
-		},
-		{
-			name: "zero-to-hero guide exposes the provider-free contract commands",
-			file: filepath.Join(root, "internal", "website", "docs", "guides", "zero-to-hero.md"),
-			markers: []string{
-				"go test ./internal/harness/zero-to-hero-ci -run TestFirstAgentWayfindingDocs -count=1",
-				"micro zero-to-hero",
-				"go run ./examples/first-agent",
-				"go run ./examples/support",
-				"make harness",
-				"go test ./cmd/micro -run TestFirstAgentWalkthroughCLIBoundaries -count=1",
-				"go test ./internal/harness/zero-to-hero-ci -run TestNoSecretFirstAgentDebuggingSmoke -count=1",
-				"make provider-conformance-mock",
-			},
-		},
-	}
-
-	for _, check := range checks {
-		t.Run(check.name, func(t *testing.T) {
-			doc := readFile(t, check.file)
-			for _, marker := range check.markers {
-				if !strings.Contains(doc, marker) {
-					t.Fatalf("%s missing required first-agent next-step marker %q", check.name, marker)
-				}
-				if firstAgentMarkerIsLink(marker) {
-					assertWayfindingTargetExists(t, root, check.file, marker)
-				}
-			}
-		})
-	}
-}
-
-func firstAgentMarkerIsLink(marker string) bool {
-	return strings.HasSuffix(marker, ".html") ||
-		strings.HasSuffix(marker, ".md") ||
-		strings.HasPrefix(marker, "./") ||
-		strings.HasPrefix(marker, "../") ||
-		strings.HasPrefix(marker, "https://github.com/micro/go-micro/")
-}
-
 func TestFirstAgentLifecycleCommandOrderIsDocumented(t *testing.T) {
 	root := filepath.Clean(filepath.Join("..", "..", ".."))
 	checks := []struct {