Compare commits

..

1 Commits

Author SHA1 Message Date
Codex c5920b5f5b Trigger test workflow on PR updates
Harness (E2E) / Harnesses (mock LLM) (push) Waiting to run
Harness (E2E) / Provider harnesses (live LLM conformance) (push) Waiting to run
Lint / golangci-lint (push) Waiting to run
Run Tests / Unit Tests (push) Waiting to run
Run Tests / Etcd Integration Tests (push) Waiting to run
2026-06-26 18:14:21 +00:00
19 changed files with 77 additions and 597 deletions
+1 -16
View File
@@ -59,19 +59,4 @@ jobs:
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
ATLASCLOUD_API_KEY: ${{ secrets.ATLASCLOUD_API_KEY }}
run: |
go run ./internal/harness/provider-conformance \
-require-configured \
-summary-json provider-conformance-summary.json \
-summary-markdown provider-conformance-summary.md \
-capabilities-markdown provider-capabilities.md
- name: Upload provider conformance summary
if: always()
uses: actions/upload-artifact@v4
with:
name: provider-conformance
path: |
provider-conformance-summary.json
provider-conformance-summary.md
provider-capabilities.md
if-no-files-found: ignore
run: go run ./internal/harness/provider-conformance -require-configured
+3 -7
View File
@@ -168,10 +168,6 @@ func (a *agentImpl) stateStore() store.Store {
// Ask sends a message and returns the agent's response.
// This is the programmatic API for direct use.
func (a *agentImpl) Ask(ctx context.Context, message string) (*Response, error) {
return a.ask(ctx, message, a.parentRunID)
}
func (a *agentImpl) ask(ctx context.Context, message, parentRunID string) (*Response, error) {
a.mu.Lock()
defer a.mu.Unlock()
@@ -192,7 +188,7 @@ func (a *agentImpl) ask(ctx context.Context, message, parentRunID string) (*Resp
a.runID = uuid.New().String()
ctx = ai.WithRunInfo(ctx, ai.RunInfo{
RunID: a.runID,
ParentID: parentRunID,
ParentID: a.parentRunID,
Agent: a.opts.Name,
})
ctx, endRun := a.startRun(ctx, message)
@@ -232,14 +228,14 @@ func (a *agentImpl) ask(ctx context.Context, message, parentRunID string) (*Resp
ToolCalls: resp.ToolCalls,
Agent: a.opts.Name,
RunID: a.runID,
ParentID: parentRunID,
ParentID: a.parentRunID,
}, nil
}
// Chat implements the proto AgentHandler interface for RPC.
// @example {"message": "What tasks are overdue?"}
func (a *agentImpl) Chat(ctx context.Context, req *pb.ChatRequest, rsp *pb.ChatResponse) error {
resp, err := a.ask(ctx, req.Message, req.ParentId)
resp, err := a.Ask(ctx, req.Message)
if err != nil {
return err
}
-23
View File
@@ -60,29 +60,6 @@ func TestChatResponseIncludesRunIDs(t *testing.T) {
}
}
func TestChatRequestParentIDPropagatesToResponse(t *testing.T) {
fakeGen = func(ctx context.Context, opts ai.Options, req *ai.Request) (*ai.Response, error) {
info, ok := ai.RunInfoFrom(ctx)
if !ok {
t.Fatal("RunInfo missing from model context")
}
if info.ParentID != "flow-run-123" {
t.Fatalf("RunInfo.ParentID = %q, want flow-run-123", info.ParentID)
}
return &ai.Response{Reply: "ok"}, nil
}
defer func() { fakeGen = nil }()
a := newTestAgent(Name("chat-child"))
var rsp pb.ChatResponse
if err := a.Chat(context.Background(), &pb.ChatRequest{Message: "hello", ParentId: "flow-run-123"}, &rsp); err != nil {
t.Fatalf("Chat: %v", err)
}
if rsp.ParentId != "flow-run-123" {
t.Errorf("ParentId = %q, want flow-run-123", rsp.ParentId)
}
}
func TestBuildPrompt(t *testing.T) {
// Custom prompt
a := New(Name("test"), Prompt("custom prompt")).(*agentImpl)
+35 -45
View File
@@ -2,7 +2,7 @@
// versions:
// protoc-gen-go v1.36.11
// protoc v3.21.12
// source: agent/proto/agent.proto
// source: proto/agent.proto
package agent
@@ -22,17 +22,15 @@ const (
)
type ChatRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
// parent_id correlates this chat with the workflow or agent run that dispatched it.
ParentId string `protobuf:"bytes,2,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"`
state protoimpl.MessageState `protogen:"open.v1"`
Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ChatRequest) Reset() {
*x = ChatRequest{}
mi := &file_agent_proto_agent_proto_msgTypes[0]
mi := &file_proto_agent_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -44,7 +42,7 @@ func (x *ChatRequest) String() string {
func (*ChatRequest) ProtoMessage() {}
func (x *ChatRequest) ProtoReflect() protoreflect.Message {
mi := &file_agent_proto_agent_proto_msgTypes[0]
mi := &file_proto_agent_proto_msgTypes[0]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -57,7 +55,7 @@ func (x *ChatRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use ChatRequest.ProtoReflect.Descriptor instead.
func (*ChatRequest) Descriptor() ([]byte, []int) {
return file_agent_proto_agent_proto_rawDescGZIP(), []int{0}
return file_proto_agent_proto_rawDescGZIP(), []int{0}
}
func (x *ChatRequest) GetMessage() string {
@@ -67,13 +65,6 @@ func (x *ChatRequest) GetMessage() string {
return ""
}
func (x *ChatRequest) GetParentId() string {
if x != nil {
return x.ParentId
}
return ""
}
type ChatResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
Reply string `protobuf:"bytes,1,opt,name=reply,proto3" json:"reply,omitempty"`
@@ -89,7 +80,7 @@ type ChatResponse struct {
func (x *ChatResponse) Reset() {
*x = ChatResponse{}
mi := &file_agent_proto_agent_proto_msgTypes[1]
mi := &file_proto_agent_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -101,7 +92,7 @@ func (x *ChatResponse) String() string {
func (*ChatResponse) ProtoMessage() {}
func (x *ChatResponse) ProtoReflect() protoreflect.Message {
mi := &file_agent_proto_agent_proto_msgTypes[1]
mi := &file_proto_agent_proto_msgTypes[1]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -114,7 +105,7 @@ func (x *ChatResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use ChatResponse.ProtoReflect.Descriptor instead.
func (*ChatResponse) Descriptor() ([]byte, []int) {
return file_agent_proto_agent_proto_rawDescGZIP(), []int{1}
return file_proto_agent_proto_rawDescGZIP(), []int{1}
}
func (x *ChatResponse) GetReply() string {
@@ -164,7 +155,7 @@ type ToolCall struct {
func (x *ToolCall) Reset() {
*x = ToolCall{}
mi := &file_agent_proto_agent_proto_msgTypes[2]
mi := &file_proto_agent_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -176,7 +167,7 @@ func (x *ToolCall) String() string {
func (*ToolCall) ProtoMessage() {}
func (x *ToolCall) ProtoReflect() protoreflect.Message {
mi := &file_agent_proto_agent_proto_msgTypes[2]
mi := &file_proto_agent_proto_msgTypes[2]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -189,7 +180,7 @@ func (x *ToolCall) ProtoReflect() protoreflect.Message {
// Deprecated: Use ToolCall.ProtoReflect.Descriptor instead.
func (*ToolCall) Descriptor() ([]byte, []int) {
return file_agent_proto_agent_proto_rawDescGZIP(), []int{2}
return file_proto_agent_proto_rawDescGZIP(), []int{2}
}
func (x *ToolCall) GetId() string {
@@ -220,14 +211,13 @@ func (x *ToolCall) GetResult() string {
return ""
}
var File_agent_proto_agent_proto protoreflect.FileDescriptor
var File_proto_agent_proto protoreflect.FileDescriptor
const file_agent_proto_agent_proto_rawDesc = "" +
const file_proto_agent_proto_rawDesc = "" +
"\n" +
"\x17agent/proto/agent.proto\x12\x05agent\"D\n" +
"\x11proto/agent.proto\x12\x05agent\"'\n" +
"\vChatRequest\x12\x18\n" +
"\amessage\x18\x01 \x01(\tR\amessage\x12\x1b\n" +
"\tparent_id\x18\x02 \x01(\tR\bparentId\"\x9e\x01\n" +
"\amessage\x18\x01 \x01(\tR\amessage\"\x9e\x01\n" +
"\fChatResponse\x12\x14\n" +
"\x05reply\x18\x01 \x01(\tR\x05reply\x12\x14\n" +
"\x05agent\x18\x02 \x01(\tR\x05agent\x12.\n" +
@@ -244,24 +234,24 @@ const file_agent_proto_agent_proto_rawDesc = "" +
"\x04Chat\x12\x12.agent.ChatRequest\x1a\x13.agent.ChatResponse\"\x00B\x0fZ\r./proto;agentb\x06proto3"
var (
file_agent_proto_agent_proto_rawDescOnce sync.Once
file_agent_proto_agent_proto_rawDescData []byte
file_proto_agent_proto_rawDescOnce sync.Once
file_proto_agent_proto_rawDescData []byte
)
func file_agent_proto_agent_proto_rawDescGZIP() []byte {
file_agent_proto_agent_proto_rawDescOnce.Do(func() {
file_agent_proto_agent_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agent_proto_agent_proto_rawDesc), len(file_agent_proto_agent_proto_rawDesc)))
func file_proto_agent_proto_rawDescGZIP() []byte {
file_proto_agent_proto_rawDescOnce.Do(func() {
file_proto_agent_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_proto_agent_proto_rawDesc), len(file_proto_agent_proto_rawDesc)))
})
return file_agent_proto_agent_proto_rawDescData
return file_proto_agent_proto_rawDescData
}
var file_agent_proto_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_agent_proto_agent_proto_goTypes = []any{
var file_proto_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_proto_agent_proto_goTypes = []any{
(*ChatRequest)(nil), // 0: agent.ChatRequest
(*ChatResponse)(nil), // 1: agent.ChatResponse
(*ToolCall)(nil), // 2: agent.ToolCall
}
var file_agent_proto_agent_proto_depIdxs = []int32{
var file_proto_agent_proto_depIdxs = []int32{
2, // 0: agent.ChatResponse.tool_calls:type_name -> agent.ToolCall
0, // 1: agent.Agent.Chat:input_type -> agent.ChatRequest
1, // 2: agent.Agent.Chat:output_type -> agent.ChatResponse
@@ -272,26 +262,26 @@ var file_agent_proto_agent_proto_depIdxs = []int32{
0, // [0:1] is the sub-list for field type_name
}
func init() { file_agent_proto_agent_proto_init() }
func file_agent_proto_agent_proto_init() {
if File_agent_proto_agent_proto != nil {
func init() { file_proto_agent_proto_init() }
func file_proto_agent_proto_init() {
if File_proto_agent_proto != nil {
return
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_agent_proto_agent_proto_rawDesc), len(file_agent_proto_agent_proto_rawDesc)),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_agent_proto_rawDesc), len(file_proto_agent_proto_rawDesc)),
NumEnums: 0,
NumMessages: 3,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_agent_proto_agent_proto_goTypes,
DependencyIndexes: file_agent_proto_agent_proto_depIdxs,
MessageInfos: file_agent_proto_agent_proto_msgTypes,
GoTypes: file_proto_agent_proto_goTypes,
DependencyIndexes: file_proto_agent_proto_depIdxs,
MessageInfos: file_proto_agent_proto_msgTypes,
}.Build()
File_agent_proto_agent_proto = out.File
file_agent_proto_agent_proto_goTypes = nil
file_agent_proto_agent_proto_depIdxs = nil
File_proto_agent_proto = out.File
file_proto_agent_proto_goTypes = nil
file_proto_agent_proto_depIdxs = nil
}
+1 -1
View File
@@ -1,5 +1,5 @@
// Code generated by protoc-gen-micro. DO NOT EDIT.
// source: agent/proto/agent.proto
// source: proto/agent.proto
package agent
-3
View File
@@ -11,9 +11,6 @@ service Agent {
message ChatRequest {
string message = 1;
// parent_id correlates this chat with the workflow or agent run that dispatched it.
string parent_id = 2;
}
message ChatResponse {
+4 -28
View File
@@ -19,10 +19,6 @@ type Capabilities struct {
Image bool `json:"image"`
// Video reports whether ai.NewVideo can construct a video model provider.
Video bool `json:"video"`
// Stream reports whether the provider has registered end-to-end token streaming.
// Providers that only satisfy the Model interface with ErrStreamingUnsupported
// leave this false until their Stream implementation is usable.
Stream bool `json:"stream"`
}
// ProviderCapabilities reports the capabilities registered for provider.
@@ -30,13 +26,11 @@ func ProviderCapabilities(provider string) Capabilities {
_, hasModel := providers[provider]
_, hasImage := imageProviders[provider]
_, hasVideo := videoProviders[provider]
_, hasStream := streamProviders[provider]
return Capabilities{
Model: hasModel,
Image: hasImage,
Video: hasVideo,
Stream: hasStream,
Model: hasModel,
Image: hasImage,
Video: hasVideo,
}
}
@@ -55,9 +49,6 @@ func CapabilityMatrix() map[string]Capabilities {
for name := range videoProviders {
names[name] = struct{}{}
}
for name := range streamProviders {
names[name] = struct{}{}
}
matrix := make(map[string]Capabilities, len(names))
for name := range names {
@@ -81,17 +72,8 @@ func CapabilityRows() []CapabilityRow {
return rows
}
// RegisterStream records that provider has a usable Stream implementation.
// Providers should call this from init alongside Register once Stream returns
// chunks instead of ErrStreamingUnsupported.
func RegisterStream(provider string) {
streamProviders[provider] = struct{}{}
}
var streamProviders = make(map[string]struct{})
// RegisteredProviders returns the registered provider names in sorted order.
// kind may be "model", "image", "video", "stream", or empty for the union of all
// kind may be "model", "image", "video", or empty for the union of all
// provider registries.
func RegisteredProviders(kind string) []string {
names := map[string]struct{}{}
@@ -109,18 +91,12 @@ func RegisteredProviders(kind string) []string {
for name := range r {
names[name] = struct{}{}
}
case map[string]struct{}:
for name := range r {
names[name] = struct{}{}
}
}
}
switch kind {
case "model":
add(providers)
case "stream":
add(streamProviders)
case "image":
add(imageProviders)
case "video":
-20
View File
@@ -32,12 +32,6 @@ func TestRegisteredProviders(t *testing.T) {
if !reflect.DeepEqual(got, want) {
t.Fatalf("RegisteredProviders(video) = %#v, want %#v", got, want)
}
got = ai.RegisteredProviders("stream")
want = []string{}
if !reflect.DeepEqual(got, want) {
t.Fatalf("RegisteredProviders(stream) = %#v, want %#v", got, want)
}
}
func TestCapabilityRows(t *testing.T) {
@@ -79,17 +73,3 @@ func TestCapabilityMatrix(t *testing.T) {
t.Fatalf("ProviderCapabilities(missing) = %#v", caps)
}
}
func TestRegisterStream(t *testing.T) {
ai.RegisterStream("test-stream")
if caps := ai.ProviderCapabilities("test-stream"); caps != (ai.Capabilities{Stream: true}) {
t.Fatalf("ProviderCapabilities(test-stream) = %#v", caps)
}
got := ai.RegisteredProviders("stream")
want := []string{"test-stream"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("RegisteredProviders(stream) = %#v, want %#v", got, want)
}
}
-96
View File
@@ -1,96 +0,0 @@
package ai
import (
"context"
"errors"
"testing"
"time"
)
type retryModel struct {
generate func(context.Context, *Request, ...GenerateOption) (*Response, error)
}
func (m retryModel) Init(...Option) error { return nil }
func (m retryModel) Options() Options { return Options{} }
func (m retryModel) Generate(ctx context.Context, req *Request, opts ...GenerateOption) (*Response, error) {
return m.generate(ctx, req, opts...)
}
func (m retryModel) Stream(context.Context, *Request, ...GenerateOption) (Stream, error) {
return nil, ErrStreamingUnsupported
}
func (m retryModel) String() string { return "retry-test" }
func TestGenerateWithRetryRetriesTransientErrors(t *testing.T) {
attempts := 0
model := retryModel{generate: func(context.Context, *Request, ...GenerateOption) (*Response, error) {
attempts++
if attempts == 1 {
return nil, errors.New("temporary provider outage")
}
return &Response{Reply: "ok"}, nil
}}
resp, err := GenerateWithRetry(context.Background(), model, &Request{Prompt: "hi"}, GeneratePolicy{
MaxAttempts: 2,
Backoff: time.Millisecond,
})
if err != nil {
t.Fatalf("GenerateWithRetry returned error: %v", err)
}
if resp.Reply != "ok" {
t.Fatalf("response reply = %q, want ok", resp.Reply)
}
if attempts != 2 {
t.Fatalf("attempts = %d, want 2", attempts)
}
}
func TestGenerateWithRetryDoesNotRetryCallerCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
attempts := 0
model := retryModel{generate: func(context.Context, *Request, ...GenerateOption) (*Response, error) {
attempts++
cancel()
return nil, errors.New("temporary provider outage")
}}
_, err := GenerateWithRetry(ctx, model, &Request{Prompt: "hi"}, GeneratePolicy{
MaxAttempts: 3,
Backoff: time.Millisecond,
})
if !errors.Is(err, context.Canceled) {
t.Fatalf("error = %v, want context.Canceled", err)
}
if attempts != 1 {
t.Fatalf("attempts = %d, want 1", attempts)
}
}
func TestGenerateWithRetryHonorsPerAttemptTimeout(t *testing.T) {
attempts := 0
model := retryModel{generate: func(ctx context.Context, _ *Request, _ ...GenerateOption) (*Response, error) {
attempts++
<-ctx.Done()
return nil, ctx.Err()
}}
_, err := GenerateWithRetry(context.Background(), model, &Request{Prompt: "hi"}, GeneratePolicy{
Timeout: time.Millisecond,
MaxAttempts: 2,
Backoff: time.Millisecond,
})
var retryErr *RetryError
if !errors.As(err, &retryErr) {
t.Fatalf("error = %T %[1]v, want RetryError", err)
}
if retryErr.Attempts != 2 {
t.Fatalf("retry attempts = %d, want 2", retryErr.Attempts)
}
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("error = %v, want context.DeadlineExceeded", err)
}
if attempts != 2 {
t.Fatalf("attempts = %d, want 2", attempts)
}
}
+7 -62
View File
@@ -9,7 +9,6 @@ import (
"io"
"os"
"os/signal"
"sort"
"syscall"
"github.com/urfave/cli/v2"
@@ -77,9 +76,6 @@ Examples:
Flags: []cli.Flag{
&cli.BoolFlag{Name: "json", Usage: "Print durable run history as JSON for automation"},
&cli.BoolFlag{Name: "pending", Usage: "Only show runs that have not completed"},
&cli.StringFlag{Name: "status", Usage: "Only show runs with this status (running, done, failed)"},
&cli.IntFlag{Name: "limit", Usage: "Show the most recently updated N runs"},
&cli.StringFlag{Name: "stage", Usage: "Only show runs currently checkpointed at this stage"},
},
Action: flowRuns,
},
@@ -134,11 +130,9 @@ func flowRuns(c *cli.Context) error {
if err != nil {
return err
}
opts, err := validateFlowRunOptions(flowRunOptions{Pending: c.Bool("pending"), Status: c.String("status"), Stage: c.String("stage"), Limit: c.Int("limit")})
if err != nil {
return err
if c.Bool("pending") {
runs = pendingFlowRuns(runs)
}
runs = filterFlowRuns(runs, opts)
if len(runs) == 0 {
if c.Bool("pending") {
fmt.Printf(" No pending runs recorded for flow %q.\n", name)
@@ -150,58 +144,17 @@ func flowRuns(c *cli.Context) error {
return writeFlowRuns(os.Stdout, runs, c.Bool("json"))
}
type flowRunOptions struct {
Pending bool
Status string
Stage string
Limit int
}
func validateFlowRunOptions(opts flowRunOptions) (flowRunOptions, error) {
switch opts.Status {
case "", "running", "done", "failed":
default:
return opts, fmt.Errorf("invalid run status %q: expected running, done, or failed", opts.Status)
}
if opts.Limit < 0 {
return opts, fmt.Errorf("invalid limit %d: expected a non-negative value", opts.Limit)
}
return opts, nil
}
func filterFlowRuns(runs []aiflow.Run, opts flowRunOptions) []aiflow.Run {
func pendingFlowRuns(runs []aiflow.Run) []aiflow.Run {
if len(runs) == 0 {
return nil
}
filtered := make([]aiflow.Run, 0, len(runs))
pending := make([]aiflow.Run, 0, len(runs))
for _, run := range runs {
if opts.Pending && run.Status == "done" {
continue
if run.Status != "done" {
pending = append(pending, run)
}
if opts.Status != "" && run.Status != opts.Status {
continue
}
if opts.Stage != "" && run.State.Stage != opts.Stage {
continue
}
filtered = append(filtered, run)
}
if opts.Limit > 0 && len(filtered) > opts.Limit {
sort.SliceStable(filtered, func(i, j int) bool {
if filtered[i].Updated.Equal(filtered[j].Updated) {
return filtered[i].Started.Before(filtered[j].Started)
}
return filtered[i].Updated.Before(filtered[j].Updated)
})
start := len(filtered) - opts.Limit
limited := append([]aiflow.Run(nil), filtered[start:]...)
return limited
}
return filtered
}
func pendingFlowRuns(runs []aiflow.Run) []aiflow.Run {
return filterFlowRuns(runs, flowRunOptions{Pending: true})
return pending
}
func writeFlowRuns(w io.Writer, runs []aiflow.Run, asJSON bool) error {
@@ -210,7 +163,6 @@ func writeFlowRuns(w io.Writer, runs []aiflow.Run, asJSON bool) error {
enc.SetIndent("", " ")
return enc.Encode(runs)
}
fmt.Fprintf(w, " %d run%s\n", len(runs), plural(len(runs)))
for _, r := range runs {
id := r.ID
if len(id) > 8 {
@@ -233,13 +185,6 @@ func writeFlowRuns(w io.Writer, runs []aiflow.Run, asJSON bool) error {
return nil
}
func plural(n int) string {
if n == 1 {
return ""
}
return "s"
}
func flowFlags() []cli.Flag {
return []cli.Flag{
&cli.StringFlag{Name: "trigger", Usage: "Broker topic to subscribe to", EnvVars: []string{"MICRO_FLOW_TRIGGER"}},
-83
View File
@@ -29,7 +29,6 @@ func TestWriteFlowRunsIncludesStepDetails(t *testing.T) {
}
got := out.String()
for _, want := range []string{
"1 run",
"12345678 failed stage=charge",
"updated=2026-06-24T12:30:00Z",
"- reserve done attempts=1",
@@ -41,20 +40,6 @@ func TestWriteFlowRunsIncludesStepDetails(t *testing.T) {
}
}
func TestValidateFlowRunOptionsRejectsInvalidStatus(t *testing.T) {
_, err := validateFlowRunOptions(flowRunOptions{Status: "stuck"})
if err == nil || !strings.Contains(err.Error(), "invalid run status") {
t.Fatalf("expected invalid status error, got %v", err)
}
}
func TestValidateFlowRunOptionsRejectsNegativeLimit(t *testing.T) {
_, err := validateFlowRunOptions(flowRunOptions{Limit: -1})
if err == nil || !strings.Contains(err.Error(), "invalid limit") {
t.Fatalf("expected invalid limit error, got %v", err)
}
}
func TestWriteFlowRunsJSON(t *testing.T) {
runs := []aiflow.Run{{ID: "run-1", Flow: "checkout", Status: "done"}}
@@ -86,71 +71,3 @@ func TestPendingFlowRunsFiltersCompletedRuns(t *testing.T) {
t.Fatalf("pending runs = %+v", got)
}
}
func TestFilterFlowRunsStatus(t *testing.T) {
runs := []aiflow.Run{
{ID: "run-1", Status: "done"},
{ID: "run-2", Status: "failed"},
{ID: "run-3", Status: "running"},
{ID: "run-4", Status: "failed"},
}
got := filterFlowRuns(runs, flowRunOptions{Status: "failed"})
if len(got) != 2 {
t.Fatalf("filterFlowRuns returned %d runs, want 2: %+v", len(got), got)
}
if got[0].ID != "run-2" || got[1].ID != "run-4" {
t.Fatalf("failed runs = %+v", got)
}
}
func TestFilterFlowRunsStage(t *testing.T) {
runs := []aiflow.Run{
{ID: "run-1", Status: "failed", State: aiflow.State{Stage: "reserve"}},
{ID: "run-2", Status: "failed", State: aiflow.State{Stage: "charge"}},
{ID: "run-3", Status: "running", State: aiflow.State{Stage: "charge"}},
{ID: "run-4", Status: "done", State: aiflow.State{}},
}
got := filterFlowRuns(runs, flowRunOptions{Stage: "charge"})
if len(got) != 2 {
t.Fatalf("filterFlowRuns returned %d runs, want 2: %+v", len(got), got)
}
if got[0].ID != "run-2" || got[1].ID != "run-3" {
t.Fatalf("charge-stage runs = %+v", got)
}
}
func TestFilterFlowRunsLimitKeepsNewestRuns(t *testing.T) {
runs := []aiflow.Run{
{ID: "run-1", Status: "done"},
{ID: "run-2", Status: "failed"},
{ID: "run-3", Status: "running"},
}
got := filterFlowRuns(runs, flowRunOptions{Limit: 2})
if len(got) != 2 {
t.Fatalf("filterFlowRuns returned %d runs, want 2: %+v", len(got), got)
}
if got[0].ID != "run-2" || got[1].ID != "run-3" {
t.Fatalf("limited runs = %+v", got)
}
}
func TestFilterFlowRunsCombinesPendingStatusAndLimit(t *testing.T) {
runs := []aiflow.Run{
{ID: "run-1", Status: "failed"},
{ID: "run-2", Status: "done"},
{ID: "run-3", Status: "failed"},
{ID: "run-4", Status: "running"},
{ID: "run-5", Status: "failed"},
}
got := filterFlowRuns(runs, flowRunOptions{Pending: true, Status: "failed", Limit: 2})
if len(got) != 2 {
t.Fatalf("filterFlowRuns returned %d runs, want 2: %+v", len(got), got)
}
if got[0].ID != "run-3" || got[1].ID != "run-5" {
t.Fatalf("filtered runs = %+v", got)
}
}
+1 -11
View File
@@ -2,7 +2,6 @@ package flow
import (
"context"
"encoding/json"
"testing"
"go-micro.dev/v6/client"
@@ -26,17 +25,11 @@ func (c *fakeClient) Call(ctx context.Context, req client.Request, rsp interface
func TestExecuteDispatchesToAgent(t *testing.T) {
f := New("welcome", Agent("comms"), Prompt("welcome {{.Data}}"))
var svc, ep, parentID string
var svc, ep string
f.client = &fakeClient{
Client: client.DefaultClient,
callFn: func(req client.Request, rsp interface{}) error {
svc, ep = req.Service(), req.Endpoint()
reqFrame := req.Body().(*codecbytes.Frame)
var body map[string]string
if err := json.Unmarshal(reqFrame.Data, &body); err != nil {
t.Fatalf("request body: %v", err)
}
parentID = body["parent_id"]
frame := rsp.(*codecbytes.Frame)
frame.Data = []byte(`{"reply":"welcomed bob","agent":"comms"}`)
return nil
@@ -50,9 +43,6 @@ func TestExecuteDispatchesToAgent(t *testing.T) {
if svc != "comms" || ep != "Agent.Chat" {
t.Errorf("dispatched to %s.%s, want comms.Agent.Chat", svc, ep)
}
if parentID == "" {
t.Fatal("dispatch request parent_id is empty")
}
results := f.Results()
if len(results) != 1 {
+1 -5
View File
@@ -205,9 +205,6 @@ func (f *Flow) Execute(ctx context.Context, data string) error {
return err
}
runID := uuid.New().String()
ctx = ai.WithRunInfo(ctx, ai.RunInfo{RunID: runID, Agent: f.name})
start := time.Now()
prompt := data
@@ -280,8 +277,7 @@ func (f *Flow) Execute(ctx context.Context, data string) error {
// callAgent hands the rendered prompt to a registered agent's Agent.Chat
// endpoint over RPC and returns its reply.
func (f *Flow) callAgent(ctx context.Context, name, message string) (string, error) {
info, _ := ai.RunInfoFrom(ctx)
body, _ := json.Marshal(map[string]string{"message": message, "parent_id": info.RunID})
body, _ := json.Marshal(map[string]string{"message": message})
req := f.client.NewRequest(name, "Agent.Chat", &codecbytes.Frame{Data: body})
var rsp codecbytes.Frame
if err := f.client.Call(ctx, req, &rsp); err != nil {
+1 -2
View File
@@ -240,8 +240,7 @@ func Dispatch(name string) StepFunc {
if d := depsFrom(ctx); d != nil && d.client != nil {
cl = d.client
}
info, _ := ai.RunInfoFrom(ctx)
body, _ := json.Marshal(map[string]string{"message": in.String(), "parent_id": info.RunID})
body, _ := json.Marshal(map[string]string{"message": in.String()})
req := cl.NewRequest(name, "Agent.Chat", &codecbytes.Frame{Data: body})
var rsp codecbytes.Frame
if err := cl.Call(ctx, req, &rsp); err != nil {
+11 -74
View File
@@ -51,8 +51,6 @@ func main() {
requireConfiguredFlag := flag.Bool("require-configured", false, "fail when a selected live provider is missing an API key")
capabilitiesFlag := flag.Bool("capabilities", true, "print the registered provider capability matrix before running conformance")
summaryJSONFlag := flag.String("summary-json", "", "write a machine-readable conformance summary to this path")
summaryMarkdownFlag := flag.String("summary-markdown", "", "write a human-readable conformance summary to this path")
capabilityMarkdownFlag := flag.String("capabilities-markdown", "", "write the registered provider capability matrix as a Markdown table")
flag.Parse()
providers := splitCSV(*providersFlag)
@@ -65,12 +63,6 @@ func main() {
if *capabilitiesFlag {
printCapabilityMatrix()
}
if *capabilityMarkdownFlag != "" {
if err := writeCapabilityMarkdown(*capabilityMarkdownFlag, ai.CapabilityRows()); err != nil {
fmt.Fprintf(os.Stderr, "write capabilities markdown: %v\n", err)
os.Exit(1)
}
}
var ran, skipped, failed int
var results []conformanceResult
@@ -103,27 +95,21 @@ func main() {
}
fmt.Printf("\nprovider conformance: %d passed, %d skipped providers, %d failed\n", ran, skipped, failed)
summary := conformanceSummary{
Providers: providers,
Harnesses: harnesses,
Capabilities: ai.CapabilityRows(),
Results: results,
Passed: ran,
Skipped: skipped,
Failed: failed,
}
if *summaryJSONFlag != "" {
summary := conformanceSummary{
Providers: providers,
Harnesses: harnesses,
Capabilities: ai.CapabilityRows(),
Results: results,
Passed: ran,
Skipped: skipped,
Failed: failed,
}
if err := writeSummaryJSON(*summaryJSONFlag, summary); err != nil {
fmt.Fprintf(os.Stderr, "write summary: %v\n", err)
os.Exit(1)
}
}
if *summaryMarkdownFlag != "" {
if err := writeSummaryMarkdown(*summaryMarkdownFlag, summary); err != nil {
fmt.Fprintf(os.Stderr, "write summary markdown: %v\n", err)
os.Exit(1)
}
}
if failed > 0 {
os.Exit(1)
}
@@ -161,60 +147,11 @@ func writeSummaryJSON(path string, summary conformanceSummary) error {
return os.WriteFile(path, b, 0o644)
}
func writeCapabilityMarkdown(path string, rows []ai.CapabilityRow) error {
return os.WriteFile(path, []byte(capabilityMarkdown(rows)), 0o644)
}
func writeSummaryMarkdown(path string, summary conformanceSummary) error {
var b strings.Builder
b.WriteString("# Provider conformance summary\n\n")
fmt.Fprintf(&b, "Passed: %d. Skipped providers: %d. Failed: %d.\n\n", summary.Passed, summary.Skipped, summary.Failed)
b.WriteString("## Capability matrix\n\n")
b.WriteString(capabilityMarkdown(summary.Capabilities))
b.WriteString("\n## Harness results\n\n")
b.WriteString("| Provider | Harness | Status | Detail |\n")
b.WriteString("| --- | --- | --- | --- |\n")
for _, result := range summary.Results {
harness := result.Harness
if harness == "" {
harness = "—"
}
fmt.Fprintf(&b, "| %s | %s | %s | %s |\n", result.Provider, harness, result.Status, markdownCell(result.Error))
}
return os.WriteFile(path, []byte(b.String()), 0o644)
}
func capabilityMarkdown(rows []ai.CapabilityRow) string {
var b strings.Builder
b.WriteString("| Provider | Model | Image | Video | Streaming |\n")
b.WriteString("| --- | --- | --- | --- | --- |\n")
for _, row := range rows {
fmt.Fprintf(&b, "| %s | %s | %s | %s | %s |\n", row.Provider, mark(row.Model), mark(row.Image), mark(row.Video), mark(row.Stream))
}
return b.String()
}
func markdownCell(s string) string {
if s == "" {
return "—"
}
s = strings.ReplaceAll(s, "|", "\\|")
s = strings.ReplaceAll(s, "\n", "<br>")
return s
}
func mark(ok bool) string {
if ok {
return "✅"
}
return "—"
}
func printCapabilityMatrix() {
fmt.Println("Provider capability matrix:")
fmt.Println("provider model image video stream")
fmt.Println("provider model image video")
for _, row := range ai.CapabilityRows() {
fmt.Printf("%-12s %-5s %-5s %-5s %-6s\n", row.Provider, yesNo(row.Model), yesNo(row.Image), yesNo(row.Video), yesNo(row.Stream))
fmt.Printf("%-12s %-5s %-5s %-5s\n", row.Provider, yesNo(row.Model), yesNo(row.Image), yesNo(row.Video))
}
fmt.Println()
}
@@ -56,65 +56,6 @@ func TestCapabilityMatrixHasRegisteredProviders(t *testing.T) {
}
}
func TestWriteCapabilityMarkdown(t *testing.T) {
path := filepath.Join(t.TempDir(), "capabilities.md")
rows := []ai.CapabilityRow{
{Provider: "mock", Capabilities: ai.Capabilities{Model: true}},
{Provider: "vision", Capabilities: ai.Capabilities{Image: true, Video: true}},
}
if err := writeCapabilityMarkdown(path, rows); err != nil {
t.Fatalf("writeCapabilityMarkdown returned error: %v", err)
}
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read capabilities markdown: %v", err)
}
got := string(b)
for _, want := range []string{
"| Provider | Model | Image | Video | Streaming |",
"| mock | ✅ | — | — | — |",
"| vision | — | ✅ | ✅ | — |",
} {
if !strings.Contains(got, want) {
t.Fatalf("capabilities markdown = %q, want row %q", got, want)
}
}
}
func TestWriteSummaryMarkdown(t *testing.T) {
path := filepath.Join(t.TempDir(), "summary.md")
summary := conformanceSummary{
Capabilities: []ai.CapabilityRow{{Provider: "mock", Capabilities: ai.Capabilities{Model: true}}},
Results: []conformanceResult{
{Provider: "mock", Harness: "agent-flow", Status: statusPassed},
{Provider: "live", Status: statusSkipped, Error: "missing | key"},
},
Passed: 1,
Skipped: 1,
}
if err := writeSummaryMarkdown(path, summary); err != nil {
t.Fatalf("writeSummaryMarkdown returned error: %v", err)
}
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read summary markdown: %v", err)
}
got := string(b)
for _, want := range []string{
"# Provider conformance summary",
"Passed: 1. Skipped providers: 1. Failed: 0.",
"| mock | ✅ | — | — | — |",
"| mock | agent-flow | passed | — |",
"| live | — | skipped | missing \\| key |",
} {
if !strings.Contains(got, want) {
t.Fatalf("summary markdown = %q, want %q", got, want)
}
}
}
func TestWriteSummaryJSON(t *testing.T) {
path := filepath.Join(t.TempDir(), "summary.json")
summary := conformanceSummary{
@@ -30,21 +30,21 @@ imports are linked in:
```go
for _, row := range ai.CapabilityRows() {
fmt.Printf("%s: chat=%t image=%t video=%t stream=%t\n", row.Provider, row.Model, row.Image, row.Video, row.Stream)
fmt.Printf("%s: chat=%t image=%t video=%t\n", row.Provider, row.Model, row.Image, row.Video)
}
```
The built-in providers currently register these capability interfaces:
| Provider | Chat/text (`ai.Model`) | Image (`ai.ImageModel`) | Video (`ai.VideoModel`) | Streaming (`ai.Stream`) |
| --- | --- | --- | --- | --- |
| `anthropic` | Yes | No | No | No |
| `atlascloud` | Yes | Yes | Yes | No |
| `gemini` | Yes | No | No | No |
| `groq` | Yes | No | No | No |
| `mistral` | Yes | No | No | No |
| `openai` | Yes | Yes | No | No |
| `together` | Yes | No | No | No |
| Provider | Chat/text (`ai.Model`) | Image (`ai.ImageModel`) | Video (`ai.VideoModel`) |
| --- | --- | --- | --- |
| `anthropic` | Yes | No | No |
| `atlascloud` | Yes | Yes | Yes |
| `gemini` | Yes | No | No |
| `groq` | Yes | No | No |
| `mistral` | Yes | No | No |
| `openai` | Yes | Yes | No |
| `together` | Yes | No | No |
## Step 1: Implement the `ai.Model` Interface
@@ -90,15 +90,12 @@ when you only want pass/fail output.
For automation, add `-summary-json` to capture the selected providers,
harnesses, registered capability rows, and pass/skip/fail results in a stable
machine-readable file. Add `-capabilities-markdown` when you also want a
ready-to-publish Markdown support table for release notes, docs, or issue
updates:
machine-readable file:
```sh
go run ./internal/harness/provider-conformance \
-providers mock \
-summary-json provider-conformance-summary.json \
-capabilities-markdown provider-capabilities.md
-summary-json provider-conformance-summary.json
```
## Related docs
@@ -1,47 +0,0 @@
package guides_test
import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"go-micro.dev/v6/ai"
_ "go-micro.dev/v6/ai/anthropic"
_ "go-micro.dev/v6/ai/atlascloud"
_ "go-micro.dev/v6/ai/gemini"
_ "go-micro.dev/v6/ai/groq"
_ "go-micro.dev/v6/ai/mistral"
_ "go-micro.dev/v6/ai/openai"
_ "go-micro.dev/v6/ai/together"
)
func TestAIProviderGuideCapabilityMatrixMatchesRegistry(t *testing.T) {
_, filename, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
guidePath := filepath.Join(filepath.Dir(filename), "ai-provider-guide.md")
b, err := os.ReadFile(guidePath)
if err != nil {
t.Fatalf("read AI provider guide: %v", err)
}
guide := string(b)
for _, row := range ai.CapabilityRows() {
want := fmt.Sprintf("| `%s` | %s | %s | %s | %s |", row.Provider, yesNo(row.Model), yesNo(row.Image), yesNo(row.Video), yesNo(row.Stream))
if !strings.Contains(guide, want) {
t.Fatalf("AI provider guide capability matrix is stale; missing row %q", want)
}
}
}
func yesNo(ok bool) string {
if ok {
return "Yes"
}
return "No"
}