wehub-resource-sync
e071084ebe
govulncheck / govulncheck (push) Waiting to run
Harness (E2E) / Harnesses (mock LLM) (push) Waiting to run
Harness (E2E) / Provider harnesses (live LLM conformance) (push) Waiting to run
Lint / golangci-lint (push) Waiting to run
Run Tests / Unit Tests (push) Waiting to run
Run Tests / Etcd Integration Tests (push) Waiting to run
90 lines
3.3 KiB
Go
90 lines
3.3 KiB
Go
package flow
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"go-micro.dev/v6/ai"
|
|
)
|
|
|
|
func TestAnalyzeRanksFailedGraderStepAbovePassingStep(t *testing.T) {
|
|
now := time.Now()
|
|
runs := []Run{
|
|
{ID: "run-1", Started: now, Updated: now.Add(time.Second), Steps: []StepRecord{
|
|
{Name: "draft", Status: "done", Attempts: 2, Result: `{"verification_passed":false,"verification_feedback":"cite sources"}`},
|
|
{Name: "publish", Status: "done", Attempts: 1, Result: `{"verification_passed":true,"verification_feedback":"ok"}`},
|
|
}},
|
|
{ID: "run-2", Started: now, Updated: now.Add(2 * time.Second), Steps: []StepRecord{
|
|
{Name: "draft", Status: "done", Attempts: 1, Result: `{"verification_passed":false,"verification_feedback":"too vague"}`},
|
|
{Name: "publish", Status: "done", Attempts: 1, Result: `{"verification_passed":true,"verification_feedback":"ok"}`},
|
|
}},
|
|
}
|
|
|
|
report := Analyze(runs)
|
|
if len(report.Candidates) != 2 {
|
|
t.Fatalf("Analyze returned %d candidates, want 2", len(report.Candidates))
|
|
}
|
|
if got := report.Candidates[0].Step; got != "draft" {
|
|
t.Fatalf("top candidate = %q, want draft", got)
|
|
}
|
|
if report.Candidates[0].PassRate != 0 {
|
|
t.Fatalf("draft pass rate = %v, want 0", report.Candidates[0].PassRate)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeCarriesFeedbackSamplesAndRunIDs(t *testing.T) {
|
|
report := Analyze([]Run{{ID: "run-9", Steps: []StepRecord{{
|
|
Name: "grade", Status: "done", Attempts: 3,
|
|
Result: `{"verification_passed":false,"verification_feedback":"include totals"}`,
|
|
}}}})
|
|
if len(report.Candidates) != 1 {
|
|
t.Fatalf("candidates = %d, want 1", len(report.Candidates))
|
|
}
|
|
c := report.Candidates[0]
|
|
if len(c.SampleFeedback) != 1 || c.SampleFeedback[0] != "include totals" {
|
|
t.Fatalf("feedback = %#v, want include totals", c.SampleFeedback)
|
|
}
|
|
if len(c.RunIDs) != 1 || c.RunIDs[0] != "run-9" {
|
|
t.Fatalf("run ids = %#v, want run-9", c.RunIDs)
|
|
}
|
|
if c.AverageRetries != 2 {
|
|
t.Fatalf("average retries = %v, want 2", c.AverageRetries)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeEmptyWindowReturnsEmptyReport(t *testing.T) {
|
|
if got := Analyze(nil); len(got.Candidates) != 0 {
|
|
t.Fatalf("empty Analyze candidates = %d, want 0", len(got.Candidates))
|
|
}
|
|
}
|
|
|
|
func TestLLMOptimizerReturnsProposalWithoutMutatingFlow(t *testing.T) {
|
|
f := New("optimize", Prompt("original prompt"))
|
|
before := f.opts.Prompt
|
|
optimizer := LLMOptimizer(&optimizerModel{reply: "revised prompt"})
|
|
proposal, err := optimizer.OptimizePrompt(context.Background(), Candidate{Step: "draft", Metric: "pass_rate", SampleFeedback: []string{"cite sources"}}, before)
|
|
if err != nil {
|
|
t.Fatalf("OptimizePrompt returned error: %v", err)
|
|
}
|
|
if !strings.Contains(proposal, "revised") {
|
|
t.Fatalf("proposal = %q, want revised prompt", proposal)
|
|
}
|
|
if f.opts.Prompt != before {
|
|
t.Fatalf("flow prompt mutated to %q, want %q", f.opts.Prompt, before)
|
|
}
|
|
}
|
|
|
|
type optimizerModel struct{ reply string }
|
|
|
|
func (m *optimizerModel) Init(...ai.Option) error { return nil }
|
|
func (m *optimizerModel) Options() ai.Options { return ai.Options{} }
|
|
func (m *optimizerModel) Generate(context.Context, *ai.Request, ...ai.GenerateOption) (*ai.Response, error) {
|
|
return &ai.Response{Reply: m.reply}, nil
|
|
}
|
|
func (m *optimizerModel) Stream(context.Context, *ai.Request, ...ai.GenerateOption) (ai.Stream, error) {
|
|
return nil, ai.ErrStreamingUnsupported
|
|
}
|
|
func (m *optimizerModel) String() string { return "optimizer" }
|