chore: import upstream snapshot with attribution

2026-07-13 13:32:57 +08:00
commit cd420f9332
4811 changed files with 884702 additions and 0 deletions
@@ -0,0 +1,374 @@
+import { evalite } from "evalite";
+import { Levenshtein } from "autoevals";
+import { AIQueryService } from "~/v3/services/aiQueryService.server";
+import { runsSchema } from "~/v3/querySchemas";
+import dotenv from "dotenv";
+import { wrapAISDKModel } from "evalite/ai-sdk";
+import { openai } from "@ai-sdk/openai";
+
+dotenv.config({ path: "../../.env" });
+
+// Helper to normalize queries for comparison
+function normalizeQuery(query: string): string {
+  return query
+    .replace(/\s+/g, " ")
+    .replace(/\(\s+/g, "(")
+    .replace(/\s+\)/g, ")")
+    .trim()
+    .toLowerCase();
+}
+
+// Type for parsed query results
+interface ParsedQueryResult {
+  success: boolean;
+  query?: string;
+  error?: string;
+}
+
+// Custom scorer that checks if the generated query is semantically similar
+// and also syntactically valid
+const QuerySimilarity = {
+  name: "QuerySimilarity",
+  scorer: async ({
+    input,
+    output,
+    expected,
+  }: {
+    input: string;
+    output: string;
+    expected?: string;
+  }) => {
+    if (!expected) {
+      return 0;
+    }
+
+    // Parse the output to extract the query
+    const outputParsed = JSON.parse(output) as ParsedQueryResult;
+    const expectedParsed = JSON.parse(expected) as ParsedQueryResult;
+
+    // Check success status first
+    if (outputParsed.success !== expectedParsed.success) {
+      return 0;
+    }
+
+    // If both failed, check if error messages are similar
+    if (!outputParsed.success && !expectedParsed.success) {
+      // Give partial credit for correctly identifying an error case
+      return 0.5;
+    }
+
+    // If both succeeded, compare the queries
+    if (outputParsed.success && expectedParsed.success) {
+      const normalizedOutput = normalizeQuery(outputParsed.query ?? "");
+      const normalizedExpected = normalizeQuery(expectedParsed.query ?? "");
+
+      // Key patterns to check
+      const patterns = [
+        // Table name
+        /from\s+runs/i,
+        // Status filter patterns
+        /status\s*=\s*'[^']+'/i,
+        /status\s+in\s*\([^)]+\)/i,
+        // Time patterns
+        /interval\s+\d+\s+(day|hour|minute|week|month)/i,
+        /triggered_at\s*>/i,
+        // Aggregation patterns
+        /count\(\)/i,
+        /sum\(/i,
+        /avg\(/i,
+        /group\s+by/i,
+        // Ordering
+        /order\s+by/i,
+        // Limit
+        /limit\s+\d+/i,
+      ];
+
+      let matchScore = 0;
+      let totalPatterns = 0;
+
+      for (const pattern of patterns) {
+        const outputMatch = pattern.test(normalizedOutput);
+        const expectedMatch = pattern.test(normalizedExpected);
+
+        if (expectedMatch) {
+          totalPatterns++;
+          if (outputMatch) {
+            matchScore++;
+          }
+        }
+      }
+
+      // Base score from pattern matching
+      const patternScore = totalPatterns > 0 ? matchScore / totalPatterns : 0.5;
+
+      // Use Levenshtein for overall similarity
+      const levenshteinResult = await Levenshtein({
+        output: normalizedOutput,
+        expected: normalizedExpected,
+      });
+      const levenshteinScore = levenshteinResult?.score ?? 0;
+
+      // Weighted combination
+      return 0.6 * patternScore + 0.4 * levenshteinScore;
+    }
+
+    return 0;
+  },
+};
+
+evalite("AI Query Generator", {
+  data: async () => {
+    return [
+      // Basic SELECT queries
+      {
+        input: "Show me all runs",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+LIMIT 100`,
+        }),
+      },
+      {
+        input: "Get the 10 most recent runs",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+ORDER BY triggered_at DESC
+LIMIT 10`,
+        }),
+      },
+
+      // Status filtering
+      {
+        input: "Show failed runs",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE status = 'Failed'
+LIMIT 100`,
+        }),
+      },
+      {
+        input: "Get all completed runs",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE status = 'Completed'
+LIMIT 100`,
+        }),
+      },
+      {
+        input: "Find runs that crashed or timed out",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE status IN ('Crashed', 'Timed out')
+LIMIT 100`,
+        }),
+      },
+
+      // Time-based filtering
+      {
+        input: "Runs from the last 7 days",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE triggered_at > now() - INTERVAL 7 DAY
+LIMIT 100`,
+        }),
+      },
+      {
+        input: "Show runs from the past hour",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE triggered_at > now() - INTERVAL 1 HOUR
+LIMIT 100`,
+        }),
+      },
+      {
+        input: "Failed runs in the last 24 hours",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE status = 'Failed'
+  AND triggered_at > now() - INTERVAL 1 DAY
+ORDER BY triggered_at DESC
+LIMIT 100`,
+        }),
+      },
+
+      // Aggregations
+      {
+        input: "Count of runs by status",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT status, count() AS count
+FROM runs
+GROUP BY status
+ORDER BY count DESC`,
+        }),
+      },
+      {
+        input: "How many runs per task?",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT task_identifier, count() AS run_count
+FROM runs
+GROUP BY task_identifier
+ORDER BY run_count DESC
+LIMIT 100`,
+        }),
+      },
+      {
+        input: "Average execution duration by task",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT task_identifier, avg(execution_duration) AS avg_duration
+FROM runs
+GROUP BY task_identifier
+ORDER BY avg_duration DESC
+LIMIT 100`,
+        }),
+      },
+      {
+        input: "Total cost by task in the last 30 days",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT task_identifier, sum(total_cost) AS total_cost
+FROM runs
+WHERE triggered_at > now() - INTERVAL 30 DAY
+GROUP BY task_identifier
+ORDER BY total_cost DESC
+LIMIT 100`,
+        }),
+      },
+
+      // Complex queries
+      {
+        input: "Top 10 most expensive failed runs from last week",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT run_id, task_identifier, status, total_cost, triggered_at
+FROM runs
+WHERE status = 'Failed'
+  AND triggered_at > now() - INTERVAL 7 DAY
+ORDER BY total_cost DESC
+LIMIT 10`,
+        }),
+      },
+      {
+        input: "Runs using large machines that took more than 5 minutes",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE machine IN ('large-1x', 'large-2x')
+  AND usage_duration > 300000
+LIMIT 100`,
+        }),
+      },
+      {
+        input: "Show p95 execution duration by task for completed runs",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT task_identifier, quantile(0.95)(execution_duration) AS p95_duration
+FROM runs
+WHERE status = 'Completed'
+  AND execution_duration IS NOT NULL
+GROUP BY task_identifier
+ORDER BY p95_duration DESC
+LIMIT 100`,
+        }),
+      },
+
+      // Specific columns
+      {
+        input: "Just show run IDs and their statuses",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT run_id, status
+FROM runs
+LIMIT 100`,
+        }),
+      },
+      {
+        input: "Get run_id, task, status and cost for recent runs",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT run_id, task_identifier, status, total_cost
+FROM runs
+ORDER BY triggered_at DESC
+LIMIT 100`,
+        }),
+      },
+
+      // Root runs
+      {
+        input: "Show only root runs (not child runs)",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE is_root_run = 1
+LIMIT 100`,
+        }),
+      },
+
+      // Queue filtering
+      {
+        input: "Runs in the shared queue",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE queue LIKE '%shared%'
+LIMIT 100`,
+        }),
+      },
+
+      // Tags
+      {
+        input: "Find runs with tag 'important'",
+        expected: JSON.stringify({
+          success: true,
+          query: `SELECT *
+FROM runs
+WHERE has(tags, 'important')
+LIMIT 100`,
+        }),
+      },
+
+      // Error cases
+      {
+        input: "Do something",
+        expected: JSON.stringify({
+          success: false,
+          error: "Please be more specific about what data you want to query",
+        }),
+      },
+      {
+        input: "Show me the weather",
+        expected: JSON.stringify({
+          success: false,
+          error: "I can only generate queries for task run data",
+        }),
+      },
+    ];
+  },
+  task: async (input) => {
+    const service = new AIQueryService([runsSchema], wrapAISDKModel(openai("gpt-4o-mini")));
+
+    const result = await service.call(input);
+    return JSON.stringify(result);
+  },
+  scorers: [QuerySimilarity, Levenshtein],
+});
@@ -0,0 +1,286 @@
+import { evalite } from "evalite";
+import { Levenshtein } from "autoevals";
+import {
+  AIRunFilterService,
+  type QueryQueues,
+  type QueryTags,
+  type QueryTasks,
+  type QueryVersions,
+} from "~/v3/services/aiRunFilterService.server";
+import dotenv from "dotenv";
+import { wrapAISDKModel } from "evalite/ai-sdk";
+import { openai } from "@ai-sdk/openai";
+
+dotenv.config({ path: "../../.env" });
+
+const queryTags: QueryTags = {
+  query: async (search) => {
+    return {
+      tags: ["user_1", "user_2", "org_1", "org_2"],
+    };
+  },
+};
+
+const queryVersions: QueryVersions = {
+  query: async (versionPrefix, isCurrent) => {
+    if (isCurrent) {
+      return {
+        version: "20250721.1",
+      };
+    }
+
+    return {
+      versions: ["20250721.1", "20250720.2", "20250720.1"],
+    };
+  },
+};
+
+const queryQueues: QueryQueues = {
+  query: async (query, type) => {
+    return {
+      queues: ["shared", "paid"],
+    };
+  },
+};
+
+const queryTasks: QueryTasks = {
+  query: async () => {
+    return {
+      tasks: [
+        { slug: "email-sender", triggerSource: "STANDARD" },
+        { slug: "email-sender-scheduled", triggerSource: "SCHEDULED" },
+      ],
+    };
+  },
+};
+
+evalite("AI Run Filter", {
+  data: async () => {
+    return [
+      // Basic status filtering
+      {
+        input: "Completed runs",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["COMPLETED_SUCCESSFULLY"],
+          },
+        }),
+      },
+      {
+        input: "Failed runs",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["COMPLETED_WITH_ERRORS", "CRASHED", "TIMED_OUT", "SYSTEM_FAILURE"],
+          },
+        }),
+      },
+      {
+        input: "Executing runs",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["EXECUTING", "RETRYING_AFTER_FAILURE", "WAITING_TO_RESUME"],
+          },
+        }),
+      },
+      // Time filters
+      {
+        input: "Runs from the past 7 days",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            period: "7d",
+          },
+        }),
+      },
+      {
+        input: "Runs from the last hour",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            period: "1h",
+          },
+        }),
+      },
+      {
+        input: "Runs from this month",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            period: "30d",
+          },
+        }),
+      },
+      {
+        input: "June 16",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            from: new Date("2025-06-16").getTime(),
+            to: new Date("2025-06-17").getTime(),
+          },
+        }),
+      },
+      // Combined filters
+      {
+        input: "Failed runs from the past week",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["COMPLETED_WITH_ERRORS", "CRASHED", "TIMED_OUT", "SYSTEM_FAILURE"],
+            period: "7d",
+          },
+        }),
+      },
+      {
+        input: "Successful runs from the last 24 hours",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["COMPLETED_SUCCESSFULLY"],
+            period: "1d",
+          },
+        }),
+      },
+      // Root-only filtering
+      {
+        input: "Root runs only",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            rootOnly: true,
+          },
+        }),
+      },
+      {
+        input: "Failed root runs from yesterday",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["COMPLETED_WITH_ERRORS", "CRASHED", "TIMED_OUT", "SYSTEM_FAILURE"],
+            rootOnly: true,
+            period: "1d",
+          },
+        }),
+      },
+      // Machine filtering
+      {
+        input: "Runs using large machines",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            machines: ["large-1x", "large-2x"],
+          },
+        }),
+      },
+      // Edge cases and error handling
+      {
+        input: "Runs with tag production",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            tags: ["production"],
+          },
+        }),
+      },
+      {
+        input: "Runs from task email-sender",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            tasks: ["email-sender"],
+          },
+        }),
+      },
+      {
+        input: "Runs in the shared queue",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            queues: ["shared"],
+          },
+        }),
+      },
+      // Complex combinations
+      {
+        input: "Failed production runs from the past 3 days using large machines",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["COMPLETED_WITH_ERRORS", "CRASHED", "TIMED_OUT", "SYSTEM_FAILURE"],
+            tags: ["production"],
+            period: "3d",
+            machines: ["large-1x", "large-2x"],
+          },
+        }),
+      },
+      // Time-specific queries with hours
+      {
+        input: "tasks that started around 8am today",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            from: new Date(new Date().toDateString() + " 07:00:00").getTime(),
+            to: new Date(new Date().toDateString() + " 09:00:00").getTime(),
+          },
+        }),
+      },
+      {
+        input: "runs that started at 2pm yesterday",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            from: new Date(
+              new Date(Date.now() - 24 * 60 * 60 * 1000).toDateString() + " 14:00:00"
+            ).getTime(),
+            to: new Date(
+              new Date(Date.now() - 24 * 60 * 60 * 1000).toDateString() + " 14:59:59"
+            ).getTime(),
+          },
+        }),
+      },
+      {
+        input: "any runs started this morning",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            from: new Date(new Date().toDateString() + " 06:00:00").getTime(),
+            to: new Date(new Date().toDateString() + " 12:00:00").getTime(),
+          },
+        }),
+      },
+      // Ambiguous cases that should return errors
+      {
+        input: "Show me something",
+        expected: JSON.stringify({
+          success: false,
+          error: "Unclear what to filter",
+        }),
+      },
+      {
+        input: "Runs with unknown status",
+        expected: JSON.stringify({
+          success: false,
+          error: "Unknown status specified",
+        }),
+      },
+    ];
+  },
+  task: async (input) => {
+    const service = new AIRunFilterService(
+      {
+        queryTags,
+        queryVersions,
+        queryQueues,
+        queryTasks,
+      },
+      wrapAISDKModel(openai("gpt-4o-mini"))
+    );
+
+    const result = await service.call(input, "123456");
+    return JSON.stringify(result);
+  },
+  scorers: [Levenshtein],
+});