chore: import upstream snapshot with attribution
This commit is contained in:
Vendored
+3
@@ -0,0 +1,3 @@
|
||||
# Redis
|
||||
|
||||
This is a simple package that is used to return a valid Redis client and provides an error callback. It will log and swallow errors if they're not handled.
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "@internal/cache",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@internal/redis": "workspace:*",
|
||||
"@trigger.dev/core": "workspace:*",
|
||||
"@unkey/cache": "^1.5.0",
|
||||
"@unkey/error": "^0.2.0",
|
||||
"lru-cache": "^11.2.4",
|
||||
"superjson": "^2.2.1"
|
||||
},
|
||||
"scripts": {
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest",
|
||||
"test:watch": "vitest"
|
||||
}
|
||||
}
|
||||
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
export {
|
||||
createCache,
|
||||
DefaultStatefulContext,
|
||||
Namespace,
|
||||
type Cache as UnkeyCache,
|
||||
type CacheError,
|
||||
} from "@unkey/cache";
|
||||
export { type Result, Ok, Err } from "@unkey/error";
|
||||
export { RedisCacheStore } from "./stores/redis.js";
|
||||
export { createMemoryStore, type MemoryStore } from "./stores/memory.js";
|
||||
export {
|
||||
LRUMemoryStore,
|
||||
createLRUMemoryStore,
|
||||
type LRUMemoryStoreConfig,
|
||||
} from "./stores/lruMemory.js";
|
||||
@@ -0,0 +1,333 @@
|
||||
import type { Entry } from "@unkey/cache/stores";
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { LRUMemoryStore, createLRUMemoryStore } from "./lruMemory.js";
|
||||
|
||||
function createEntry<T>(value: T, freshUntil: number, staleUntil: number): Entry<T> {
|
||||
return { value, freshUntil, staleUntil };
|
||||
}
|
||||
|
||||
describe("LRUMemoryStore", () => {
|
||||
let store: LRUMemoryStore<string, string>;
|
||||
|
||||
beforeEach(() => {
|
||||
store = new LRUMemoryStore({ max: 5, name: "test-store" });
|
||||
});
|
||||
|
||||
describe("basic operations", () => {
|
||||
it("should set and get a value", async () => {
|
||||
const entry = createEntry("test-value", Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
const setResult = await store.set("ns", "key1", entry);
|
||||
expect(setResult.err).toBeUndefined();
|
||||
|
||||
const getResult = await store.get("ns", "key1");
|
||||
expect(getResult.err).toBeUndefined();
|
||||
expect(getResult.val).toEqual(entry);
|
||||
});
|
||||
|
||||
it("should return undefined for missing keys", async () => {
|
||||
const result = await store.get("ns", "nonexistent");
|
||||
expect(result.err).toBeUndefined();
|
||||
expect(result.val).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should remove a single key", async () => {
|
||||
const entry = createEntry("value", Date.now() + 60000, Date.now() + 120000);
|
||||
await store.set("ns", "key1", entry);
|
||||
|
||||
const removeResult = await store.remove("ns", "key1");
|
||||
expect(removeResult.err).toBeUndefined();
|
||||
|
||||
const getResult = await store.get("ns", "key1");
|
||||
expect(getResult.val).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should remove multiple keys", async () => {
|
||||
const entry = createEntry("value", Date.now() + 60000, Date.now() + 120000);
|
||||
await store.set("ns", "key1", entry);
|
||||
await store.set("ns", "key2", entry);
|
||||
await store.set("ns", "key3", entry);
|
||||
|
||||
const removeResult = await store.remove("ns", ["key1", "key2"]);
|
||||
expect(removeResult.err).toBeUndefined();
|
||||
|
||||
expect((await store.get("ns", "key1")).val).toBeUndefined();
|
||||
expect((await store.get("ns", "key2")).val).toBeUndefined();
|
||||
expect((await store.get("ns", "key3")).val).not.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("namespace isolation", () => {
|
||||
it("should isolate keys by namespace", async () => {
|
||||
const entry1 = createEntry("value1", Date.now() + 60000, Date.now() + 120000);
|
||||
const entry2 = createEntry("value2", Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
await store.set("ns1", "key", entry1);
|
||||
await store.set("ns2", "key", entry2);
|
||||
|
||||
const result1 = await store.get("ns1", "key");
|
||||
const result2 = await store.get("ns2", "key");
|
||||
|
||||
expect(result1.val?.value).toBe("value1");
|
||||
expect(result2.val?.value).toBe("value2");
|
||||
});
|
||||
});
|
||||
|
||||
describe("TTL expiration", () => {
|
||||
it("should return undefined for expired entries (past staleUntil)", async () => {
|
||||
const entry = createEntry("value", Date.now() - 2000, Date.now() - 1000); // Already expired
|
||||
|
||||
await store.set("ns", "expired-key", entry);
|
||||
|
||||
const result = await store.get("ns", "expired-key");
|
||||
expect(result.val).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should return entry that is stale but not expired", async () => {
|
||||
const now = Date.now();
|
||||
// Fresh until 1 second ago, stale until 1 hour from now
|
||||
const entry = createEntry("value", now - 1000, now + 3600000);
|
||||
|
||||
await store.set("ns", "stale-key", entry);
|
||||
|
||||
const result = await store.get("ns", "stale-key");
|
||||
expect(result.val).not.toBeUndefined();
|
||||
expect(result.val?.value).toBe("value");
|
||||
});
|
||||
|
||||
it("should delete expired entry on get", async () => {
|
||||
const entry = createEntry("value", Date.now() - 2000, Date.now() - 1000);
|
||||
await store.set("ns", "key", entry);
|
||||
|
||||
// First get should return undefined and delete
|
||||
await store.get("ns", "key");
|
||||
|
||||
// Size should reflect deletion
|
||||
expect(store.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("LRU eviction", () => {
|
||||
it("should evict least recently used items when at capacity", async () => {
|
||||
const entry = (val: string) => createEntry(val, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
// Fill the cache (max: 5)
|
||||
await store.set("ns", "key1", entry("value1"));
|
||||
await store.set("ns", "key2", entry("value2"));
|
||||
await store.set("ns", "key3", entry("value3"));
|
||||
await store.set("ns", "key4", entry("value4"));
|
||||
await store.set("ns", "key5", entry("value5"));
|
||||
|
||||
expect(store.size).toBe(5);
|
||||
|
||||
// Add one more - should evict key1 (least recently used)
|
||||
await store.set("ns", "key6", entry("value6"));
|
||||
|
||||
expect(store.size).toBe(5);
|
||||
expect((await store.get("ns", "key1")).val).toBeUndefined(); // Evicted
|
||||
expect((await store.get("ns", "key6")).val?.value).toBe("value6"); // Present
|
||||
});
|
||||
|
||||
it("should update LRU order on get", async () => {
|
||||
const entry = (val: string) => createEntry(val, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
// Fill the cache
|
||||
await store.set("ns", "key1", entry("value1"));
|
||||
await store.set("ns", "key2", entry("value2"));
|
||||
await store.set("ns", "key3", entry("value3"));
|
||||
await store.set("ns", "key4", entry("value4"));
|
||||
await store.set("ns", "key5", entry("value5"));
|
||||
|
||||
// Access key1 to make it recently used
|
||||
await store.get("ns", "key1");
|
||||
|
||||
// Add new item - should evict key2 (now least recently used)
|
||||
await store.set("ns", "key6", entry("value6"));
|
||||
|
||||
expect((await store.get("ns", "key1")).val?.value).toBe("value1"); // Still present
|
||||
expect((await store.get("ns", "key2")).val).toBeUndefined(); // Evicted
|
||||
});
|
||||
|
||||
it("should update LRU order on set (update existing)", async () => {
|
||||
const entry = (val: string) => createEntry(val, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
// Fill the cache
|
||||
await store.set("ns", "key1", entry("value1"));
|
||||
await store.set("ns", "key2", entry("value2"));
|
||||
await store.set("ns", "key3", entry("value3"));
|
||||
await store.set("ns", "key4", entry("value4"));
|
||||
await store.set("ns", "key5", entry("value5"));
|
||||
|
||||
// Update key1 to make it recently used
|
||||
await store.set("ns", "key1", entry("updated-value1"));
|
||||
|
||||
// Add new item - should evict key2 (now least recently used)
|
||||
await store.set("ns", "key6", entry("value6"));
|
||||
|
||||
expect((await store.get("ns", "key1")).val?.value).toBe("updated-value1");
|
||||
expect((await store.get("ns", "key2")).val).toBeUndefined(); // Evicted
|
||||
});
|
||||
});
|
||||
|
||||
describe("hard limit enforcement", () => {
|
||||
it("should never exceed max size regardless of write rate", async () => {
|
||||
const smallStore = new LRUMemoryStore<string, number>({ max: 10 });
|
||||
const entry = (val: number) => createEntry(val, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
// Write 1000 items rapidly
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
await smallStore.set("ns", `key${i}`, entry(i));
|
||||
// Verify size never exceeds max
|
||||
expect(smallStore.size).toBeLessThanOrEqual(10);
|
||||
}
|
||||
|
||||
expect(smallStore.size).toBe(10);
|
||||
});
|
||||
|
||||
it("should maintain most recent items when at capacity", async () => {
|
||||
const smallStore = new LRUMemoryStore<string, number>({ max: 3 });
|
||||
const entry = (val: number) => createEntry(val, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
// Write items sequentially
|
||||
await smallStore.set("ns", "key1", entry(1));
|
||||
await smallStore.set("ns", "key2", entry(2));
|
||||
await smallStore.set("ns", "key3", entry(3));
|
||||
await smallStore.set("ns", "key4", entry(4));
|
||||
await smallStore.set("ns", "key5", entry(5));
|
||||
|
||||
// Only the 3 most recent should remain
|
||||
expect((await smallStore.get("ns", "key1")).val).toBeUndefined();
|
||||
expect((await smallStore.get("ns", "key2")).val).toBeUndefined();
|
||||
expect((await smallStore.get("ns", "key3")).val?.value).toBe(3);
|
||||
expect((await smallStore.get("ns", "key4")).val?.value).toBe(4);
|
||||
expect((await smallStore.get("ns", "key5")).val?.value).toBe(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("utility methods", () => {
|
||||
it("should report correct size", async () => {
|
||||
const entry = createEntry("value", Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
expect(store.size).toBe(0);
|
||||
|
||||
await store.set("ns", "key1", entry);
|
||||
expect(store.size).toBe(1);
|
||||
|
||||
await store.set("ns", "key2", entry);
|
||||
expect(store.size).toBe(2);
|
||||
|
||||
await store.remove("ns", "key1");
|
||||
expect(store.size).toBe(1);
|
||||
});
|
||||
|
||||
it("should clear all items", async () => {
|
||||
const entry = createEntry("value", Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
await store.set("ns1", "key1", entry);
|
||||
await store.set("ns2", "key2", entry);
|
||||
await store.set("ns3", "key3", entry);
|
||||
|
||||
expect(store.size).toBe(3);
|
||||
|
||||
store.clear();
|
||||
|
||||
expect(store.size).toBe(0);
|
||||
expect((await store.get("ns1", "key1")).val).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should use custom name", () => {
|
||||
const customStore = new LRUMemoryStore({ max: 10, name: "custom-name" });
|
||||
expect(customStore.name).toBe("custom-name");
|
||||
});
|
||||
|
||||
it("should use default name when not provided", () => {
|
||||
const defaultStore = new LRUMemoryStore({ max: 10 });
|
||||
expect(defaultStore.name).toBe("lru-memory");
|
||||
});
|
||||
});
|
||||
|
||||
describe("createLRUMemoryStore helper", () => {
|
||||
it("should create a store with specified max size", async () => {
|
||||
const helperStore = createLRUMemoryStore(3);
|
||||
const entry = (val: number) => createEntry(val, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
await helperStore.set("ns", "key1", entry(1));
|
||||
await helperStore.set("ns", "key2", entry(2));
|
||||
await helperStore.set("ns", "key3", entry(3));
|
||||
await helperStore.set("ns", "key4", entry(4));
|
||||
|
||||
expect(helperStore.size).toBe(3);
|
||||
expect((await helperStore.get("ns", "key1")).val).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should accept custom name", () => {
|
||||
const namedStore = createLRUMemoryStore(10, "my-cache");
|
||||
expect(namedStore.name).toBe("my-cache");
|
||||
});
|
||||
});
|
||||
|
||||
describe("complex value types", () => {
|
||||
it("should handle object values", async () => {
|
||||
const objectStore = new LRUMemoryStore<string, { id: number; data: string[] }>({ max: 5 });
|
||||
const complexValue = { id: 123, data: ["a", "b", "c"] };
|
||||
const entry = createEntry(complexValue, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
await objectStore.set("ns", "obj-key", entry);
|
||||
|
||||
const result = await objectStore.get("ns", "obj-key");
|
||||
expect(result.val?.value).toEqual(complexValue);
|
||||
});
|
||||
|
||||
it("should handle null and undefined values", async () => {
|
||||
const nullStore = new LRUMemoryStore<string, null | undefined>({ max: 5 });
|
||||
|
||||
const nullEntry = createEntry(null, Date.now() + 60000, Date.now() + 120000);
|
||||
const undefinedEntry = createEntry(undefined, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
await nullStore.set("ns", "null-key", nullEntry);
|
||||
await nullStore.set("ns", "undefined-key", undefinedEntry);
|
||||
|
||||
expect((await nullStore.get("ns", "null-key")).val?.value).toBeNull();
|
||||
expect((await nullStore.get("ns", "undefined-key")).val?.value).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("concurrent operations", () => {
|
||||
it("should handle concurrent writes safely", async () => {
|
||||
const concurrentStore = new LRUMemoryStore<string, number>({ max: 100 });
|
||||
const entry = (val: number) => createEntry(val, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
// Simulate concurrent writes
|
||||
const writes = Array.from({ length: 50 }, (_, i) =>
|
||||
concurrentStore.set("ns", `key${i}`, entry(i))
|
||||
);
|
||||
|
||||
await Promise.all(writes);
|
||||
|
||||
expect(concurrentStore.size).toBe(50);
|
||||
});
|
||||
|
||||
it("should handle concurrent reads and writes", async () => {
|
||||
const concurrentStore = new LRUMemoryStore<string, number>({ max: 100 });
|
||||
const entry = (val: number) => createEntry(val, Date.now() + 60000, Date.now() + 120000);
|
||||
|
||||
// Pre-populate
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await concurrentStore.set("ns", `key${i}`, entry(i));
|
||||
}
|
||||
|
||||
// Mix of reads and writes
|
||||
const operations = [
|
||||
...Array.from({ length: 25 }, (_, i) => concurrentStore.get("ns", `key${i}`)),
|
||||
...Array.from({ length: 25 }, (_, i) =>
|
||||
concurrentStore.set("ns", `new-key${i}`, entry(i + 100))
|
||||
),
|
||||
];
|
||||
|
||||
await Promise.all(operations);
|
||||
|
||||
// Should not exceed max
|
||||
expect(concurrentStore.size).toBeLessThanOrEqual(100);
|
||||
});
|
||||
});
|
||||
});
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
import { LRUCache } from "lru-cache";
|
||||
import { CacheError } from "@unkey/cache";
|
||||
import type { Store, Entry } from "@unkey/cache/stores";
|
||||
import { Ok, Err, type Result } from "@unkey/error";
|
||||
|
||||
export type LRUMemoryStoreConfig = {
|
||||
/**
|
||||
* Maximum number of items to store in the cache.
|
||||
* This is a hard limit - the cache will never exceed this size.
|
||||
*/
|
||||
max: number;
|
||||
|
||||
/**
|
||||
* Name for metrics/tracing.
|
||||
* @default "lru-memory"
|
||||
*/
|
||||
name?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A memory store implementation using lru-cache.
|
||||
*
|
||||
* This provides O(1) get/set/delete operations and automatic LRU eviction
|
||||
* without blocking the event loop (unlike @unkey/cache's MemoryStore which
|
||||
* uses O(n) synchronous iteration for eviction).
|
||||
*
|
||||
* TTL is checked lazily on get() - expired items are not proactively removed
|
||||
* but will be evicted by LRU when the cache is full.
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export class LRUMemoryStore<TNamespace extends string, TValue = any> implements Store<
|
||||
TNamespace,
|
||||
TValue
|
||||
> {
|
||||
readonly name: string;
|
||||
private readonly cache: LRUCache<string, Entry<TValue>>;
|
||||
|
||||
constructor(config: LRUMemoryStoreConfig) {
|
||||
this.name = config.name ?? "lru-memory";
|
||||
this.cache = new LRUCache<string, Entry<TValue>>({
|
||||
max: config.max,
|
||||
// Don't use ttlAutopurge - it creates a setTimeout per item which
|
||||
// doesn't scale well at high throughput (thousands of items/second).
|
||||
// Instead, we check TTL lazily on get().
|
||||
ttlAutopurge: false,
|
||||
// Allow returning stale values - the cache layer handles SWR semantics
|
||||
allowStale: true,
|
||||
// Use the staleUntil timestamp for TTL calculation
|
||||
ttl: 1, // Placeholder, we set per-item TTL in set()
|
||||
});
|
||||
}
|
||||
|
||||
private buildCacheKey(namespace: TNamespace, key: string): string {
|
||||
return `${namespace}::${key}`;
|
||||
}
|
||||
|
||||
async get(
|
||||
namespace: TNamespace,
|
||||
key: string
|
||||
): Promise<Result<Entry<TValue> | undefined, CacheError>> {
|
||||
try {
|
||||
const cacheKey = this.buildCacheKey(namespace, key);
|
||||
const entry = this.cache.get(cacheKey);
|
||||
|
||||
if (!entry) {
|
||||
return Ok(undefined);
|
||||
}
|
||||
|
||||
// Check if entry is expired (past staleUntil)
|
||||
// The cache layer will handle fresh vs stale semantics
|
||||
if (entry.staleUntil <= Date.now()) {
|
||||
// Remove expired entry
|
||||
this.cache.delete(cacheKey);
|
||||
return Ok(undefined);
|
||||
}
|
||||
|
||||
return Ok(entry);
|
||||
} catch (err) {
|
||||
return Err(
|
||||
new CacheError({
|
||||
tier: this.name,
|
||||
key,
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async set(
|
||||
namespace: TNamespace,
|
||||
key: string,
|
||||
entry: Entry<TValue>
|
||||
): Promise<Result<void, CacheError>> {
|
||||
try {
|
||||
const cacheKey = this.buildCacheKey(namespace, key);
|
||||
|
||||
// Calculate TTL from staleUntil timestamp
|
||||
const ttl = Math.max(0, entry.staleUntil - Date.now());
|
||||
|
||||
this.cache.set(cacheKey, entry, { ttl });
|
||||
|
||||
return Ok(undefined as void);
|
||||
} catch (err) {
|
||||
return Err(
|
||||
new CacheError({
|
||||
tier: this.name,
|
||||
key,
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async remove(namespace: TNamespace, keys: string | string[]): Promise<Result<void, CacheError>> {
|
||||
try {
|
||||
const keyArray = Array.isArray(keys) ? keys : [keys];
|
||||
|
||||
for (const key of keyArray) {
|
||||
const cacheKey = this.buildCacheKey(namespace, key);
|
||||
this.cache.delete(cacheKey);
|
||||
}
|
||||
|
||||
return Ok(undefined as void);
|
||||
} catch (err) {
|
||||
return Err(
|
||||
new CacheError({
|
||||
tier: this.name,
|
||||
key: Array.isArray(keys) ? keys.join(",") : keys,
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current number of items in the cache.
|
||||
*/
|
||||
get size(): number {
|
||||
return this.cache.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all items from the cache.
|
||||
*/
|
||||
clear(): void {
|
||||
this.cache.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an LRU memory store with the specified maximum size.
|
||||
*
|
||||
* This is a drop-in replacement for createMemoryStore() that uses lru-cache
|
||||
* instead of @unkey/cache's MemoryStore, providing:
|
||||
* - O(1) operations (vs O(n) eviction in MemoryStore)
|
||||
* - No event loop blocking
|
||||
* - Strict memory bounds (hard max vs soft cap)
|
||||
*
|
||||
* @param maxItems Maximum number of items to store
|
||||
* @param name Optional name for metrics/tracing (default: "lru-memory")
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export function createLRUMemoryStore(maxItems: number, name?: string): LRUMemoryStore<string, any> {
|
||||
return new LRUMemoryStore({ max: maxItems, name });
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
import { MemoryStore } from "@unkey/cache/stores";
|
||||
|
||||
export type { MemoryStore };
|
||||
|
||||
export function createMemoryStore(maxItems: number, frequency: number = 0.01) {
|
||||
return new MemoryStore({
|
||||
persistentMap: new Map(),
|
||||
unstableEvictOnSet: {
|
||||
frequency,
|
||||
maxItems,
|
||||
},
|
||||
});
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
import { CacheError } from "@unkey/cache";
|
||||
import type { Entry, Store } from "@unkey/cache/stores";
|
||||
import { Err, Ok, type Result } from "@unkey/error";
|
||||
import type { Redis, RedisOptions } from "@internal/redis";
|
||||
import { createRedisClient } from "@internal/redis";
|
||||
|
||||
export type RedisCacheStoreConfig = {
|
||||
connection: RedisOptions;
|
||||
name?: string;
|
||||
useModernCacheKeyBuilder?: boolean;
|
||||
};
|
||||
|
||||
export class RedisCacheStore<TNamespace extends string, TValue = any> implements Store<
|
||||
TNamespace,
|
||||
TValue
|
||||
> {
|
||||
public readonly name = "redis";
|
||||
private readonly redis: Redis;
|
||||
|
||||
constructor(private readonly config: RedisCacheStoreConfig) {
|
||||
this.redis = createRedisClient({
|
||||
...config.connection,
|
||||
name: config.name ?? "trigger:cacheStore",
|
||||
});
|
||||
}
|
||||
|
||||
private buildCacheKey(namespace: TNamespace, key: string): string {
|
||||
if (this.config.useModernCacheKeyBuilder) {
|
||||
return [namespace, key].join(":");
|
||||
}
|
||||
|
||||
return [namespace, key].join("::");
|
||||
}
|
||||
|
||||
public async get(
|
||||
namespace: TNamespace,
|
||||
key: string
|
||||
): Promise<Result<Entry<TValue> | undefined, CacheError>> {
|
||||
let raw: string | null;
|
||||
try {
|
||||
raw = await this.redis.get(this.buildCacheKey(namespace, key));
|
||||
} catch (err) {
|
||||
return Err(
|
||||
new CacheError({
|
||||
tier: this.name,
|
||||
key,
|
||||
message: (err as Error).message,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
if (!raw) {
|
||||
return Promise.resolve(Ok(undefined));
|
||||
}
|
||||
|
||||
try {
|
||||
const superjson = await import("superjson");
|
||||
const entry = superjson.parse(raw) as Entry<TValue>;
|
||||
return Ok(entry);
|
||||
} catch (err) {
|
||||
return Err(
|
||||
new CacheError({
|
||||
tier: this.name,
|
||||
key,
|
||||
message: (err as Error).message,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public async set(
|
||||
namespace: TNamespace,
|
||||
key: string,
|
||||
entry: Entry<TValue>
|
||||
): Promise<Result<void, CacheError>> {
|
||||
const cacheKey = this.buildCacheKey(namespace, key);
|
||||
try {
|
||||
const superjson = await import("superjson");
|
||||
await this.redis.set(cacheKey, superjson.stringify(entry), "PXAT", entry.staleUntil);
|
||||
return Ok();
|
||||
} catch (err) {
|
||||
return Err(
|
||||
new CacheError({
|
||||
tier: this.name,
|
||||
key,
|
||||
message: (err as Error).message,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public async remove(namespace: TNamespace, key: string): Promise<Result<void, CacheError>> {
|
||||
try {
|
||||
const cacheKey = this.buildCacheKey(namespace, key);
|
||||
await this.redis.del(cacheKey);
|
||||
return Promise.resolve(Ok());
|
||||
} catch (err) {
|
||||
return Err(
|
||||
new CacheError({
|
||||
tier: this.name,
|
||||
key,
|
||||
message: (err as Error).message,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2019",
|
||||
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"types": ["vitest/globals"],
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"strict": true,
|
||||
"paths": {
|
||||
"@trigger.dev/core": ["../../packages/core/src/index"],
|
||||
"@trigger.dev/core/*": ["../../packages/core/src/*"]
|
||||
}
|
||||
},
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ["**/*.test.ts"],
|
||||
globals: true,
|
||||
isolate: true,
|
||||
testTimeout: 10_000,
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
# ClickHouse Package
|
||||
|
||||
`@internal/clickhouse` - ClickHouse client for analytics and observability data.
|
||||
|
||||
## Migrations
|
||||
|
||||
Goose-format SQL migrations live in `schema/`. Two rules below are load-bearing — both can block a deploy.
|
||||
|
||||
### Rule 1: number to `max + 1`, never slot in
|
||||
|
||||
Goose runs in strict mode in the deploy pipeline. If a migration file numbered *below* the version currently recorded in `goose_db_version` ever shows up, goose refuses to apply it and the deploy fails:
|
||||
|
||||
```
|
||||
goose run: error: found 1 missing migrations before current version 30:
|
||||
version 29: 029_add_task_kind_to_task_runs_v2.sql
|
||||
```
|
||||
|
||||
When adding a migration:
|
||||
|
||||
1. Look at `schema/` and take the largest existing number, call it `N`.
|
||||
2. Name your file `0(N+1)_descriptive_name.sql`.
|
||||
3. If you've been on a branch while main added migrations, **rebase and renumber** before opening the PR — a file numbered below the new max will block the next deploy after your PR merges.
|
||||
|
||||
### Rule 2: DDL must be idempotent
|
||||
|
||||
Migrations can be applied out of order in some environments (`goose up --allow-missing` for local recovery, manual fixups, etc.) and may be retried. Always use idempotent forms so a re-apply is a no-op:
|
||||
|
||||
```sql
|
||||
-- +goose Up
|
||||
ALTER TABLE trigger_dev.your_table
|
||||
ADD COLUMN IF NOT EXISTS new_column String DEFAULT '';
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.your_table
|
||||
DROP COLUMN IF EXISTS new_column;
|
||||
```
|
||||
|
||||
Equivalent forms for other DDL:
|
||||
|
||||
- `CREATE TABLE IF NOT EXISTS …`
|
||||
- `DROP TABLE IF EXISTS …`
|
||||
- `ADD INDEX IF NOT EXISTS …` / `DROP INDEX IF EXISTS …`
|
||||
- `CREATE MATERIALIZED VIEW IF NOT EXISTS …` / `DROP VIEW IF EXISTS …`
|
||||
|
||||
ClickHouse supports `IF [NOT] EXISTS` on all of the above. Older migrations in this directory predate the rule and are not idempotent — leave them as-is unless you're explicitly hardening one.
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
- `raw_` prefix for input tables (where data lands first)
|
||||
- `_v1`, `_v2` suffixes for table versioning
|
||||
- `_mv_v1` suffix for materialized views
|
||||
- `_per_day`, `_per_month` for aggregation tables
|
||||
|
||||
See `README.md` in this directory for full naming convention documentation.
|
||||
|
||||
## Purpose
|
||||
|
||||
Stores time-series data for task run analytics, event streams, and performance metrics. Separate from PostgreSQL to handle high-volume writes from task execution.
|
||||
@@ -0,0 +1,15 @@
|
||||
FROM golang:1.26@sha256:68cb6d68bed024785b69195b89af7ac7a444f27791435f98647edff595aa0479
|
||||
|
||||
|
||||
RUN go install github.com/pressly/goose/v3/cmd/goose@v3.27.1
|
||||
|
||||
|
||||
COPY ./schema ./schema
|
||||
|
||||
ENV GOOSE_DRIVER=clickhouse
|
||||
ENV GOOSE_DBSTRING="tcp://default:password@clickhouse:9000"
|
||||
ENV GOOSE_MIGRATION_DIR=./schema
|
||||
|
||||
# Run migrations as non-root (dev-only migration helper; goose needs no root).
|
||||
USER nobody
|
||||
CMD ["goose", "up"]
|
||||
@@ -0,0 +1,64 @@
|
||||
# ClickHouse Table Naming Conventions
|
||||
|
||||
The following document is heavily inspired by the [Unkey](https://unkey.dev) ClickHouse naming conventions.
|
||||
|
||||
This document outlines the naming conventions for tables and materialized views in our ClickHouse setup. Adhering to these conventions ensures consistency, clarity, and ease of management across our data infrastructure.
|
||||
|
||||
## General Rules
|
||||
|
||||
1. Use lowercase letters and separate words with underscores.
|
||||
2. Avoid ClickHouse reserved words and special characters in names.
|
||||
3. Be descriptive but concise.
|
||||
|
||||
## Table Naming Convention
|
||||
|
||||
Format: `[prefix]_[domain]_[description]_[version]`
|
||||
|
||||
### Prefixes
|
||||
|
||||
- `raw_`: Input data tables
|
||||
- `tmp_{yourname}_`: Temporary tables for experiments, add your name, so it's easy to identify ownership.
|
||||
|
||||
### Versioning
|
||||
|
||||
- Version numbers: `_v1`, `_v2`, etc.
|
||||
|
||||
### Aggregation Suffixes
|
||||
|
||||
For aggregated or summary tables, use suffixes like:
|
||||
|
||||
- `_per_day`
|
||||
- `_per_month`
|
||||
- `_summary`
|
||||
|
||||
## Materialized View Naming Convention
|
||||
|
||||
Format: `[description]_[aggregation]_mv_[version]`
|
||||
|
||||
- Always suffix with `mv_[version]`
|
||||
- Include a description of the view's purpose
|
||||
- Add aggregation level if applicable
|
||||
|
||||
## Examples
|
||||
|
||||
1. Raw Data Table:
|
||||
`raw_sales_transactions_v1`
|
||||
|
||||
2. Materialized View:
|
||||
`active_users_per_day_mv_v2`
|
||||
|
||||
3. Temporary Table:
|
||||
`tmp_eric_user_analysis_v1`
|
||||
|
||||
4. Aggregated Table:
|
||||
`sales_summary_per_hour_mv_v1`
|
||||
|
||||
## Consistency Across Related Objects
|
||||
|
||||
Maintain consistent naming across related tables, views, and other objects:
|
||||
|
||||
- `raw_user_activity_v1`
|
||||
- `user_activity_per_day_v1`
|
||||
- `user_activity_per_day_mv_v1`
|
||||
|
||||
By following these conventions, we ensure a clear, consistent, and scalable naming structure for our ClickHouse setup.
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"name": "@internal/clickhouse",
|
||||
"private": true,
|
||||
"version": "0.0.2",
|
||||
"main": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@clickhouse/client": "^1.12.1",
|
||||
"@internal/tracing": "workspace:*",
|
||||
"@internal/tsql": "workspace:*",
|
||||
"@trigger.dev/core": "workspace:*",
|
||||
"zod": "3.25.76",
|
||||
"zod-error": "1.5.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@internal/testcontainers": "workspace:*",
|
||||
"rimraf": "6.0.1"
|
||||
},
|
||||
"scripts": {
|
||||
"clean": "rimraf dist",
|
||||
"typecheck": "tsc --noEmit -p tsconfig.build.json",
|
||||
"build": "pnpm run clean && tsc -p tsconfig.build.json",
|
||||
"dev": "tsc --watch -p tsconfig.build.json",
|
||||
"db:migrate": "node ../../scripts/docker.mjs -f docker/docker-compose.yml up clickhouse_migrator --build",
|
||||
"db:migrate:down": "GOOSE_COMMAND=down pnpm run db:migrate",
|
||||
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
|
||||
"test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled"
|
||||
},
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"@triggerdotdev/source": "./src/index.ts",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"default": "./dist/src/index.js"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
-- +goose up
|
||||
|
||||
CREATE DATABASE trigger_dev;
|
||||
|
||||
-- +goose down
|
||||
DROP DATABASE trigger_dev;
|
||||
@@ -0,0 +1,11 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS trigger_dev.smoke_test (
|
||||
id UUID DEFAULT generateUUIDv4(),
|
||||
timestamp DateTime64(3) DEFAULT now64(3),
|
||||
message String,
|
||||
number UInt32
|
||||
) ENGINE = MergeTree()
|
||||
ORDER BY (timestamp, id);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.smoke_test;
|
||||
@@ -0,0 +1,100 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE trigger_dev.task_runs_v1
|
||||
(
|
||||
/* ─── ids & hierarchy ─────────────────────────────────────── */
|
||||
environment_id String,
|
||||
organization_id String,
|
||||
project_id String,
|
||||
run_id String,
|
||||
|
||||
environment_type LowCardinality(String),
|
||||
friendly_id String,
|
||||
attempt UInt8 DEFAULT 1,
|
||||
|
||||
/* ─── enums / status ──────────────────────────────────────── */
|
||||
engine LowCardinality(String),
|
||||
status LowCardinality(String),
|
||||
|
||||
/* ─── queue / concurrency / schedule ─────────────────────── */
|
||||
task_identifier String,
|
||||
queue String,
|
||||
|
||||
schedule_id String,
|
||||
batch_id String,
|
||||
|
||||
/* ─── related runs ─────────────────────────────────────────────── */
|
||||
root_run_id String,
|
||||
parent_run_id String,
|
||||
depth UInt8 DEFAULT 0,
|
||||
|
||||
/* ─── telemetry ─────────────────────────────────────────────── */
|
||||
span_id String,
|
||||
trace_id String,
|
||||
idempotency_key String,
|
||||
|
||||
/* ─── timing ─────────────────────────────────────────────── */
|
||||
created_at DateTime64(3),
|
||||
updated_at DateTime64(3),
|
||||
started_at Nullable(DateTime64(3)),
|
||||
executed_at Nullable(DateTime64(3)),
|
||||
completed_at Nullable(DateTime64(3)),
|
||||
delay_until Nullable(DateTime64(3)),
|
||||
queued_at Nullable(DateTime64(3)),
|
||||
expired_at Nullable(DateTime64(3)),
|
||||
expiration_ttl String,
|
||||
|
||||
/* ─── cost / usage ───────────────────────────────────────── */
|
||||
usage_duration_ms UInt32 DEFAULT 0,
|
||||
cost_in_cents Float64 DEFAULT 0,
|
||||
base_cost_in_cents Float64 DEFAULT 0,
|
||||
|
||||
/* ─── payload & context ──────────────────────────────────── */
|
||||
output JSON(max_dynamic_paths = 1024),
|
||||
error JSON(max_dynamic_paths = 64),
|
||||
|
||||
/* ─── tagging / versions ─────────────────────────────────── */
|
||||
tags Array(String) CODEC(ZSTD(1)),
|
||||
task_version String CODEC(LZ4),
|
||||
sdk_version String CODEC(LZ4),
|
||||
cli_version String CODEC(LZ4),
|
||||
machine_preset LowCardinality(String) CODEC(LZ4),
|
||||
|
||||
is_test UInt8 DEFAULT 0,
|
||||
|
||||
/* ─── commit lsn ─────────────────────────────────────────────── */
|
||||
_version UInt64,
|
||||
_is_deleted UInt8 DEFAULT 0
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(_version, _is_deleted)
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (toDate(created_at), environment_id, task_identifier, created_at, run_id)
|
||||
SETTINGS enable_json_type = 1;
|
||||
|
||||
/* Fast tag filtering */
|
||||
ALTER TABLE trigger_dev.task_runs_v1
|
||||
ADD INDEX idx_tags tags TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4;
|
||||
|
||||
CREATE TABLE trigger_dev.raw_task_runs_payload_v1
|
||||
(
|
||||
run_id String,
|
||||
created_at DateTime64(3),
|
||||
payload JSON(max_dynamic_paths = 1024)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (run_id)
|
||||
SETTINGS enable_json_type = 1;
|
||||
|
||||
CREATE VIEW trigger_dev.tmp_eric_task_runs_full_v1 AS
|
||||
SELECT
|
||||
s.*,
|
||||
p.payload as payload
|
||||
FROM trigger_dev.task_runs_v1 AS s FINAL
|
||||
LEFT JOIN trigger_dev.raw_task_runs_payload_v1 AS p ON s.run_id = p.run_id
|
||||
SETTINGS enable_json_type = 1;
|
||||
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.task_runs_v1;
|
||||
DROP TABLE IF EXISTS trigger_dev.raw_task_runs_payload_v1;
|
||||
DROP VIEW IF EXISTS trigger_dev.tmp_eric_task_runs_full_v1;
|
||||
@@ -0,0 +1,94 @@
|
||||
-- +goose Up
|
||||
|
||||
/*
|
||||
This is the second version of the task runs table.
|
||||
The main change is we've added organization_id and project_id to the sort key, and removed the toDate(created_at) and task_identifier columns from the sort key.
|
||||
We will add a skip index for the task_identifier column in a future migration.
|
||||
*/
|
||||
CREATE TABLE trigger_dev.task_runs_v2
|
||||
(
|
||||
/* ─── ids & hierarchy ─────────────────────────────────────── */
|
||||
environment_id String,
|
||||
organization_id String,
|
||||
project_id String,
|
||||
run_id String,
|
||||
|
||||
environment_type LowCardinality(String),
|
||||
friendly_id String,
|
||||
attempt UInt8 DEFAULT 1,
|
||||
|
||||
/* ─── enums / status ──────────────────────────────────────── */
|
||||
engine LowCardinality(String),
|
||||
status LowCardinality(String),
|
||||
|
||||
/* ─── queue / concurrency / schedule ─────────────────────── */
|
||||
task_identifier String,
|
||||
queue String,
|
||||
|
||||
schedule_id String,
|
||||
batch_id String,
|
||||
|
||||
/* ─── related runs ─────────────────────────────────────────────── */
|
||||
root_run_id String,
|
||||
parent_run_id String,
|
||||
depth UInt8 DEFAULT 0,
|
||||
|
||||
/* ─── telemetry ─────────────────────────────────────────────── */
|
||||
span_id String,
|
||||
trace_id String,
|
||||
idempotency_key String,
|
||||
|
||||
/* ─── timing ─────────────────────────────────────────────── */
|
||||
created_at DateTime64(3),
|
||||
updated_at DateTime64(3),
|
||||
started_at Nullable(DateTime64(3)),
|
||||
executed_at Nullable(DateTime64(3)),
|
||||
completed_at Nullable(DateTime64(3)),
|
||||
delay_until Nullable(DateTime64(3)),
|
||||
queued_at Nullable(DateTime64(3)),
|
||||
expired_at Nullable(DateTime64(3)),
|
||||
expiration_ttl String,
|
||||
|
||||
/* ─── cost / usage ───────────────────────────────────────── */
|
||||
usage_duration_ms UInt32 DEFAULT 0,
|
||||
cost_in_cents Float64 DEFAULT 0,
|
||||
base_cost_in_cents Float64 DEFAULT 0,
|
||||
|
||||
/* ─── payload & context ──────────────────────────────────── */
|
||||
output JSON(max_dynamic_paths = 1024),
|
||||
error JSON(max_dynamic_paths = 64),
|
||||
|
||||
/* ─── tagging / versions ─────────────────────────────────── */
|
||||
tags Array(String) CODEC(ZSTD(1)),
|
||||
task_version String CODEC(LZ4),
|
||||
sdk_version String CODEC(LZ4),
|
||||
cli_version String CODEC(LZ4),
|
||||
machine_preset LowCardinality(String) CODEC(LZ4),
|
||||
|
||||
is_test UInt8 DEFAULT 0,
|
||||
|
||||
/* ─── commit lsn ─────────────────────────────────────────────── */
|
||||
_version UInt64,
|
||||
_is_deleted UInt8 DEFAULT 0
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(_version, _is_deleted)
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (organization_id, project_id, environment_id, created_at, run_id)
|
||||
SETTINGS enable_json_type = 1;
|
||||
|
||||
/* Fast tag filtering */
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD INDEX idx_tags tags TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4;
|
||||
|
||||
CREATE VIEW trigger_dev.tmp_eric_task_runs_full_v2 AS
|
||||
SELECT
|
||||
s.*,
|
||||
p.payload as payload
|
||||
FROM trigger_dev.task_runs_v2 AS s FINAL
|
||||
LEFT JOIN trigger_dev.raw_task_runs_payload_v1 AS p ON s.run_id = p.run_id
|
||||
SETTINGS enable_json_type = 1;
|
||||
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.task_runs_v2;
|
||||
DROP VIEW IF EXISTS trigger_dev.tmp_eric_task_runs_full_v2
|
||||
@@ -0,0 +1,12 @@
|
||||
-- +goose Up
|
||||
/*
|
||||
Add concurrency_key and bulk_action_group_ids columns with defaults.
|
||||
*/
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN concurrency_key String DEFAULT '',
|
||||
ADD COLUMN bulk_action_group_ids Array(String) DEFAULT [];
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN concurrency_key,
|
||||
DROP COLUMN bulk_action_group_ids;
|
||||
@@ -0,0 +1,10 @@
|
||||
-- +goose Up
|
||||
/*
|
||||
Add worker_queue column.
|
||||
*/
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN worker_queue String DEFAULT '';
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN worker_queue;
|
||||
@@ -0,0 +1,52 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS trigger_dev.task_events_v1
|
||||
(
|
||||
-- This the main "tenant" ID
|
||||
environment_id String,
|
||||
-- The organization ID here so we can do MV rollups of usage
|
||||
organization_id String,
|
||||
-- The project ID here so we can do MV rollups of usage
|
||||
project_id String,
|
||||
-- The task slug (e.g. "my-task")
|
||||
task_identifier String CODEC(ZSTD(1)),
|
||||
-- The non-friendly ID for the run
|
||||
run_id String CODEC(ZSTD(1)),
|
||||
-- nanoseconds since the epoch
|
||||
start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
trace_id String CODEC(ZSTD(1)),
|
||||
span_id String CODEC(ZSTD(1)),
|
||||
-- will be an empty string for root spans
|
||||
parent_span_id String CODEC(ZSTD(1)),
|
||||
-- Log body, event name, or span name
|
||||
message String CODEC(ZSTD(1)),
|
||||
-- this is the new level column, can be
|
||||
-- SPAN, SPAN_EVENT, DEBUG_EVENT, LOG_DEBUG, LOG_LOG, LOG_SUCCESS, LOG_INFO, LOG_WARN, LOG_ERROR, ANCESTOR_OVERRIDE
|
||||
kind LowCardinality(String) CODEC(ZSTD(1)),
|
||||
-- isError, isPartial, isCancelled will now be in this status column
|
||||
-- OK, ERROR, PARTIAL, CANCELLED
|
||||
status LowCardinality(String) CODEC(ZSTD(1)),
|
||||
-- span/log/event attributes and resource attributes
|
||||
-- includes error attributes, gen_ai attributes, and other attributes
|
||||
attributes JSON CODEC(ZSTD(1)),
|
||||
attributes_text String MATERIALIZED toJSONString(attributes),
|
||||
-- This is the metadata column, includes style for styling the event in the UI
|
||||
-- is a JSON stringified object, e.g. {"style":{"icon":"play","variant":"primary"},"error":{"message":"Error message","attributes":{"error.type":"ErrorType","error.code":"123"}}}
|
||||
metadata String CODEC(ZSTD(1)),
|
||||
-- nanoseconds since the start time, only non-zero for spans
|
||||
duration UInt64 CODEC(ZSTD(1)),
|
||||
-- The TTL for the event, will be deleted 7 days after the event expires
|
||||
expires_at DateTime64(3),
|
||||
|
||||
INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_duration duration TYPE minmax GRANULARITY 1,
|
||||
INDEX idx_attributes_text attributes_text TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(start_time)
|
||||
ORDER BY (environment_id, toUnixTimestamp(start_time), trace_id)
|
||||
TTL toDateTime(expires_at) + INTERVAL 7 DAY
|
||||
SETTINGS ttl_only_drop_parts = 1;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.task_events_v1;
|
||||
@@ -0,0 +1,55 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS trigger_dev.task_event_usage_by_minute_v1
|
||||
(
|
||||
organization_id String,
|
||||
project_id String,
|
||||
environment_id String,
|
||||
bucket_start DateTime,
|
||||
event_count UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY toYYYYMM(bucket_start)
|
||||
ORDER BY (organization_id, project_id, environment_id, bucket_start)
|
||||
TTL bucket_start + INTERVAL 8 DAY;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS trigger_dev.task_event_usage_by_hour_v1
|
||||
(
|
||||
organization_id String,
|
||||
project_id String,
|
||||
environment_id String,
|
||||
bucket_start DateTime,
|
||||
event_count UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY toYYYYMM(bucket_start)
|
||||
ORDER BY (organization_id, project_id, environment_id, bucket_start)
|
||||
TTL bucket_start + INTERVAL 400 DAY;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_usage_by_minute_v1
|
||||
TO trigger_dev.task_event_usage_by_minute_v1 AS
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
toStartOfMinute(start_time) AS bucket_start,
|
||||
count() AS event_count
|
||||
FROM trigger_dev.task_events_v1
|
||||
GROUP BY organization_id, project_id, environment_id, bucket_start;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_usage_by_hour_v1
|
||||
TO trigger_dev.task_event_usage_by_hour_v1 AS
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
toStartOfHour(bucket_start) AS bucket_start,
|
||||
sum(event_count) AS event_count
|
||||
FROM trigger_dev.task_event_usage_by_minute_v1
|
||||
GROUP BY organization_id, project_id, environment_id, bucket_start;
|
||||
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.mv_task_event_usage_by_minute_v1;
|
||||
DROP TABLE IF EXISTS trigger_dev.mv_task_event_usage_by_hour_v1;
|
||||
DROP TABLE IF EXISTS trigger_dev.task_event_usage_by_hour_v1;
|
||||
DROP TABLE IF EXISTS trigger_dev.task_event_usage_by_minute_v1;
|
||||
@@ -0,0 +1,29 @@
|
||||
-- +goose Up
|
||||
DROP TABLE IF EXISTS trigger_dev.mv_task_event_usage_by_minute_v1;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_usage_by_minute_v2
|
||||
TO trigger_dev.task_event_usage_by_minute_v1 AS
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
toStartOfMinute(start_time) AS bucket_start,
|
||||
count() AS event_count
|
||||
FROM trigger_dev.task_events_v1
|
||||
WHERE kind != 'DEBUG_EVENT' AND kind != 'ANCESTOR_OVERRIDE' AND status != 'PARTIAL'
|
||||
GROUP BY organization_id, project_id, environment_id, bucket_start;
|
||||
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.mv_task_event_usage_by_minute_v2;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_usage_by_minute_v1
|
||||
TO trigger_dev.task_event_usage_by_minute_v1 AS
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
toStartOfMinute(start_time) AS bucket_start,
|
||||
count() AS event_count
|
||||
FROM trigger_dev.task_events_v1
|
||||
GROUP BY organization_id, project_id, environment_id, bucket_start;
|
||||
@@ -0,0 +1,55 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS trigger_dev.task_events_v2
|
||||
(
|
||||
-- This the main "tenant" ID
|
||||
environment_id String,
|
||||
-- The organization ID here so we can do MV rollups of usage
|
||||
organization_id String,
|
||||
-- The project ID here so we can do MV rollups of usage
|
||||
project_id String,
|
||||
-- The task slug (e.g. "my-task")
|
||||
task_identifier String CODEC(ZSTD(1)),
|
||||
-- The non-friendly ID for the run
|
||||
run_id String CODEC(ZSTD(1)),
|
||||
-- nanoseconds since the epoch
|
||||
start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
trace_id String CODEC(ZSTD(1)),
|
||||
span_id String CODEC(ZSTD(1)),
|
||||
-- will be an empty string for root spans
|
||||
parent_span_id String CODEC(ZSTD(1)),
|
||||
-- Log body, event name, or span name
|
||||
message String CODEC(ZSTD(1)),
|
||||
-- this is the new level column, can be
|
||||
-- SPAN, SPAN_EVENT, DEBUG_EVENT, LOG_DEBUG, LOG_LOG, LOG_SUCCESS, LOG_INFO, LOG_WARN, LOG_ERROR, ANCESTOR_OVERRIDE
|
||||
kind LowCardinality(String) CODEC(ZSTD(1)),
|
||||
-- isError, isPartial, isCancelled will now be in this status column
|
||||
-- OK, ERROR, PARTIAL, CANCELLED
|
||||
status LowCardinality(String) CODEC(ZSTD(1)),
|
||||
-- span/log/event attributes and resource attributes
|
||||
-- includes error attributes, gen_ai attributes, and other attributes
|
||||
attributes JSON CODEC(ZSTD(1)),
|
||||
attributes_text String MATERIALIZED toJSONString(attributes),
|
||||
-- This is the metadata column, includes style for styling the event in the UI
|
||||
-- is a JSON stringified object, e.g. {"style":{"icon":"play","variant":"primary"},"error":{"message":"Error message","attributes":{"error.type":"ErrorType","error.code":"123"}}}
|
||||
metadata String CODEC(ZSTD(1)),
|
||||
-- nanoseconds since the start time, only non-zero for spans
|
||||
duration UInt64 CODEC(ZSTD(1)),
|
||||
-- The TTL for the event, will be deleted 7 days after the event expires
|
||||
expires_at DateTime64(3),
|
||||
-- NEW: Insert timestamp for partitioning (avoids "too many parts" errors from late-arriving events)
|
||||
inserted_at DateTime64(3) DEFAULT now64(3),
|
||||
|
||||
INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_duration duration TYPE minmax GRANULARITY 1,
|
||||
INDEX idx_attributes_text attributes_text TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(inserted_at)
|
||||
ORDER BY (environment_id, toUnixTimestamp(start_time), trace_id)
|
||||
TTL toDateTime(expires_at) + INTERVAL 7 DAY
|
||||
SETTINGS ttl_only_drop_parts = 1;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.task_events_v2;
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
-- +goose Up
|
||||
-- Create materialized views for task_events_v2 table (partitioned by inserted_at)
|
||||
-- These write to the same target tables as the v1 MVs so usage is aggregated across both tables
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_v2_usage_by_minute
|
||||
TO trigger_dev.task_event_usage_by_minute_v1 AS
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
toStartOfMinute(start_time) AS bucket_start,
|
||||
count() AS event_count
|
||||
FROM trigger_dev.task_events_v2
|
||||
WHERE kind != 'DEBUG_EVENT' AND kind != 'ANCESTOR_OVERRIDE' AND status != 'PARTIAL'
|
||||
GROUP BY organization_id, project_id, environment_id, bucket_start;
|
||||
|
||||
-- +goose Down
|
||||
DROP VIEW IF EXISTS trigger_dev.mv_task_event_v2_usage_by_minute;
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
-- +goose Up
|
||||
/*
|
||||
Add max_duration_in_seconds column.
|
||||
*/
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN max_duration_in_seconds Nullable (UInt32) DEFAULT NULL;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN max_duration_in_seconds;
|
||||
@@ -0,0 +1,16 @@
|
||||
-- +goose Up
|
||||
|
||||
-- Add columns for storing user-provided idempotency key and scope for searching
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN idempotency_key_user String DEFAULT '';
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN idempotency_key_scope String DEFAULT '';
|
||||
|
||||
-- +goose Down
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN idempotency_key_user;
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN idempotency_key_scope;
|
||||
@@ -0,0 +1,45 @@
|
||||
-- +goose Up
|
||||
-- Update the materialized columns to extract the 'data' field if it exists
|
||||
-- This avoids the {"data": ...} wrapper in the text representation
|
||||
-- Note: Direct JSON path access (output.data) returns null for nested objects,
|
||||
-- so we use JSONExtractRaw on the stringified JSON instead
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN output_text String MATERIALIZED if (
|
||||
toJSONString (output) = '{}',
|
||||
'',
|
||||
if (
|
||||
length (JSONExtractRaw (toJSONString (output), 'data')) > 0,
|
||||
JSONExtractRaw (toJSONString (output), 'data'),
|
||||
toJSONString (output)
|
||||
)
|
||||
);
|
||||
|
||||
-- For error: extract error.data if it exists
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN error_text String MATERIALIZED if (
|
||||
toJSONString (error) = '{}',
|
||||
'',
|
||||
if (
|
||||
length (JSONExtractRaw (toJSONString (error), 'data')) > 0,
|
||||
JSONExtractRaw (toJSONString (error), 'data'),
|
||||
toJSONString (error)
|
||||
)
|
||||
);
|
||||
|
||||
-- Add the indexes
|
||||
ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_output_text output_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4;
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_error_text error_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP INDEX IF EXISTS idx_output_text;
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP INDEX IF EXISTS idx_error_text;
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN IF EXISTS output_text;
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN IF EXISTS error_text;
|
||||
@@ -0,0 +1,20 @@
|
||||
-- +goose Up
|
||||
|
||||
-- Add indexes for text search on task task_events_v2 tables for message and attributes fields
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
ADD INDEX IF NOT EXISTS idx_attributes_text_search lower(attributes_text)
|
||||
TYPE ngrambf_v1(3, 32768, 2, 0)
|
||||
GRANULARITY 1;
|
||||
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
ADD INDEX IF NOT EXISTS idx_message_text_search lower(message)
|
||||
TYPE ngrambf_v1(3, 32768, 2, 0)
|
||||
GRANULARITY 1;
|
||||
|
||||
-- +goose Down
|
||||
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
DROP INDEX idx_attributes_text_search;
|
||||
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
DROP INDEX idx_message_text_search;
|
||||
@@ -0,0 +1,63 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS trigger_dev.task_events_search_v1
|
||||
(
|
||||
environment_id String,
|
||||
organization_id String,
|
||||
project_id String,
|
||||
triggered_timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
trace_id String CODEC(ZSTD(1)),
|
||||
span_id String CODEC(ZSTD(1)),
|
||||
run_id String CODEC(ZSTD(1)),
|
||||
task_identifier String CODEC(ZSTD(1)),
|
||||
start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
inserted_at DateTime64(3),
|
||||
message String CODEC(ZSTD(1)),
|
||||
kind LowCardinality(String) CODEC(ZSTD(1)),
|
||||
status LowCardinality(String) CODEC(ZSTD(1)),
|
||||
duration UInt64 CODEC(ZSTD(1)),
|
||||
parent_span_id String CODEC(ZSTD(1)),
|
||||
attributes_text String CODEC(ZSTD(1)),
|
||||
|
||||
INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_message_text_search lower(message) TYPE ngrambf_v1(3, 32768, 2, 0) GRANULARITY 1,
|
||||
INDEX idx_attributes_text_search lower(attributes_text) TYPE ngrambf_v1(3, 32768, 2, 0) GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(triggered_timestamp)
|
||||
ORDER BY (organization_id, environment_id, triggered_timestamp, trace_id)
|
||||
--Right now we have maximum retention of up to 30 days based on plan.
|
||||
--We put a logical limit for now, the 90 DAY TTL is just a backup
|
||||
--This might need to be updated for longer retention periods
|
||||
TTL toDateTime(triggered_timestamp) + INTERVAL 90 DAY
|
||||
SETTINGS ttl_only_drop_parts = 1;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.task_events_search_mv_v1
|
||||
TO trigger_dev.task_events_search_v1 AS
|
||||
SELECT
|
||||
environment_id,
|
||||
organization_id,
|
||||
project_id,
|
||||
trace_id,
|
||||
span_id,
|
||||
run_id,
|
||||
task_identifier,
|
||||
start_time,
|
||||
inserted_at,
|
||||
message,
|
||||
kind,
|
||||
status,
|
||||
duration,
|
||||
parent_span_id,
|
||||
attributes_text,
|
||||
fromUnixTimestamp64Nano(toUnixTimestamp64Nano(start_time) + toInt64(duration)) AS triggered_timestamp
|
||||
FROM trigger_dev.task_events_v2
|
||||
WHERE
|
||||
kind != 'DEBUG_EVENT'
|
||||
AND status != 'PARTIAL'
|
||||
AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}')
|
||||
AND kind != 'ANCESTOR_OVERRIDE'
|
||||
AND message != 'trigger.dev/start';
|
||||
|
||||
-- +goose Down
|
||||
DROP VIEW IF EXISTS trigger_dev.task_events_search_mv_v1;
|
||||
DROP TABLE IF EXISTS trigger_dev.task_events_search_v1;
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
-- +goose Up
|
||||
-- We drop the existing MV and recreate it with the new filter condition
|
||||
DROP VIEW IF EXISTS trigger_dev.task_events_search_mv_v1;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.task_events_search_mv_v1
|
||||
TO trigger_dev.task_events_search_v1 AS
|
||||
SELECT
|
||||
environment_id,
|
||||
organization_id,
|
||||
project_id,
|
||||
trace_id,
|
||||
span_id,
|
||||
run_id,
|
||||
task_identifier,
|
||||
start_time,
|
||||
inserted_at,
|
||||
message,
|
||||
kind,
|
||||
status,
|
||||
duration,
|
||||
parent_span_id,
|
||||
attributes_text,
|
||||
fromUnixTimestamp64Nano(toUnixTimestamp64Nano(start_time) + toInt64(duration)) AS triggered_timestamp
|
||||
FROM trigger_dev.task_events_v2
|
||||
WHERE
|
||||
trace_id != '' -- New condition added here
|
||||
AND kind != 'DEBUG_EVENT'
|
||||
AND status != 'PARTIAL'
|
||||
AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}')
|
||||
AND kind != 'ANCESTOR_OVERRIDE'
|
||||
AND message != 'trigger.dev/start';
|
||||
|
||||
-- +goose Down
|
||||
-- In the down migration, we revert to the previous filter set
|
||||
DROP VIEW IF EXISTS trigger_dev.task_events_search_mv_v1;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.task_events_search_mv_v1
|
||||
TO trigger_dev.task_events_search_v1 AS
|
||||
SELECT
|
||||
environment_id,
|
||||
organization_id,
|
||||
project_id,
|
||||
trace_id,
|
||||
span_id,
|
||||
run_id,
|
||||
task_identifier,
|
||||
start_time,
|
||||
inserted_at,
|
||||
message,
|
||||
kind,
|
||||
status,
|
||||
duration,
|
||||
parent_span_id,
|
||||
attributes_text,
|
||||
fromUnixTimestamp64Nano(toUnixTimestamp64Nano(start_time) + toInt64(duration)) AS triggered_timestamp
|
||||
FROM trigger_dev.task_events_v2
|
||||
WHERE
|
||||
kind != 'DEBUG_EVENT'
|
||||
AND status != 'PARTIAL'
|
||||
AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}')
|
||||
AND kind != 'ANCESTOR_OVERRIDE'
|
||||
AND message != 'trigger.dev/start';
|
||||
@@ -0,0 +1,22 @@
|
||||
-- +goose Up
|
||||
|
||||
-- These indexes are not used in any WHERE clause.
|
||||
-- idx_duration: duration is only written/read as a column, never filtered on.
|
||||
-- idx_attributes_text: search queries use the task_events_search_v1 table instead.
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
DROP INDEX IF EXISTS idx_duration;
|
||||
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
DROP INDEX IF EXISTS idx_attributes_text;
|
||||
|
||||
-- +goose Down
|
||||
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
ADD INDEX IF NOT EXISTS idx_duration duration
|
||||
TYPE minmax
|
||||
GRANULARITY 1;
|
||||
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
ADD INDEX IF NOT EXISTS idx_attributes_text attributes_text
|
||||
TYPE tokenbf_v1(32768, 3, 0)
|
||||
GRANULARITY 8;
|
||||
@@ -0,0 +1,44 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS trigger_dev.metrics_v1
|
||||
(
|
||||
organization_id LowCardinality(String),
|
||||
project_id LowCardinality(String),
|
||||
environment_id String CODEC(ZSTD(1)),
|
||||
metric_name LowCardinality(String),
|
||||
metric_type LowCardinality(String),
|
||||
metric_subject String CODEC(ZSTD(1)),
|
||||
bucket_start DateTime CODEC(Delta(4), ZSTD(1)),
|
||||
value Float64 DEFAULT 0 CODEC(ZSTD(1)),
|
||||
attributes JSON(
|
||||
`trigger.run_id` String,
|
||||
`trigger.task_slug` String,
|
||||
`trigger.attempt_number` Int64,
|
||||
`trigger.environment_type` LowCardinality(String),
|
||||
`trigger.machine_id` String,
|
||||
`trigger.machine_name` LowCardinality(String),
|
||||
`trigger.worker_id` String,
|
||||
`trigger.worker_version` String,
|
||||
`system.cpu.logical_number` String,
|
||||
`system.cpu.state` LowCardinality(String),
|
||||
`system.memory.state` LowCardinality(String),
|
||||
`system.device` String,
|
||||
`system.filesystem.type` LowCardinality(String),
|
||||
`system.filesystem.mountpoint` String,
|
||||
`system.filesystem.mode` LowCardinality(String),
|
||||
`system.filesystem.state` LowCardinality(String),
|
||||
`disk.io.direction` LowCardinality(String),
|
||||
`process.cpu.state` LowCardinality(String),
|
||||
`network.io.direction` LowCardinality(String),
|
||||
max_dynamic_paths=8
|
||||
),
|
||||
INDEX idx_run_id attributes.trigger.run_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_task_slug attributes.trigger.task_slug TYPE bloom_filter(0.001) GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toDate(bucket_start)
|
||||
ORDER BY (organization_id, project_id, environment_id, metric_name, metric_subject, bucket_start)
|
||||
TTL bucket_start + INTERVAL 60 DAY
|
||||
SETTINGS ttl_only_drop_parts = 1;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.metrics_v1;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
ADD COLUMN machine_id String DEFAULT '' CODEC(ZSTD(1));
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_events_v2
|
||||
DROP COLUMN machine_id;
|
||||
@@ -0,0 +1,11 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN error_fingerprint String DEFAULT '';
|
||||
|
||||
-- Bloom filter index for fast error fingerprint lookups
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD INDEX idx_error_fingerprint error_fingerprint TYPE bloom_filter GRANULARITY 4;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_runs_v2 DROP INDEX idx_error_fingerprint;
|
||||
ALTER TABLE trigger_dev.task_runs_v2 DROP COLUMN error_fingerprint;
|
||||
@@ -0,0 +1,78 @@
|
||||
-- +goose Up
|
||||
|
||||
-- Aggregated error groups table (per task + fingerprint)
|
||||
CREATE TABLE trigger_dev.errors_v1
|
||||
(
|
||||
organization_id String,
|
||||
project_id String,
|
||||
environment_id String,
|
||||
task_identifier String,
|
||||
error_fingerprint String,
|
||||
|
||||
-- Error details (samples from occurrences)
|
||||
error_type String,
|
||||
error_message String,
|
||||
sample_stack_trace String,
|
||||
|
||||
-- SimpleAggregateFunction stores raw values and applies the function during merge,
|
||||
-- avoiding binary state encoding issues with AggregateFunction.
|
||||
last_seen_date SimpleAggregateFunction(max, DateTime),
|
||||
|
||||
first_seen SimpleAggregateFunction(min, DateTime64(3)),
|
||||
last_seen SimpleAggregateFunction(max, DateTime64(3)),
|
||||
occurrence_count AggregateFunction(sum, UInt64),
|
||||
affected_task_versions AggregateFunction(uniq, String),
|
||||
|
||||
-- Samples for debugging
|
||||
sample_run_id AggregateFunction(any, String),
|
||||
sample_friendly_id AggregateFunction(any, String),
|
||||
|
||||
-- Status distribution
|
||||
status_distribution AggregateFunction(sumMap, Array(String), Array(UInt64))
|
||||
)
|
||||
ENGINE = AggregatingMergeTree()
|
||||
ORDER BY (organization_id, project_id, environment_id, task_identifier, error_fingerprint)
|
||||
TTL last_seen_date + INTERVAL 90 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- Materialized view to auto-populate from task_runs_v2
|
||||
CREATE MATERIALIZED VIEW trigger_dev.errors_mv_v1
|
||||
TO trigger_dev.errors_v1
|
||||
AS
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
|
||||
any(coalesce(nullIf(toString(error.data.type), ''), nullIf(toString(error.data.name), ''), 'Error')) as error_type,
|
||||
any(coalesce(nullIf(substring(toString(error.data.message), 1, 500), ''), 'Unknown error')) as error_message,
|
||||
any(coalesce(substring(toString(error.data.stack), 1, 2000), '')) as sample_stack_trace,
|
||||
|
||||
toDateTime(max(created_at)) as last_seen_date,
|
||||
|
||||
min(created_at) as first_seen,
|
||||
max(created_at) as last_seen,
|
||||
sumState(toUInt64(1)) as occurrence_count,
|
||||
uniqState(task_version) as affected_task_versions,
|
||||
|
||||
anyState(run_id) as sample_run_id,
|
||||
anyState(friendly_id) as sample_friendly_id,
|
||||
|
||||
sumMapState([status], [toUInt64(1)]) as status_distribution
|
||||
FROM trigger_dev.task_runs_v2
|
||||
WHERE
|
||||
error_fingerprint != ''
|
||||
AND status IN ('SYSTEM_FAILURE', 'CRASHED', 'INTERRUPTED', 'COMPLETED_WITH_ERRORS', 'TIMED_OUT')
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint;
|
||||
|
||||
-- +goose Down
|
||||
DROP VIEW IF EXISTS trigger_dev.errors_mv_v1;
|
||||
DROP TABLE IF EXISTS trigger_dev.errors_v1;
|
||||
@@ -0,0 +1,89 @@
|
||||
-- +goose Up
|
||||
-- Per-minute error occurrence counts, keyed by fingerprint + task + version.
|
||||
-- Powers precise time-range filtering and dynamic-granularity occurrence charts.
|
||||
CREATE TABLE
|
||||
trigger_dev.error_occurrences_v1 (
|
||||
organization_id String,
|
||||
project_id String,
|
||||
environment_id String,
|
||||
task_identifier String,
|
||||
error_fingerprint String,
|
||||
task_version String,
|
||||
minute DateTime,
|
||||
error_type String,
|
||||
error_message String,
|
||||
stack_trace String,
|
||||
count UInt64,
|
||||
INDEX idx_error_type_search lower(error_type) TYPE ngrambf_v1 (3, 32768, 2, 0) GRANULARITY 1,
|
||||
INDEX idx_error_message_search lower(error_message) TYPE ngrambf_v1 (3, 32768, 2, 0) GRANULARITY 1
|
||||
) ENGINE = SummingMergeTree (count)
|
||||
PARTITION BY
|
||||
toDate (minute)
|
||||
ORDER BY
|
||||
(
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
task_version,
|
||||
minute
|
||||
) TTL minute + INTERVAL 90 DAY SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE MATERIALIZED VIEW trigger_dev.error_occurrences_mv_v1 TO trigger_dev.error_occurrences_v1 AS
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
task_version,
|
||||
toStartOfMinute (created_at) as minute,
|
||||
any (
|
||||
coalesce(
|
||||
nullIf(toString (error.data.type), ''),
|
||||
nullIf(toString (error.data.name), ''),
|
||||
'Error'
|
||||
)
|
||||
) as error_type,
|
||||
any (
|
||||
coalesce(
|
||||
nullIf(
|
||||
substring(toString (error.data.message), 1, 500),
|
||||
''
|
||||
),
|
||||
'Unknown error'
|
||||
)
|
||||
) as error_message,
|
||||
any (
|
||||
coalesce(
|
||||
substring(toString (error.data.stack), 1, 2000),
|
||||
''
|
||||
)
|
||||
) as stack_trace,
|
||||
count() as count
|
||||
FROM
|
||||
trigger_dev.task_runs_v2
|
||||
WHERE
|
||||
error_fingerprint != ''
|
||||
AND status IN (
|
||||
'SYSTEM_FAILURE',
|
||||
'CRASHED',
|
||||
'INTERRUPTED',
|
||||
'COMPLETED_WITH_ERRORS',
|
||||
'TIMED_OUT'
|
||||
)
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
task_version,
|
||||
minute;
|
||||
|
||||
-- +goose Down
|
||||
DROP VIEW IF EXISTS trigger_dev.error_occurrences_mv_v1;
|
||||
|
||||
DROP TABLE IF EXISTS trigger_dev.error_occurrences_v1;
|
||||
@@ -0,0 +1,64 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS trigger_dev.llm_metrics_v1
|
||||
(
|
||||
-- Tenant context
|
||||
organization_id LowCardinality(String),
|
||||
project_id LowCardinality(String),
|
||||
environment_id String CODEC(ZSTD(1)),
|
||||
run_id String CODEC(ZSTD(1)),
|
||||
task_identifier LowCardinality(String),
|
||||
trace_id String CODEC(ZSTD(1)),
|
||||
span_id String CODEC(ZSTD(1)),
|
||||
|
||||
-- Model & provider
|
||||
gen_ai_system LowCardinality(String),
|
||||
request_model String CODEC(ZSTD(1)),
|
||||
response_model String CODEC(ZSTD(1)),
|
||||
matched_model_id String CODEC(ZSTD(1)),
|
||||
operation_id LowCardinality(String),
|
||||
finish_reason LowCardinality(String),
|
||||
cost_source LowCardinality(String),
|
||||
|
||||
-- Pricing
|
||||
pricing_tier_id String CODEC(ZSTD(1)),
|
||||
pricing_tier_name LowCardinality(String),
|
||||
|
||||
-- Token usage
|
||||
input_tokens UInt64 DEFAULT 0,
|
||||
output_tokens UInt64 DEFAULT 0,
|
||||
total_tokens UInt64 DEFAULT 0,
|
||||
usage_details Map(LowCardinality(String), UInt64),
|
||||
|
||||
-- Cost
|
||||
input_cost Decimal64(12) DEFAULT 0,
|
||||
output_cost Decimal64(12) DEFAULT 0,
|
||||
total_cost Decimal64(12) DEFAULT 0,
|
||||
cost_details Map(LowCardinality(String), Decimal64(12)),
|
||||
provider_cost Decimal64(12) DEFAULT 0,
|
||||
|
||||
-- Performance
|
||||
ms_to_first_chunk Float64 DEFAULT 0,
|
||||
tokens_per_second Float64 DEFAULT 0,
|
||||
|
||||
-- Attribution
|
||||
metadata Map(LowCardinality(String), String),
|
||||
|
||||
-- Timing
|
||||
start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
duration UInt64 DEFAULT 0 CODEC(ZSTD(1)),
|
||||
inserted_at DateTime64(3) DEFAULT now64(3),
|
||||
|
||||
-- Indexes
|
||||
INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_metadata_keys mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(inserted_at)
|
||||
ORDER BY (organization_id, project_id, environment_id, toDate(inserted_at), run_id)
|
||||
TTL toDateTime(inserted_at) + INTERVAL 365 DAY
|
||||
SETTINGS ttl_only_drop_parts = 1;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.llm_metrics_v1;
|
||||
@@ -0,0 +1,13 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE trigger_dev.llm_metrics_v1
|
||||
ADD COLUMN prompt_slug LowCardinality(String) DEFAULT '';
|
||||
|
||||
ALTER TABLE trigger_dev.llm_metrics_v1
|
||||
ADD COLUMN prompt_version UInt32 DEFAULT 0;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.llm_metrics_v1
|
||||
DROP COLUMN prompt_slug;
|
||||
|
||||
ALTER TABLE trigger_dev.llm_metrics_v1
|
||||
DROP COLUMN prompt_version;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE trigger_dev.llm_metrics_v1
|
||||
ADD COLUMN base_response_model String DEFAULT '' CODEC(ZSTD(1));
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.llm_metrics_v1
|
||||
DROP COLUMN base_response_model;
|
||||
@@ -0,0 +1,57 @@
|
||||
-- +goose Up
|
||||
|
||||
-- Pre-aggregated model performance metrics with no tenant information.
|
||||
-- Used for cross-tenant model comparisons in the Model Registry.
|
||||
-- Aggregated per minute for high-resolution model performance tracking.
|
||||
CREATE TABLE IF NOT EXISTS trigger_dev.llm_model_aggregates_v1
|
||||
(
|
||||
response_model String,
|
||||
base_response_model String DEFAULT '',
|
||||
gen_ai_system LowCardinality(String),
|
||||
minute DateTime,
|
||||
|
||||
-- Counts & totals (SimpleAggregateFunction for sum)
|
||||
call_count SimpleAggregateFunction(sum, UInt64),
|
||||
total_input_tokens SimpleAggregateFunction(sum, UInt64),
|
||||
total_output_tokens SimpleAggregateFunction(sum, UInt64),
|
||||
total_cost SimpleAggregateFunction(sum, Float64),
|
||||
|
||||
-- Performance quantiles (AggregateFunction for merge across parts)
|
||||
ttfc_quantiles AggregateFunction(quantiles(0.5, 0.9, 0.95, 0.99), Float64),
|
||||
tps_quantiles AggregateFunction(quantiles(0.5, 0.9, 0.95, 0.99), Float64),
|
||||
duration_quantiles AggregateFunction(quantiles(0.5, 0.9, 0.95, 0.99), UInt64),
|
||||
|
||||
-- Finish reason distribution
|
||||
finish_reason_counts SimpleAggregateFunction(sumMap, Map(String, UInt64))
|
||||
)
|
||||
ENGINE = AggregatingMergeTree
|
||||
PARTITION BY toYYYYMM(minute)
|
||||
ORDER BY (response_model, base_response_model, gen_ai_system, minute)
|
||||
TTL toDate(minute) + INTERVAL 365 DAY
|
||||
SETTINGS ttl_only_drop_parts = 1;
|
||||
|
||||
-- Materialized view that feeds the aggregate table from llm_metrics_v1.
|
||||
-- Strips all tenant-specific columns (org, project, env, run, span, trace).
|
||||
-- base_response_model comes from the source table (populated during event enrichment).
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.llm_model_aggregates_mv_v1
|
||||
TO trigger_dev.llm_model_aggregates_v1
|
||||
AS SELECT
|
||||
response_model,
|
||||
base_response_model,
|
||||
gen_ai_system,
|
||||
toStartOfMinute(start_time) AS minute,
|
||||
count() AS call_count,
|
||||
sum(input_tokens) AS total_input_tokens,
|
||||
sum(output_tokens) AS total_output_tokens,
|
||||
sum(total_cost) AS total_cost,
|
||||
quantilesStateIf(0.5, 0.9, 0.95, 0.99)(ms_to_first_chunk, ms_to_first_chunk > 0) AS ttfc_quantiles,
|
||||
quantilesStateIf(0.5, 0.9, 0.95, 0.99)(tokens_per_second, tokens_per_second > 0) AS tps_quantiles,
|
||||
quantilesState(0.5, 0.9, 0.95, 0.99)(duration) AS duration_quantiles,
|
||||
sumMap(map(finish_reason, toUInt64(1))) AS finish_reason_counts
|
||||
FROM trigger_dev.llm_metrics_v1
|
||||
WHERE response_model != ''
|
||||
GROUP BY response_model, base_response_model, gen_ai_system, minute;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.llm_model_aggregates_mv_v1;
|
||||
DROP TABLE IF EXISTS trigger_dev.llm_model_aggregates_v1;
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN trigger_source LowCardinality(String) DEFAULT '';
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN root_trigger_source LowCardinality(String) DEFAULT '';
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN is_warm_start Nullable(UInt8) DEFAULT NULL;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN trigger_source;
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN root_trigger_source;
|
||||
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN is_warm_start;
|
||||
@@ -0,0 +1,42 @@
|
||||
-- +goose Up
|
||||
|
||||
CREATE TABLE trigger_dev.sessions_v1
|
||||
(
|
||||
/* ─── identity ─────────────────────────────────────────────── */
|
||||
environment_id String,
|
||||
organization_id String,
|
||||
project_id String,
|
||||
session_id String,
|
||||
|
||||
environment_type LowCardinality(String),
|
||||
friendly_id String,
|
||||
external_id String DEFAULT '',
|
||||
|
||||
/* ─── type discriminator ──────────────────────────────────── */
|
||||
type LowCardinality(String),
|
||||
task_identifier String DEFAULT '',
|
||||
|
||||
/* ─── filtering / free-form ──────────────────────────────── */
|
||||
tags Array(String) CODEC(ZSTD(1)),
|
||||
metadata JSON(max_dynamic_paths = 256),
|
||||
|
||||
/* ─── terminal markers ────────────────────────────────────── */
|
||||
closed_at Nullable(DateTime64(3)),
|
||||
closed_reason String DEFAULT '',
|
||||
expires_at Nullable(DateTime64(3)),
|
||||
|
||||
/* ─── timing ─────────────────────────────────────────────── */
|
||||
created_at DateTime64(3),
|
||||
updated_at DateTime64(3),
|
||||
|
||||
/* ─── commit lsn ────────────────────────────────────────── */
|
||||
_version UInt64,
|
||||
_is_deleted UInt8 DEFAULT 0
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(_version, _is_deleted)
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (organization_id, project_id, environment_id, created_at, session_id)
|
||||
SETTINGS enable_json_type = 1;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS trigger_dev.sessions_v1;
|
||||
@@ -0,0 +1,12 @@
|
||||
-- +goose Up
|
||||
-- IF NOT EXISTS is required because this migration was previously numbered
|
||||
-- 029 and may have been applied in environments where goose accepted it
|
||||
-- before 030_create_sessions_v1 advanced the version counter. Renaming to
|
||||
-- 031 makes goose treat this as new everywhere, so the DDL must tolerate
|
||||
-- the column already being present.
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN IF NOT EXISTS task_kind LowCardinality(String) DEFAULT '';
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN IF EXISTS task_kind;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN IF NOT EXISTS region String DEFAULT '';
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN IF EXISTS region;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
ADD COLUMN IF NOT EXISTS plan_type LowCardinality(String) DEFAULT '';
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.task_runs_v2
|
||||
DROP COLUMN IF EXISTS plan_type;
|
||||
@@ -0,0 +1,11 @@
|
||||
-- +goose Up
|
||||
-- Existing rows default to 0 and are intentionally NOT backfilled: the Sessions
|
||||
-- list reads isTest from Postgres (ClickHouse only supplies session IDs), so the
|
||||
-- UI is correct without it. A backfill is only needed if a ClickHouse-side
|
||||
-- isTest filter/aggregate over sessions_v1 is added later.
|
||||
ALTER TABLE trigger_dev.sessions_v1
|
||||
ADD COLUMN IF NOT EXISTS is_test UInt8 DEFAULT 0;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE trigger_dev.sessions_v1
|
||||
DROP COLUMN IF EXISTS is_test;
|
||||
@@ -0,0 +1,192 @@
|
||||
-- +goose Up
|
||||
-- Fix how the error materialized views derive their display columns from the
|
||||
-- stored error JSON:
|
||||
-- * error_type: use the real class name (or internal code), not the generic
|
||||
-- serialization tag (BUILT_IN_ERROR / STRING_ERROR / ...).
|
||||
-- * error_message: fall back name -> raw before 'Unknown error' so messageless
|
||||
-- errors (tagged errors) and non-Error throws still get a meaningful title.
|
||||
-- * stack trace: read error.data.stackTrace (the stored field), not
|
||||
-- error.data.stack, which was always empty.
|
||||
-- Display-only. Only affects rows inserted after this migration.
|
||||
|
||||
ALTER TABLE trigger_dev.errors_mv_v1 MODIFY QUERY
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
|
||||
any(coalesce(nullIf(toString(error.data.name), ''), nullIf(toString(error.data.code), ''), 'Error')) as error_type,
|
||||
any(coalesce(
|
||||
nullIf(substring(toString(error.data.message), 1, 500), ''),
|
||||
nullIf(toString(error.data.name), ''),
|
||||
nullIf(substring(toString(error.data.raw), 1, 500), ''),
|
||||
'Unknown error'
|
||||
)) as error_message,
|
||||
any(coalesce(substring(toString(error.data.stackTrace), 1, 2000), '')) as sample_stack_trace,
|
||||
|
||||
toDateTime(max(created_at)) as last_seen_date,
|
||||
|
||||
min(created_at) as first_seen,
|
||||
max(created_at) as last_seen,
|
||||
sumState(toUInt64(1)) as occurrence_count,
|
||||
uniqState(task_version) as affected_task_versions,
|
||||
|
||||
anyState(run_id) as sample_run_id,
|
||||
anyState(friendly_id) as sample_friendly_id,
|
||||
|
||||
sumMapState([status], [toUInt64(1)]) as status_distribution
|
||||
FROM trigger_dev.task_runs_v2
|
||||
WHERE
|
||||
error_fingerprint != ''
|
||||
AND status IN ('SYSTEM_FAILURE', 'CRASHED', 'INTERRUPTED', 'COMPLETED_WITH_ERRORS', 'TIMED_OUT')
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint;
|
||||
|
||||
ALTER TABLE trigger_dev.error_occurrences_mv_v1 MODIFY QUERY
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
task_version,
|
||||
toStartOfMinute (created_at) as minute,
|
||||
any (
|
||||
coalesce(
|
||||
nullIf(toString (error.data.name), ''),
|
||||
nullIf(toString (error.data.code), ''),
|
||||
'Error'
|
||||
)
|
||||
) as error_type,
|
||||
any (
|
||||
coalesce(
|
||||
nullIf(substring(toString (error.data.message), 1, 500), ''),
|
||||
nullIf(toString (error.data.name), ''),
|
||||
nullIf(substring(toString (error.data.raw), 1, 500), ''),
|
||||
'Unknown error'
|
||||
)
|
||||
) as error_message,
|
||||
any (
|
||||
coalesce(
|
||||
substring(toString (error.data.stackTrace), 1, 2000),
|
||||
''
|
||||
)
|
||||
) as stack_trace,
|
||||
count() as count
|
||||
FROM
|
||||
trigger_dev.task_runs_v2
|
||||
WHERE
|
||||
error_fingerprint != ''
|
||||
AND status IN (
|
||||
'SYSTEM_FAILURE',
|
||||
'CRASHED',
|
||||
'INTERRUPTED',
|
||||
'COMPLETED_WITH_ERRORS',
|
||||
'TIMED_OUT'
|
||||
)
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
task_version,
|
||||
minute;
|
||||
|
||||
-- +goose Down
|
||||
|
||||
ALTER TABLE trigger_dev.errors_mv_v1 MODIFY QUERY
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
|
||||
any(coalesce(nullIf(toString(error.data.type), ''), nullIf(toString(error.data.name), ''), 'Error')) as error_type,
|
||||
any(coalesce(nullIf(substring(toString(error.data.message), 1, 500), ''), 'Unknown error')) as error_message,
|
||||
any(coalesce(substring(toString(error.data.stack), 1, 2000), '')) as sample_stack_trace,
|
||||
|
||||
toDateTime(max(created_at)) as last_seen_date,
|
||||
|
||||
min(created_at) as first_seen,
|
||||
max(created_at) as last_seen,
|
||||
sumState(toUInt64(1)) as occurrence_count,
|
||||
uniqState(task_version) as affected_task_versions,
|
||||
|
||||
anyState(run_id) as sample_run_id,
|
||||
anyState(friendly_id) as sample_friendly_id,
|
||||
|
||||
sumMapState([status], [toUInt64(1)]) as status_distribution
|
||||
FROM trigger_dev.task_runs_v2
|
||||
WHERE
|
||||
error_fingerprint != ''
|
||||
AND status IN ('SYSTEM_FAILURE', 'CRASHED', 'INTERRUPTED', 'COMPLETED_WITH_ERRORS', 'TIMED_OUT')
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint;
|
||||
|
||||
ALTER TABLE trigger_dev.error_occurrences_mv_v1 MODIFY QUERY
|
||||
SELECT
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
task_version,
|
||||
toStartOfMinute (created_at) as minute,
|
||||
any (
|
||||
coalesce(
|
||||
nullIf(toString (error.data.type), ''),
|
||||
nullIf(toString (error.data.name), ''),
|
||||
'Error'
|
||||
)
|
||||
) as error_type,
|
||||
any (
|
||||
coalesce(
|
||||
nullIf(
|
||||
substring(toString (error.data.message), 1, 500),
|
||||
''
|
||||
),
|
||||
'Unknown error'
|
||||
)
|
||||
) as error_message,
|
||||
any (
|
||||
coalesce(
|
||||
substring(toString (error.data.stack), 1, 2000),
|
||||
''
|
||||
)
|
||||
) as stack_trace,
|
||||
count() as count
|
||||
FROM
|
||||
trigger_dev.task_runs_v2
|
||||
WHERE
|
||||
error_fingerprint != ''
|
||||
AND status IN (
|
||||
'SYSTEM_FAILURE',
|
||||
'CRASHED',
|
||||
'INTERRUPTED',
|
||||
'COMPLETED_WITH_ERRORS',
|
||||
'TIMED_OUT'
|
||||
)
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
organization_id,
|
||||
project_id,
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
task_version,
|
||||
minute;
|
||||
@@ -0,0 +1,20 @@
|
||||
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
||||
|
||||
exports[`Task Runs V2 > should be able to query task runs using the query builder 1`] = `
|
||||
{
|
||||
"params": {
|
||||
"environmentId": "cm9kddfcs01zqdy88ld9mmrli",
|
||||
},
|
||||
"query": "SELECT run_id, toUnixTimestamp64Milli(created_at) AS created_at_ms FROM trigger_dev.task_runs_v2 FINAL WHERE environment_id = {environmentId: String}",
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Task Runs V2 > should be able to query task runs using the query builder 2`] = `
|
||||
{
|
||||
"params": {
|
||||
"environmentId": "cm9kddfcs01zqdy88ld9mmrli",
|
||||
"status": "COMPLETED_SUCCESSFULLY",
|
||||
},
|
||||
"query": "SELECT run_id, toUnixTimestamp64Milli(created_at) AS created_at_ms FROM trigger_dev.task_runs_v2 FINAL WHERE environment_id = {environmentId: String} AND status = {status: String}",
|
||||
}
|
||||
`;
|
||||
@@ -0,0 +1,149 @@
|
||||
import { clickhouseTest } from "@internal/testcontainers";
|
||||
import { ClickhouseClient } from "./client.js";
|
||||
import { z } from "zod";
|
||||
import { setTimeout } from "timers/promises";
|
||||
|
||||
describe("ClickHouse Client", () => {
|
||||
clickhouseTest("should be able to insert and query data", async ({ clickhouseContainer }) => {
|
||||
const client = new ClickhouseClient({
|
||||
name: "test",
|
||||
url: clickhouseContainer.getConnectionUrl(),
|
||||
});
|
||||
|
||||
const insertSmokeTest = client.insert({
|
||||
name: "insert-smoke-test",
|
||||
table: "trigger_dev.smoke_test",
|
||||
schema: z.object({
|
||||
message: z.string(),
|
||||
number: z.number(),
|
||||
}),
|
||||
});
|
||||
|
||||
const querySmokeTest = client.query({
|
||||
name: "query-smoke-test",
|
||||
query: "SELECT * FROM trigger_dev.smoke_test",
|
||||
schema: z.object({
|
||||
message: z.string(),
|
||||
number: z.number(),
|
||||
timestamp: z.string(),
|
||||
id: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
const [insertError, insertResult] = await insertSmokeTest([
|
||||
{ message: "hello", number: 42 },
|
||||
{ message: "world", number: 100 },
|
||||
]);
|
||||
|
||||
expect(insertError).toBeNull();
|
||||
expect(insertResult).toEqual(
|
||||
expect.objectContaining({
|
||||
executed: true,
|
||||
query_id: expect.any(String),
|
||||
summary: expect.objectContaining({ read_rows: "2", elapsed_ns: expect.any(String) }),
|
||||
})
|
||||
);
|
||||
|
||||
const [queryError, result] = await querySmokeTest({});
|
||||
|
||||
expect(queryError).toBeNull();
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
message: "hello",
|
||||
number: 42,
|
||||
timestamp: expect.any(String),
|
||||
id: expect.any(String),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
message: "world",
|
||||
number: 100,
|
||||
timestamp: expect.any(String),
|
||||
id: expect.any(String),
|
||||
}),
|
||||
])
|
||||
);
|
||||
|
||||
const insertSmokeTestAsyncWaiting = client.insert({
|
||||
name: "insert-smoke-test-async-waiting",
|
||||
table: "trigger_dev.smoke_test",
|
||||
schema: z.object({
|
||||
message: z.string(),
|
||||
number: z.number(),
|
||||
}),
|
||||
settings: {
|
||||
async_insert: 1,
|
||||
wait_for_async_insert: 1,
|
||||
async_insert_busy_timeout_ms: 1000,
|
||||
},
|
||||
});
|
||||
|
||||
const [insertErrorAsyncWaiting, insertResultAsyncWaiting] = await insertSmokeTestAsyncWaiting([
|
||||
{ message: "async-waiting-hello", number: 42 },
|
||||
{ message: "async-waiting-world", number: 100 },
|
||||
]);
|
||||
|
||||
expect(insertErrorAsyncWaiting).toBeNull();
|
||||
expect(insertResultAsyncWaiting).toEqual(expect.objectContaining({ executed: true }));
|
||||
|
||||
// Should be able to query for the data right away
|
||||
const [queryErrorAsyncWaiting, resultAsyncWaiting] = await querySmokeTest({});
|
||||
|
||||
expect(queryErrorAsyncWaiting).toBeNull();
|
||||
expect(resultAsyncWaiting).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ message: "async-waiting-hello", number: 42 }),
|
||||
expect.objectContaining({ message: "async-waiting-world", number: 100 }),
|
||||
])
|
||||
);
|
||||
|
||||
const insertSmokeTestAsyncDontWait = client.insert({
|
||||
name: "insert-smoke-test-async-dont-wait",
|
||||
table: "trigger_dev.smoke_test",
|
||||
schema: z.object({
|
||||
message: z.string(),
|
||||
number: z.number(),
|
||||
}),
|
||||
settings: {
|
||||
async_insert: 1,
|
||||
wait_for_async_insert: 0,
|
||||
async_insert_busy_timeout_ms: 1000,
|
||||
},
|
||||
});
|
||||
|
||||
const [insertErrorAsyncDontWait, insertResultAsyncDontWait] =
|
||||
await insertSmokeTestAsyncDontWait([
|
||||
{ message: "async-dont-wait-hello", number: 42 },
|
||||
{ message: "async-dont-wait-world", number: 100 },
|
||||
]);
|
||||
|
||||
expect(insertErrorAsyncDontWait).toBeNull();
|
||||
expect(insertResultAsyncDontWait).toEqual(expect.objectContaining({ executed: true }));
|
||||
|
||||
// Querying now should return an array without the data
|
||||
const [queryErrorAsyncDontWait, resultAsyncDontWait] = await querySmokeTest({});
|
||||
|
||||
expect(queryErrorAsyncDontWait).toBeNull();
|
||||
expect(resultAsyncDontWait).toEqual(
|
||||
expect.not.arrayContaining([
|
||||
expect.objectContaining({ message: "async-dont-wait-hello", number: 42 }),
|
||||
expect.objectContaining({ message: "async-dont-wait-world", number: 100 }),
|
||||
])
|
||||
);
|
||||
|
||||
// Now we wait for the data to be flushed
|
||||
await setTimeout(2000);
|
||||
|
||||
// Querying now should return the data
|
||||
const [queryErrorAsyncDontWait2, resultAsyncDontWait2] = await querySmokeTest({});
|
||||
|
||||
expect(queryErrorAsyncDontWait2).toBeNull();
|
||||
expect(resultAsyncDontWait2).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ message: "async-dont-wait-hello", number: 42 }),
|
||||
expect.objectContaining({ message: "async-dont-wait-world", number: 100 }),
|
||||
])
|
||||
);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
export type ErrorContext = Record<string, unknown>;
|
||||
|
||||
export abstract class BaseError<TContext extends ErrorContext = ErrorContext> extends Error {
|
||||
public abstract readonly retry: boolean;
|
||||
public readonly cause: BaseError | undefined;
|
||||
public readonly context: TContext | undefined;
|
||||
public readonly message: string;
|
||||
public abstract readonly name: string;
|
||||
|
||||
constructor(opts: { message: string; cause?: BaseError; context?: TContext }) {
|
||||
super(opts.message);
|
||||
this.message = opts.message;
|
||||
this.cause = opts.cause;
|
||||
this.context = opts.context;
|
||||
}
|
||||
|
||||
public toString(): string {
|
||||
return `${this.name}: ${this.message} - ${JSON.stringify(
|
||||
this.context
|
||||
)} - caused by ${this.cause?.toString()}`;
|
||||
}
|
||||
}
|
||||
|
||||
export class InsertError extends BaseError {
|
||||
public readonly retry = true;
|
||||
public readonly name = InsertError.name;
|
||||
constructor(message: string) {
|
||||
super({
|
||||
message,
|
||||
});
|
||||
}
|
||||
}
|
||||
export class QueryError extends BaseError<{ query: string }> {
|
||||
public readonly retry = true;
|
||||
public readonly name = QueryError.name;
|
||||
constructor(message: string, context: { query: string }) {
|
||||
super({
|
||||
message,
|
||||
context,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
import type { Result } from "@trigger.dev/core/v3";
|
||||
import { InsertError, QueryError } from "./errors.js";
|
||||
import type {
|
||||
ClickhouseQueryBuilderFastFunction,
|
||||
ClickhouseQueryBuilderFunction,
|
||||
ClickhouseReader,
|
||||
ClickhouseWriter,
|
||||
QueryResultWithStats,
|
||||
} from "./types.js";
|
||||
import type { z } from "zod";
|
||||
import type { ClickHouseSettings, InsertResult } from "@clickhouse/client";
|
||||
import { ClickhouseQueryBuilder, ClickhouseQueryFastBuilder } from "./queryBuilder.js";
|
||||
|
||||
export class NoopClient implements ClickhouseReader, ClickhouseWriter {
|
||||
public async close() {
|
||||
return;
|
||||
}
|
||||
|
||||
public queryBuilder<TOut extends z.ZodSchema<any>>(req: {
|
||||
name: string;
|
||||
baseQuery: string;
|
||||
schema: TOut;
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseQueryBuilderFunction<z.input<TOut>> {
|
||||
return () =>
|
||||
new ClickhouseQueryBuilder(req.name, req.baseQuery, this, req.schema, req.settings);
|
||||
}
|
||||
|
||||
public queryBuilderFast<TOut extends Record<string, any>>(req: {
|
||||
name: string;
|
||||
table: string;
|
||||
columns: string[];
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseQueryBuilderFastFunction<TOut> {
|
||||
return () =>
|
||||
new ClickhouseQueryFastBuilder(req.name, req.table, req.columns, this, req.settings);
|
||||
}
|
||||
|
||||
public query<TIn extends z.ZodSchema<any>, TOut extends z.ZodSchema<any>>(req: {
|
||||
query: string;
|
||||
params?: TIn;
|
||||
schema: TOut;
|
||||
}): (params: z.input<TIn>) => Promise<Result<z.output<TOut>[], QueryError>> {
|
||||
return async (params: z.input<TIn>) => {
|
||||
const validParams = req.params?.safeParse(params);
|
||||
|
||||
if (validParams?.error) {
|
||||
return [new QueryError(`Bad params: ${validParams.error.message}`, { query: "" }), null];
|
||||
}
|
||||
|
||||
return [null, []];
|
||||
};
|
||||
}
|
||||
|
||||
public queryWithStats<TIn extends z.ZodSchema<any>, TOut extends z.ZodSchema<any>>(req: {
|
||||
query: string;
|
||||
params?: TIn;
|
||||
schema: TOut;
|
||||
}): (params: z.input<TIn>) => Promise<Result<QueryResultWithStats<z.output<TOut>>, QueryError>> {
|
||||
return async (params: z.input<TIn>) => {
|
||||
const validParams = req.params?.safeParse(params);
|
||||
|
||||
if (validParams?.error) {
|
||||
return [new QueryError(`Bad params: ${validParams.error.message}`, { query: "" }), null];
|
||||
}
|
||||
|
||||
return [
|
||||
null,
|
||||
{
|
||||
rows: [],
|
||||
stats: {
|
||||
read_rows: "0",
|
||||
read_bytes: "0",
|
||||
written_rows: "0",
|
||||
written_bytes: "0",
|
||||
total_rows_to_read: "0",
|
||||
result_rows: "0",
|
||||
result_bytes: "0",
|
||||
elapsed_ns: "0",
|
||||
byte_seconds: "0",
|
||||
},
|
||||
},
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
public queryFast<TOut extends Record<string, any>, TParams extends Record<string, any>>(req: {
|
||||
name: string;
|
||||
query: string;
|
||||
columns: string[];
|
||||
settings?: ClickHouseSettings;
|
||||
}): (params: TParams) => Promise<Result<TOut[], QueryError>> {
|
||||
return async (params: TParams) => {
|
||||
return [null, []];
|
||||
};
|
||||
}
|
||||
|
||||
public queryFastStream<
|
||||
TOut extends Record<string, any>,
|
||||
TParams extends Record<string, any>,
|
||||
>(req: {
|
||||
name: string;
|
||||
query: string;
|
||||
columns: string[];
|
||||
settings?: ClickHouseSettings;
|
||||
}): (params: TParams) => AsyncIterable<TOut> {
|
||||
return async function* () {
|
||||
// Noop: empty stream.
|
||||
};
|
||||
}
|
||||
|
||||
public insert<TSchema extends z.ZodSchema<any>>(req: {
|
||||
name: string;
|
||||
table: string;
|
||||
schema: TSchema;
|
||||
settings?: ClickHouseSettings;
|
||||
}): (
|
||||
events: z.input<TSchema> | z.input<TSchema>[]
|
||||
) => Promise<Result<InsertResult, InsertError>> {
|
||||
return async (events: z.input<TSchema> | z.input<TSchema>[]) => {
|
||||
const v = Array.isArray(events)
|
||||
? req.schema.array().safeParse(events)
|
||||
: req.schema.safeParse(events);
|
||||
|
||||
if (!v.success) {
|
||||
return [new InsertError(v.error.message), null];
|
||||
}
|
||||
|
||||
return [
|
||||
null,
|
||||
{
|
||||
executed: true,
|
||||
query_id: "noop",
|
||||
summary: {
|
||||
read_rows: "0",
|
||||
read_bytes: "0",
|
||||
written_rows: "0",
|
||||
written_bytes: "0",
|
||||
total_rows_to_read: "0",
|
||||
result_rows: "0",
|
||||
result_bytes: "0",
|
||||
elapsed_ns: "0",
|
||||
},
|
||||
response_headers: {},
|
||||
},
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
public insertUnsafe<TRecord extends Record<string, any>>(req: {
|
||||
name: string;
|
||||
table: string;
|
||||
settings?: ClickHouseSettings;
|
||||
}): (events: TRecord | TRecord[]) => Promise<Result<InsertResult, InsertError>> {
|
||||
return async (events: TRecord | TRecord[]) => {
|
||||
return [
|
||||
null,
|
||||
{
|
||||
executed: true,
|
||||
query_id: "noop",
|
||||
summary: {
|
||||
read_rows: "0",
|
||||
read_bytes: "0",
|
||||
written_rows: "0",
|
||||
written_bytes: "0",
|
||||
total_rows_to_read: "0",
|
||||
result_rows: "0",
|
||||
result_bytes: "0",
|
||||
elapsed_ns: "0",
|
||||
},
|
||||
response_headers: {},
|
||||
},
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
public insertCompact<TRecord extends Record<string, any>>(req: {
|
||||
name: string;
|
||||
table: string;
|
||||
columns: readonly string[];
|
||||
toArray: (record: TRecord) => any[];
|
||||
settings?: ClickHouseSettings;
|
||||
}): (events: TRecord | TRecord[]) => Promise<Result<InsertResult, InsertError>> {
|
||||
return async (events: TRecord | TRecord[]) => {
|
||||
return [
|
||||
null,
|
||||
{
|
||||
executed: true,
|
||||
query_id: "noop",
|
||||
summary: {
|
||||
read_rows: "0",
|
||||
read_bytes: "0",
|
||||
written_rows: "0",
|
||||
written_bytes: "0",
|
||||
total_rows_to_read: "0",
|
||||
result_rows: "0",
|
||||
result_bytes: "0",
|
||||
elapsed_ns: "0",
|
||||
},
|
||||
response_headers: {},
|
||||
},
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
public insertCompactRaw(req: {
|
||||
name: string;
|
||||
table: string;
|
||||
columns: readonly string[];
|
||||
settings?: ClickHouseSettings;
|
||||
}): (events: readonly any[][] | any[]) => Promise<Result<InsertResult, InsertError>> {
|
||||
return async (events: readonly any[][] | any[]) => {
|
||||
return [
|
||||
null,
|
||||
{
|
||||
executed: true,
|
||||
query_id: "noop",
|
||||
summary: {
|
||||
read_rows: "0",
|
||||
read_bytes: "0",
|
||||
written_rows: "0",
|
||||
written_bytes: "0",
|
||||
total_rows_to_read: "0",
|
||||
result_rows: "0",
|
||||
result_bytes: "0",
|
||||
elapsed_ns: "0",
|
||||
},
|
||||
response_headers: {},
|
||||
},
|
||||
];
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
import { z } from "zod";
|
||||
import type { ClickhouseQueryFunction, ClickhouseReader, ColumnExpression } from "./types.js";
|
||||
import type { ClickHouseSettings } from "@clickhouse/client";
|
||||
export type QueryParamValue = string | number | boolean | Array<string | number | boolean> | null;
|
||||
export type QueryParams = Record<string, QueryParamValue>;
|
||||
|
||||
export type WhereCondition = {
|
||||
clause: string;
|
||||
params?: QueryParams;
|
||||
};
|
||||
|
||||
export class ClickhouseQueryBuilder<TOutput> {
|
||||
private name: string;
|
||||
private baseQuery: string;
|
||||
private whereClauses: string[] = [];
|
||||
private havingClauses: string[] = [];
|
||||
private params: QueryParams = {};
|
||||
private orderByClause: string | null = null;
|
||||
private limitClause: string | null = null;
|
||||
private reader: ClickhouseReader;
|
||||
private schema: z.ZodSchema<TOutput>;
|
||||
private settings: ClickHouseSettings | undefined;
|
||||
private groupByClause: string | null = null;
|
||||
|
||||
constructor(
|
||||
name: string,
|
||||
baseQuery: string,
|
||||
reader: ClickhouseReader,
|
||||
schema: z.ZodSchema<TOutput>,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
this.name = name;
|
||||
this.baseQuery = baseQuery;
|
||||
this.reader = reader;
|
||||
this.schema = schema;
|
||||
this.settings = settings;
|
||||
}
|
||||
|
||||
/** Set query parameters without adding a WHERE clause. Use for base queries with inline params. */
|
||||
setParams(params: QueryParams): this {
|
||||
Object.assign(this.params, params);
|
||||
return this;
|
||||
}
|
||||
|
||||
where(clause: string, params?: QueryParams): this {
|
||||
this.whereClauses.push(clause);
|
||||
if (params) {
|
||||
Object.assign(this.params, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
whereIf(condition: any, clause: string, params?: QueryParams): this {
|
||||
if (condition) {
|
||||
this.where(clause, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
whereOr(conditions: WhereCondition[]): this {
|
||||
if (conditions.length === 0) {
|
||||
return this;
|
||||
}
|
||||
const combinedClause = conditions.map((c) => `(${c.clause})`).join(" OR ");
|
||||
this.whereClauses.push(`(${combinedClause})`);
|
||||
for (const condition of conditions) {
|
||||
if (condition.params) {
|
||||
Object.assign(this.params, condition.params);
|
||||
}
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
groupBy(clause: string): this {
|
||||
this.groupByClause = clause;
|
||||
return this;
|
||||
}
|
||||
|
||||
having(clause: string, params?: QueryParams): this {
|
||||
this.havingClauses.push(clause);
|
||||
if (params) {
|
||||
Object.assign(this.params, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
havingIf(condition: any, clause: string, params?: QueryParams): this {
|
||||
if (condition) {
|
||||
this.having(clause, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
orderBy(clause: string): this {
|
||||
this.orderByClause = clause;
|
||||
return this;
|
||||
}
|
||||
|
||||
limit(limit: number): this {
|
||||
this.limitClause = `LIMIT ${limit}`;
|
||||
return this;
|
||||
}
|
||||
|
||||
execute(): ReturnType<ClickhouseQueryFunction<void, TOutput>> {
|
||||
const { query, params } = this.build();
|
||||
|
||||
const queryFunction = this.reader.query({
|
||||
name: this.name,
|
||||
query,
|
||||
params: z.any(),
|
||||
schema: this.schema,
|
||||
settings: this.settings,
|
||||
});
|
||||
|
||||
return queryFunction(params);
|
||||
}
|
||||
|
||||
build(): { query: string; params: QueryParams } {
|
||||
let query = this.baseQuery;
|
||||
if (this.whereClauses.length > 0) {
|
||||
query += " WHERE " + this.whereClauses.join(" AND ");
|
||||
}
|
||||
if (this.groupByClause) {
|
||||
query += ` GROUP BY ${this.groupByClause}`;
|
||||
}
|
||||
if (this.havingClauses.length > 0) {
|
||||
query += " HAVING " + this.havingClauses.join(" AND ");
|
||||
}
|
||||
if (this.orderByClause) {
|
||||
query += ` ORDER BY ${this.orderByClause}`;
|
||||
}
|
||||
if (this.limitClause) {
|
||||
query += ` ${this.limitClause}`;
|
||||
}
|
||||
return { query, params: this.params };
|
||||
}
|
||||
}
|
||||
|
||||
export class ClickhouseQueryFastBuilder<TOutput extends Record<string, any>> {
|
||||
private name: string;
|
||||
private table: string;
|
||||
private columns: Array<string | ColumnExpression>;
|
||||
private reader: ClickhouseReader;
|
||||
private settings: ClickHouseSettings | undefined;
|
||||
private prewhereClauses: string[] = [];
|
||||
private whereClauses: string[] = [];
|
||||
private havingClauses: string[] = [];
|
||||
private params: QueryParams = {};
|
||||
private orderByClause: string | null = null;
|
||||
private limitClause: string | null = null;
|
||||
private groupByClause: string | null = null;
|
||||
|
||||
constructor(
|
||||
name: string,
|
||||
table: string,
|
||||
columns: Array<string | ColumnExpression>,
|
||||
reader: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
this.name = name;
|
||||
this.table = table;
|
||||
this.columns = columns;
|
||||
this.reader = reader;
|
||||
this.settings = settings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a PREWHERE clause - filters applied before reading columns.
|
||||
* Use for primary key columns (environment_id, start_time) to reduce I/O.
|
||||
*/
|
||||
prewhere(clause: string, params?: QueryParams): this {
|
||||
this.prewhereClauses.push(clause);
|
||||
if (params) {
|
||||
Object.assign(this.params, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
prewhereIf(condition: any, clause: string, params?: QueryParams): this {
|
||||
if (condition) {
|
||||
this.prewhere(clause, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
where(clause: string, params?: QueryParams): this {
|
||||
this.whereClauses.push(clause);
|
||||
if (params) {
|
||||
Object.assign(this.params, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
whereIf(condition: any, clause: string, params?: QueryParams): this {
|
||||
if (condition) {
|
||||
this.where(clause, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
whereOr(conditions: WhereCondition[]): this {
|
||||
if (conditions.length === 0) {
|
||||
return this;
|
||||
}
|
||||
const combinedClause = conditions.map((c) => `(${c.clause})`).join(" OR ");
|
||||
this.whereClauses.push(`(${combinedClause})`);
|
||||
for (const condition of conditions) {
|
||||
if (condition.params) {
|
||||
Object.assign(this.params, condition.params);
|
||||
}
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
groupBy(clause: string): this {
|
||||
this.groupByClause = clause;
|
||||
return this;
|
||||
}
|
||||
|
||||
having(clause: string, params?: QueryParams): this {
|
||||
this.havingClauses.push(clause);
|
||||
if (params) {
|
||||
Object.assign(this.params, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
havingIf(condition: any, clause: string, params?: QueryParams): this {
|
||||
if (condition) {
|
||||
this.having(clause, params);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
orderBy(clause: string): this {
|
||||
this.orderByClause = clause;
|
||||
return this;
|
||||
}
|
||||
|
||||
limit(limit: number): this {
|
||||
this.limitClause = `LIMIT ${limit}`;
|
||||
return this;
|
||||
}
|
||||
|
||||
execute(): ReturnType<ClickhouseQueryFunction<void, TOutput>> {
|
||||
const { query, params } = this.build();
|
||||
|
||||
const queryFunction = this.reader.queryFast<TOutput, Record<string, any>>({
|
||||
name: this.name,
|
||||
query,
|
||||
columns: this.columns,
|
||||
settings: this.settings,
|
||||
});
|
||||
|
||||
return queryFunction(params);
|
||||
}
|
||||
|
||||
/**
|
||||
* Like {@link execute} but streams rows instead of buffering them into an
|
||||
* array. Returns an async iterable so the whole result set is never resident
|
||||
* in memory at once.
|
||||
*/
|
||||
executeStream(): AsyncIterable<TOutput> {
|
||||
const { query, params } = this.build();
|
||||
|
||||
const streamFunction = this.reader.queryFastStream<TOutput, Record<string, any>>({
|
||||
name: this.name,
|
||||
query,
|
||||
columns: this.columns,
|
||||
settings: this.settings,
|
||||
});
|
||||
|
||||
return streamFunction(params);
|
||||
}
|
||||
|
||||
build(): { query: string; params: QueryParams } {
|
||||
let query = `SELECT ${this.buildColumns().join(", ")} FROM ${this.table}`;
|
||||
if (this.prewhereClauses.length > 0) {
|
||||
query += " PREWHERE " + this.prewhereClauses.join(" AND ");
|
||||
}
|
||||
if (this.whereClauses.length > 0) {
|
||||
query += " WHERE " + this.whereClauses.join(" AND ");
|
||||
}
|
||||
if (this.groupByClause) {
|
||||
query += ` GROUP BY ${this.groupByClause}`;
|
||||
}
|
||||
if (this.havingClauses.length > 0) {
|
||||
query += " HAVING " + this.havingClauses.join(" AND ");
|
||||
}
|
||||
if (this.orderByClause) {
|
||||
query += ` ORDER BY ${this.orderByClause}`;
|
||||
}
|
||||
if (this.limitClause) {
|
||||
query += ` ${this.limitClause}`;
|
||||
}
|
||||
return { query, params: this.params };
|
||||
}
|
||||
|
||||
buildColumns(): string[] {
|
||||
return this.columns.map((column) => {
|
||||
if (typeof column === "string") {
|
||||
return column;
|
||||
}
|
||||
return [column.expression, column.name].join(" AS ");
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
/**
|
||||
* TSQL Query Execution for ClickHouse
|
||||
*
|
||||
* This module provides a safe interface for executing TSQL queries against ClickHouse
|
||||
* with enforced WHERE clause conditions (tenant isolation + plan limits) and SQL injection protection.
|
||||
*/
|
||||
|
||||
import type { ClickHouseSettings } from "@clickhouse/client";
|
||||
import {
|
||||
compileTSQL,
|
||||
type OutputColumnMetadata,
|
||||
sanitizeErrorMessage,
|
||||
transformResults,
|
||||
type FieldMappings,
|
||||
type QuerySettings,
|
||||
type TableSchema,
|
||||
type TimeRange,
|
||||
type WhereClauseCondition,
|
||||
} from "@internal/tsql";
|
||||
import { Logger } from "@trigger.dev/core/logger";
|
||||
import { z } from "zod";
|
||||
import { QueryError } from "./errors.js";
|
||||
import type { ClickhouseReader, QueryStats } from "./types.js";
|
||||
|
||||
const logger = new Logger("tsql", "info");
|
||||
|
||||
export type { QueryStats };
|
||||
|
||||
export type { FieldMappings, QuerySettings, TableSchema, TimeRange, WhereClauseCondition };
|
||||
|
||||
/**
|
||||
* Options for executing a TSQL query
|
||||
*/
|
||||
export interface ExecuteTSQLOptions<TOut extends z.ZodSchema> {
|
||||
/** The name of the operation (for logging/tracing) */
|
||||
name: string;
|
||||
/** The TSQL query string to execute */
|
||||
query: string;
|
||||
/** The Zod schema for validating output rows */
|
||||
schema: TOut;
|
||||
/** Schema registry defining allowed tables and columns */
|
||||
tableSchema: TableSchema[];
|
||||
/**
|
||||
* REQUIRED: Conditions always applied at the table level.
|
||||
* Must include tenant columns (e.g., organization_id) for multi-tenant tables.
|
||||
* Applied to every table reference including subqueries, CTEs, and JOINs.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* {
|
||||
* // Tenant isolation
|
||||
* organization_id: { op: "eq", value: "org_123" },
|
||||
* project_id: { op: "eq", value: "proj_456" },
|
||||
* environment_id: { op: "eq", value: "env_789" },
|
||||
* // Plan-based time limit
|
||||
* triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
enforcedWhereClause: Record<string, WhereClauseCondition | undefined>;
|
||||
/** Optional ClickHouse query settings */
|
||||
clickhouseSettings?: ClickHouseSettings;
|
||||
/** Optional TSQL query settings (maxRows, timezone, etc.) */
|
||||
querySettings?: Partial<QuerySettings>;
|
||||
/**
|
||||
* Whether to transform result values using the schema's valueMap
|
||||
* When enabled, internal ClickHouse values (e.g., 'COMPLETED_SUCCESSFULLY')
|
||||
* are converted to user-friendly display names (e.g., 'Completed')
|
||||
* @default true
|
||||
*/
|
||||
transformValues?: boolean;
|
||||
/**
|
||||
* Runtime field mappings for dynamic value translation.
|
||||
* Maps internal ClickHouse values to external user-facing values.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* {
|
||||
* project: { "cm12345": "my-project-ref" },
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
fieldMappings?: FieldMappings;
|
||||
/**
|
||||
* Run EXPLAIN instead of executing the query.
|
||||
* Returns the ClickHouse execution plan with index information.
|
||||
* Should only be used by admins for debugging query performance.
|
||||
* @default false
|
||||
*/
|
||||
explain?: boolean;
|
||||
/**
|
||||
* Fallback WHERE conditions to apply when the user hasn't filtered on a column.
|
||||
* Key is the column name, value is the fallback condition.
|
||||
* These are applied at the AST level (top-level query only).
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Apply triggered_at >= 7 days ago if user doesn't filter on triggered_at
|
||||
* whereClauseFallback: {
|
||||
* triggered_at: { op: 'gte', value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
whereClauseFallback?: Record<string, WhereClauseCondition>;
|
||||
/**
|
||||
* Time range for `timeBucket()` interval calculation.
|
||||
* When provided, `timeBucket()` uses this to determine the appropriate bucket size
|
||||
* based on the span of the time range.
|
||||
*/
|
||||
timeRange?: TimeRange;
|
||||
}
|
||||
|
||||
/**
|
||||
* Successful result from TSQL query execution
|
||||
*/
|
||||
export interface TSQLQuerySuccess<T> {
|
||||
rows: T[];
|
||||
columns: OutputColumnMetadata[];
|
||||
stats: QueryStats;
|
||||
/**
|
||||
* Columns that were hidden when SELECT * was used.
|
||||
* Only populated when SELECT * is transformed to core columns only.
|
||||
*/
|
||||
hiddenColumns?: string[];
|
||||
/**
|
||||
* Whether the result count equals the maxRows limit.
|
||||
* When true, the results may be truncated and more rows may exist.
|
||||
*/
|
||||
reachedMaxRows: boolean;
|
||||
/**
|
||||
* The raw EXPLAIN output from ClickHouse.
|
||||
* Only populated when `explain: true` is passed.
|
||||
*/
|
||||
explainOutput?: string;
|
||||
/**
|
||||
* The generated ClickHouse SQL query.
|
||||
* Only populated when `explain: true` is passed.
|
||||
*/
|
||||
generatedSql?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result type for TSQL query execution
|
||||
*/
|
||||
export type TSQLQueryResult<T> = [QueryError, null] | [null, TSQLQuerySuccess<T>];
|
||||
|
||||
/**
|
||||
* Execute a TSQL query against ClickHouse
|
||||
*
|
||||
* This function:
|
||||
* 1. Compiles the TSQL query to ClickHouse SQL (parse, validate, inject enforced WHERE clauses)
|
||||
* 2. Executes the query and returns validated results
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const [error, rows] = await executeTSQL(reader, {
|
||||
* name: "get_task_runs",
|
||||
* query: "SELECT id, status FROM task_runs WHERE status = 'completed' ORDER BY created_at DESC LIMIT 100",
|
||||
* schema: z.object({ id: z.string(), status: z.string() }),
|
||||
* tableSchema: [taskRunsSchema],
|
||||
* enforcedWhereClause: {
|
||||
* organization_id: { op: "eq", value: "org_123" },
|
||||
* project_id: { op: "eq", value: "proj_456" },
|
||||
* environment_id: { op: "eq", value: "env_789" },
|
||||
* },
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export async function executeTSQL<TOut extends z.ZodSchema>(
|
||||
reader: ClickhouseReader,
|
||||
options: ExecuteTSQLOptions<TOut>
|
||||
): Promise<TSQLQueryResult<z.output<TOut>>> {
|
||||
const shouldTransformValues = options.transformValues ?? true;
|
||||
const isExplain = options.explain ?? false;
|
||||
const maxRows = options.querySettings?.maxRows;
|
||||
|
||||
let generatedSql: string | undefined;
|
||||
let generatedParams: Record<string, unknown> | undefined;
|
||||
|
||||
try {
|
||||
// 1. Compile the TSQL query to ClickHouse SQL
|
||||
// Pass maxRows + 1 to fetch one extra row for overflow detection
|
||||
const compiledSettings =
|
||||
maxRows !== undefined
|
||||
? { ...options.querySettings, maxRows: maxRows + 1 }
|
||||
: options.querySettings;
|
||||
|
||||
const { sql, params, columns, hiddenColumns } = compileTSQL(options.query, {
|
||||
tableSchema: options.tableSchema,
|
||||
enforcedWhereClause: options.enforcedWhereClause,
|
||||
settings: compiledSettings,
|
||||
fieldMappings: options.fieldMappings,
|
||||
whereClauseFallback: options.whereClauseFallback,
|
||||
timeRange: options.timeRange,
|
||||
});
|
||||
|
||||
generatedSql = sql;
|
||||
generatedParams = params;
|
||||
|
||||
// 2. Execute the query (or EXPLAIN) with stats
|
||||
const queryToExecute = isExplain ? `EXPLAIN indexes = 1 ${sql}` : sql;
|
||||
|
||||
const queryFn = reader.queryWithStats({
|
||||
name: isExplain ? `${options.name}-explain` : options.name,
|
||||
query: queryToExecute,
|
||||
params: z.record(z.any()),
|
||||
// EXPLAIN returns rows with an 'explain' column
|
||||
schema: isExplain ? z.object({ explain: z.string() }) : options.schema,
|
||||
settings: options.clickhouseSettings,
|
||||
});
|
||||
|
||||
const [error, result] = await queryFn(params);
|
||||
|
||||
if (error) {
|
||||
// Sanitize error message to show TSQL names instead of ClickHouse internals
|
||||
const sanitizedMessage = sanitizeErrorMessage(error.message, options.tableSchema);
|
||||
return [new QueryError(sanitizedMessage, { query: options.query }), null];
|
||||
}
|
||||
|
||||
const { rows, stats } = result;
|
||||
|
||||
// Handle EXPLAIN mode - run multiple explain types and combine outputs
|
||||
if (isExplain) {
|
||||
const explainRows = rows as Array<{ explain: string }>;
|
||||
const indexesOutput = explainRows.map((r) => r.explain).join("\n");
|
||||
|
||||
// Run additional explain queries for more comprehensive output
|
||||
const explainTypes = [
|
||||
{ name: "ESTIMATE", query: `EXPLAIN ESTIMATE ${sql}` },
|
||||
{ name: "PIPELINE", query: `EXPLAIN PIPELINE ${sql}` },
|
||||
];
|
||||
|
||||
const additionalOutputs: string[] = [];
|
||||
|
||||
for (const explainType of explainTypes) {
|
||||
try {
|
||||
const additionalQueryFn = reader.queryWithStats({
|
||||
name: `${options.name}-explain-${explainType.name.toLowerCase()}`,
|
||||
query: explainType.query,
|
||||
params: z.record(z.any()),
|
||||
schema: z.object({ explain: z.string() }),
|
||||
settings: options.clickhouseSettings,
|
||||
});
|
||||
|
||||
const [additionalError, additionalResult] = await additionalQueryFn(params);
|
||||
|
||||
if (!additionalError && additionalResult) {
|
||||
const additionalRows = additionalResult.rows as Array<{ explain: string }>;
|
||||
const output = additionalRows.map((r) => r.explain).join("\n");
|
||||
additionalOutputs.push(`── ${explainType.name} ──\n${output}`);
|
||||
}
|
||||
} catch {
|
||||
// Ignore errors from additional explain queries
|
||||
}
|
||||
}
|
||||
|
||||
// Combine all explain outputs
|
||||
const combinedOutput = ["── INDEXES ──", indexesOutput, "", ...additionalOutputs].join("\n");
|
||||
|
||||
return [
|
||||
null,
|
||||
{
|
||||
rows: [] as z.output<TOut>[],
|
||||
columns: [],
|
||||
stats,
|
||||
hiddenColumns,
|
||||
reachedMaxRows: false,
|
||||
explainOutput: combinedOutput,
|
||||
generatedSql,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// Determine if we exceeded maxRows (we fetched maxRows + 1 to detect overflow)
|
||||
const reachedMaxRows = maxRows !== undefined && rows !== undefined && rows.length > maxRows;
|
||||
|
||||
// Remove the overflow row if we got one (pop is O(1), slice would be O(n))
|
||||
const finalRows = rows ?? [];
|
||||
if (reachedMaxRows) {
|
||||
finalRows.pop();
|
||||
}
|
||||
|
||||
// Build the result, including hiddenColumns if present
|
||||
const baseResult = { columns, stats, hiddenColumns, reachedMaxRows };
|
||||
|
||||
// 3. Transform result values if enabled
|
||||
if (shouldTransformValues && finalRows.length > 0) {
|
||||
const transformedRows = transformResults(
|
||||
finalRows as Record<string, unknown>[],
|
||||
options.tableSchema,
|
||||
{ fieldMappings: options.fieldMappings }
|
||||
);
|
||||
return [null, { rows: transformedRows as z.output<TOut>[], ...baseResult }];
|
||||
}
|
||||
|
||||
return [null, { rows: finalRows as z.output<TOut>[], ...baseResult }];
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : "Unknown error";
|
||||
|
||||
// Log TSQL compilation or unexpected errors (with original message for debugging)
|
||||
logger.error("[TSQL] Query error", {
|
||||
name: options.name,
|
||||
error: errorMessage,
|
||||
tsql: options.query,
|
||||
generatedSql: generatedSql ?? "(compilation failed)",
|
||||
generatedParams: generatedParams ?? {},
|
||||
});
|
||||
|
||||
// Sanitize error message to show TSQL names instead of ClickHouse internals
|
||||
const sanitizedMessage = sanitizeErrorMessage(errorMessage, options.tableSchema);
|
||||
|
||||
if (error instanceof Error) {
|
||||
return [new QueryError(sanitizedMessage, { query: options.query }), null];
|
||||
}
|
||||
return [new QueryError("Unknown error executing TSQL query", { query: options.query }), null];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a reusable TSQL query executor bound to specific table schemas
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const tsqlExecutor = createTSQLExecutor(reader, [taskRunsSchema, taskEventsSchema]);
|
||||
*
|
||||
* const [error, rows] = await tsqlExecutor.execute({
|
||||
* name: "get_task_runs",
|
||||
* query: "SELECT * FROM task_runs LIMIT 10",
|
||||
* schema: taskRunRowSchema,
|
||||
* enforcedWhereClause: {
|
||||
* organization_id: { op: "eq", value: "org_123" },
|
||||
* project_id: { op: "eq", value: "proj_456" },
|
||||
* environment_id: { op: "eq", value: "env_789" },
|
||||
* },
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export function createTSQLExecutor(reader: ClickhouseReader, tableSchema: TableSchema[]) {
|
||||
return {
|
||||
execute: <TOut extends z.ZodSchema>(
|
||||
options: Omit<ExecuteTSQLOptions<TOut>, "tableSchema">
|
||||
): Promise<TSQLQueryResult<z.output<TOut>>> => {
|
||||
return executeTSQL(reader, { ...options, tableSchema });
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
import type { Result } from "@trigger.dev/core/v3";
|
||||
import type { z } from "zod";
|
||||
import type { InsertError, QueryError } from "./errors.js";
|
||||
import {
|
||||
type ClickHouseSettings,
|
||||
type BaseQueryParams,
|
||||
type InsertResult,
|
||||
} from "@clickhouse/client";
|
||||
import type { ClickhouseQueryBuilder, ClickhouseQueryFastBuilder } from "./queryBuilder.js";
|
||||
|
||||
export type ClickhouseQueryFunction<TInput, TOutput> = (
|
||||
params: TInput,
|
||||
options?: {
|
||||
attributes?: Record<string, string | number | boolean>;
|
||||
params?: BaseQueryParams;
|
||||
}
|
||||
) => Promise<Result<TOutput[], QueryError>>;
|
||||
|
||||
/**
|
||||
* Like {@link ClickhouseQueryFunction} but yields rows as they stream off the
|
||||
* socket instead of buffering them into an array. The whole result set is never
|
||||
* held in memory at once. Errors are thrown (the async iterable rejects) rather
|
||||
* than returned as a Result tuple.
|
||||
*/
|
||||
export type ClickhouseQueryStreamFunction<TInput, TOutput> = (
|
||||
params: TInput,
|
||||
options?: {
|
||||
attributes?: Record<string, string | number | boolean>;
|
||||
params?: BaseQueryParams;
|
||||
}
|
||||
) => AsyncIterable<TOutput>;
|
||||
|
||||
/**
|
||||
* Query statistics returned by ClickHouse
|
||||
*/
|
||||
export interface QueryStats {
|
||||
read_rows: string;
|
||||
read_bytes: string;
|
||||
written_rows: string;
|
||||
written_bytes: string;
|
||||
total_rows_to_read: string;
|
||||
result_rows: string;
|
||||
result_bytes: string;
|
||||
elapsed_ns: string;
|
||||
byte_seconds: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result type for queries that include stats
|
||||
*/
|
||||
export interface QueryResultWithStats<TOutput> {
|
||||
rows: TOutput[];
|
||||
stats: QueryStats;
|
||||
}
|
||||
|
||||
export type ClickhouseQueryWithStatsFunction<TInput, TOutput> = (
|
||||
params: TInput,
|
||||
options?: {
|
||||
attributes?: Record<string, string | number | boolean>;
|
||||
params?: BaseQueryParams;
|
||||
}
|
||||
) => Promise<Result<QueryResultWithStats<TOutput>, QueryError>>;
|
||||
|
||||
export type ClickhouseQueryBuilderFunction<TOutput> = (options?: {
|
||||
settings?: ClickHouseSettings;
|
||||
}) => ClickhouseQueryBuilder<TOutput>;
|
||||
|
||||
export type ClickhouseQueryBuilderFastFunction<TOutput extends Record<string, any>> = (options?: {
|
||||
settings?: ClickHouseSettings;
|
||||
}) => ClickhouseQueryFastBuilder<TOutput>;
|
||||
|
||||
export type ColumnExpression = {
|
||||
name: string;
|
||||
expression: string;
|
||||
};
|
||||
|
||||
export interface ClickhouseReader {
|
||||
query<TIn extends z.ZodSchema<any>, TOut extends z.ZodSchema<any>>(req: {
|
||||
/**
|
||||
* The name of the operation.
|
||||
* This will be used to identify the operation in the span.
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* The SQL query to run.
|
||||
* Use {paramName: Type} to define parameters
|
||||
* Example: `SELECT * FROM table WHERE id = {id: String}`
|
||||
*/
|
||||
query: string;
|
||||
/**
|
||||
* The schema of the parameters
|
||||
* Example: z.object({ id: z.string() })
|
||||
*/
|
||||
params?: TIn;
|
||||
/**
|
||||
* The schema of the output of each row
|
||||
* Example: z.object({ id: z.string() })
|
||||
*/
|
||||
schema: TOut;
|
||||
/**
|
||||
* The settings to use for the query.
|
||||
* These will be merged with the default settings.
|
||||
*/
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseQueryFunction<z.input<TIn>, z.output<TOut>>;
|
||||
|
||||
/**
|
||||
* Execute a query and return both rows and query statistics.
|
||||
* Same as `query` but includes ClickHouse query stats in the result.
|
||||
*/
|
||||
queryWithStats<TIn extends z.ZodSchema<any>, TOut extends z.ZodSchema<any>>(req: {
|
||||
/**
|
||||
* The name of the operation.
|
||||
* This will be used to identify the operation in the span.
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* The SQL query to run.
|
||||
* Use {paramName: Type} to define parameters
|
||||
* Example: `SELECT * FROM table WHERE id = {id: String}`
|
||||
*/
|
||||
query: string;
|
||||
/**
|
||||
* The schema of the parameters
|
||||
* Example: z.object({ id: z.string() })
|
||||
*/
|
||||
params?: TIn;
|
||||
/**
|
||||
* The schema of the output of each row
|
||||
* Example: z.object({ id: z.string() })
|
||||
*/
|
||||
schema: TOut;
|
||||
/**
|
||||
* The settings to use for the query.
|
||||
* These will be merged with the default settings.
|
||||
*/
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseQueryWithStatsFunction<z.input<TIn>, z.output<TOut>>;
|
||||
|
||||
queryFast<TOut extends Record<string, any>, TParams extends Record<string, any>>(req: {
|
||||
/**
|
||||
* The name of the operation.
|
||||
* This will be used to identify the operation in the span.
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* The SQL query to run.
|
||||
* Use {paramName: Type} to define parameters
|
||||
* Example: `SELECT * FROM table WHERE id = {id: String}`
|
||||
*/
|
||||
query: string;
|
||||
|
||||
/**
|
||||
* The columns returned by the query, in the order
|
||||
*
|
||||
* @example ["run_id", "created_at", "updated_at"]
|
||||
*/
|
||||
columns: Array<string | ColumnExpression>;
|
||||
/**
|
||||
* The settings to use for the query.
|
||||
* These will be merged with the default settings.
|
||||
*/
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseQueryFunction<TParams, TOut>;
|
||||
|
||||
/**
|
||||
* Like {@link queryFast} but streams rows instead of buffering them. Returns an
|
||||
* async iterable so the caller can process arbitrarily large result sets with
|
||||
* bounded memory.
|
||||
*/
|
||||
queryFastStream<TOut extends Record<string, any>, TParams extends Record<string, any>>(req: {
|
||||
name: string;
|
||||
query: string;
|
||||
columns: Array<string | ColumnExpression>;
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseQueryStreamFunction<TParams, TOut>;
|
||||
|
||||
queryBuilder<TOut extends z.ZodSchema<any>>(req: {
|
||||
/**
|
||||
* The name of the operation.
|
||||
* This will be used to identify the operation in the span.
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* The initial select clause
|
||||
*
|
||||
* @example SELECT run_id from trigger_dev.task_runs_v1
|
||||
*/
|
||||
baseQuery: string;
|
||||
/**
|
||||
* The schema of the output of each row
|
||||
* Example: z.object({ id: z.string() })
|
||||
*/
|
||||
schema: TOut;
|
||||
/**
|
||||
* The settings to use for the query.
|
||||
* These will be merged with the default settings.
|
||||
*/
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseQueryBuilderFunction<z.input<TOut>>;
|
||||
|
||||
queryBuilderFast<TOut extends Record<string, any>>(req: {
|
||||
/**
|
||||
* The name of the operation.
|
||||
* This will be used to identify the operation in the span.
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* The table to query
|
||||
*
|
||||
* @example trigger_dev.task_runs_v1
|
||||
*/
|
||||
table: string;
|
||||
/**
|
||||
* The columns to query
|
||||
*
|
||||
* @example ["run_id", "created_at", "updated_at"]
|
||||
*/
|
||||
columns: Array<string | ColumnExpression>;
|
||||
/**
|
||||
* The settings to use for the query.
|
||||
* These will be merged with the default settings.
|
||||
*/
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseQueryBuilderFastFunction<TOut>;
|
||||
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export type ClickhouseInsertFunction<TInput> = (
|
||||
events: TInput | TInput[],
|
||||
options?: {
|
||||
attributes?: Record<string, string | number | boolean>;
|
||||
params?: BaseQueryParams;
|
||||
}
|
||||
) => Promise<Result<InsertResult, InsertError>>;
|
||||
|
||||
export interface ClickhouseWriter {
|
||||
insert<TSchema extends z.ZodSchema<any>>(req: {
|
||||
name: string;
|
||||
table: string;
|
||||
schema: TSchema;
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseInsertFunction<z.input<TSchema>>;
|
||||
|
||||
insertUnsafe<TRecord extends Record<string, any>>(req: {
|
||||
name: string;
|
||||
table: string;
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseInsertFunction<TRecord>;
|
||||
|
||||
insertCompact<TRecord extends Record<string, any>>(req: {
|
||||
name: string;
|
||||
table: string;
|
||||
columns: readonly string[];
|
||||
toArray: (record: TRecord) => any[];
|
||||
settings?: ClickHouseSettings;
|
||||
}): ClickhouseInsertFunction<TRecord>;
|
||||
|
||||
insertCompactRaw(req: {
|
||||
name: string;
|
||||
table: string;
|
||||
columns: readonly string[];
|
||||
settings?: ClickHouseSettings;
|
||||
}): (
|
||||
events: readonly any[][] | any[],
|
||||
options?: {
|
||||
attributes?: Record<string, string | number | boolean>;
|
||||
params?: BaseQueryParams;
|
||||
}
|
||||
) => Promise<Result<InsertResult, InsertError>>;
|
||||
|
||||
close(): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,461 @@
|
||||
import type { ClickHouseSettings } from "@clickhouse/client";
|
||||
import { z } from "zod";
|
||||
import type { ClickhouseReader } from "./client/types.js";
|
||||
import { ClickhouseQueryBuilder } from "./client/queryBuilder.js";
|
||||
|
||||
export const ErrorGroupsListQueryResult = z.object({
|
||||
error_fingerprint: z.string(),
|
||||
task_identifier: z.string(),
|
||||
error_type: z.string(),
|
||||
error_message: z.string(),
|
||||
first_seen: z.string(),
|
||||
last_seen: z.string(),
|
||||
occurrence_count: z.number(),
|
||||
sample_run_id: z.string(),
|
||||
sample_friendly_id: z.string(),
|
||||
});
|
||||
|
||||
export type ErrorGroupsListQueryResult = z.infer<typeof ErrorGroupsListQueryResult>;
|
||||
|
||||
/**
|
||||
* Gets a query builder for listing error groups from the pre-aggregated errors_v1 table.
|
||||
* Allows flexible filtering and pagination.
|
||||
*/
|
||||
export function getErrorGroupsListQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilder({
|
||||
name: "getErrorGroupsList",
|
||||
baseQuery: `
|
||||
SELECT
|
||||
error_fingerprint,
|
||||
task_identifier,
|
||||
any(error_type) as error_type,
|
||||
any(error_message) as error_message,
|
||||
toString(toUnixTimestamp64Milli(min(first_seen))) as first_seen,
|
||||
toString(toUnixTimestamp64Milli(max(last_seen))) as last_seen,
|
||||
toUInt64(sumMerge(occurrence_count)) as occurrence_count,
|
||||
anyMerge(sample_run_id) as sample_run_id,
|
||||
anyMerge(sample_friendly_id) as sample_friendly_id
|
||||
FROM trigger_dev.errors_v1
|
||||
`,
|
||||
schema: ErrorGroupsListQueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const ErrorGroupQueryResult = z.object({
|
||||
error_fingerprint: z.string(),
|
||||
task_identifier: z.string(),
|
||||
error_type: z.string(),
|
||||
error_message: z.string(),
|
||||
first_seen: z.string(),
|
||||
last_seen: z.string(),
|
||||
occurrence_count: z.number(),
|
||||
sample_run_id: z.string(),
|
||||
sample_friendly_id: z.string(),
|
||||
});
|
||||
|
||||
export type ErrorGroupQueryResult = z.infer<typeof ErrorGroupQueryResult>;
|
||||
|
||||
export const ErrorGroupQueryParams = z.object({
|
||||
organizationId: z.string(),
|
||||
projectId: z.string(),
|
||||
environmentId: z.string(),
|
||||
days: z.number().int().default(30),
|
||||
limit: z.number().int().default(50),
|
||||
offset: z.number().int().default(0),
|
||||
});
|
||||
|
||||
export type ErrorGroupQueryParams = z.infer<typeof ErrorGroupQueryParams>;
|
||||
|
||||
/**
|
||||
* Gets error groups from the pre-aggregated errors_v1 table.
|
||||
* Much faster than on-the-fly aggregation.
|
||||
*/
|
||||
export function getErrorGroups(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.query({
|
||||
name: "getErrorGroups",
|
||||
query: `
|
||||
SELECT
|
||||
error_fingerprint,
|
||||
task_identifier,
|
||||
any(error_type) as error_type,
|
||||
any(error_message) as error_message,
|
||||
toString(toUnixTimestamp64Milli(min(first_seen))) as first_seen,
|
||||
toString(toUnixTimestamp64Milli(max(last_seen))) as last_seen,
|
||||
toUInt64(sumMerge(occurrence_count)) as occurrence_count,
|
||||
anyMerge(sample_run_id) as sample_run_id,
|
||||
anyMerge(sample_friendly_id) as sample_friendly_id
|
||||
FROM trigger_dev.errors_v1
|
||||
WHERE
|
||||
organization_id = {organizationId: String}
|
||||
AND project_id = {projectId: String}
|
||||
AND environment_id = {environmentId: String}
|
||||
GROUP BY error_fingerprint, task_identifier
|
||||
HAVING toInt64(last_seen) >= toInt64(toUnixTimestamp(now() - INTERVAL {days: Int64} DAY)) * 1000
|
||||
ORDER BY toInt64(last_seen) DESC
|
||||
LIMIT {limit: Int64}
|
||||
OFFSET {offset: Int64}
|
||||
`,
|
||||
schema: ErrorGroupQueryResult,
|
||||
params: ErrorGroupQueryParams,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const ErrorInstanceQueryResult = z.object({
|
||||
run_id: z.string(),
|
||||
friendly_id: z.string(),
|
||||
task_identifier: z.string(),
|
||||
created_at: z.string(),
|
||||
status: z.string(),
|
||||
error_text: z.string(),
|
||||
trace_id: z.string(),
|
||||
task_version: z.string(),
|
||||
});
|
||||
|
||||
export type ErrorInstanceQueryResult = z.infer<typeof ErrorInstanceQueryResult>;
|
||||
|
||||
export const ErrorInstanceQueryParams = z.object({
|
||||
organizationId: z.string(),
|
||||
projectId: z.string(),
|
||||
environmentId: z.string(),
|
||||
errorFingerprint: z.string(),
|
||||
limit: z.number().int().default(50),
|
||||
offset: z.number().int().default(0),
|
||||
});
|
||||
|
||||
export type ErrorInstanceQueryParams = z.infer<typeof ErrorInstanceQueryParams>;
|
||||
|
||||
export const ErrorHourlyOccurrencesQueryResult = z.object({
|
||||
error_fingerprint: z.string(),
|
||||
hour_epoch: z.number(),
|
||||
count: z.number(),
|
||||
});
|
||||
|
||||
export type ErrorHourlyOccurrencesQueryResult = z.infer<typeof ErrorHourlyOccurrencesQueryResult>;
|
||||
|
||||
export const ErrorHourlyOccurrencesQueryParams = z.object({
|
||||
organizationId: z.string(),
|
||||
projectId: z.string(),
|
||||
environmentId: z.string(),
|
||||
fingerprints: z.array(z.string()),
|
||||
hours: z.number().int().default(24),
|
||||
});
|
||||
|
||||
export type ErrorHourlyOccurrencesQueryParams = z.infer<typeof ErrorHourlyOccurrencesQueryParams>;
|
||||
|
||||
/**
|
||||
* Gets hourly occurrence counts for specific error fingerprints over the past N hours.
|
||||
* Queries task_runs_v2 directly, grouped by fingerprint and hour.
|
||||
*/
|
||||
export function getErrorHourlyOccurrences(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.query({
|
||||
name: "getErrorHourlyOccurrences",
|
||||
query: `
|
||||
SELECT
|
||||
error_fingerprint,
|
||||
toUnixTimestamp(toStartOfHour(created_at)) as hour_epoch,
|
||||
count() as count
|
||||
FROM trigger_dev.task_runs_v2 FINAL
|
||||
WHERE
|
||||
organization_id = {organizationId: String}
|
||||
AND project_id = {projectId: String}
|
||||
AND environment_id = {environmentId: String}
|
||||
AND created_at >= now() - INTERVAL {hours: Int64} HOUR
|
||||
AND error_fingerprint IN {fingerprints: Array(String)}
|
||||
AND status IN ('SYSTEM_FAILURE', 'CRASHED', 'INTERRUPTED', 'COMPLETED_WITH_ERRORS', 'TIMED_OUT')
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
error_fingerprint,
|
||||
hour_epoch
|
||||
ORDER BY
|
||||
error_fingerprint ASC,
|
||||
hour_epoch ASC
|
||||
`,
|
||||
schema: ErrorHourlyOccurrencesQueryResult,
|
||||
params: ErrorHourlyOccurrencesQueryParams,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets individual run instances for a specific error fingerprint.
|
||||
*/
|
||||
export function getErrorInstances(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.query({
|
||||
name: "getErrorInstances",
|
||||
query: `
|
||||
SELECT
|
||||
run_id,
|
||||
friendly_id,
|
||||
task_identifier,
|
||||
toString(created_at) as created_at,
|
||||
status,
|
||||
error_text,
|
||||
trace_id,
|
||||
task_version
|
||||
FROM trigger_dev.task_runs_v2 FINAL
|
||||
WHERE
|
||||
organization_id = {organizationId: String}
|
||||
AND project_id = {projectId: String}
|
||||
AND environment_id = {environmentId: String}
|
||||
AND error_fingerprint = {errorFingerprint: String}
|
||||
AND _is_deleted = 0
|
||||
ORDER BY created_at DESC
|
||||
LIMIT {limit: Int64}
|
||||
OFFSET {offset: Int64}
|
||||
`,
|
||||
schema: ErrorInstanceQueryResult,
|
||||
params: ErrorInstanceQueryParams,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Affected versions – distinct task_version from error_occurrences_v1
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const ErrorAffectedVersionsQueryResult = z.object({
|
||||
task_version: z.string(),
|
||||
});
|
||||
|
||||
export type ErrorAffectedVersionsQueryResult = z.infer<typeof ErrorAffectedVersionsQueryResult>;
|
||||
|
||||
/**
|
||||
* Query builder for fetching distinct task_version values for an error fingerprint
|
||||
* from the error_occurrences_v1 SummingMergeTree table.
|
||||
* task_version is part of the ORDER BY key, so this is efficient.
|
||||
*/
|
||||
export function getErrorAffectedVersionsQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilder({
|
||||
name: "getErrorAffectedVersions",
|
||||
baseQuery: `
|
||||
SELECT DISTINCT task_version
|
||||
FROM trigger_dev.error_occurrences_v1
|
||||
`,
|
||||
schema: ErrorAffectedVersionsQueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// error_occurrences_v1 – per-minute bucketed error counts
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const ErrorOccurrencesListQueryResult = z.object({
|
||||
error_fingerprint: z.string(),
|
||||
task_identifier: z.string(),
|
||||
error_type: z.string(),
|
||||
error_message: z.string(),
|
||||
occurrence_count: z.number(),
|
||||
});
|
||||
|
||||
export type ErrorOccurrencesListQueryResult = z.infer<typeof ErrorOccurrencesListQueryResult>;
|
||||
|
||||
/**
|
||||
* Query builder for listing error groups from the per-minute error_occurrences_v1 table.
|
||||
* Time filtering is done via WHERE on the `minute` column, giving precise time-scoped counts.
|
||||
*/
|
||||
export function getErrorOccurrencesListQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilder({
|
||||
name: "getErrorOccurrencesList",
|
||||
baseQuery: `
|
||||
SELECT
|
||||
error_fingerprint,
|
||||
task_identifier,
|
||||
any(error_type) as error_type,
|
||||
any(error_message) as error_message,
|
||||
sum(count) as occurrence_count
|
||||
FROM trigger_dev.error_occurrences_v1
|
||||
`,
|
||||
schema: ErrorOccurrencesListQueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const ErrorOccurrencesBucketQueryResult = z.object({
|
||||
error_fingerprint: z.string(),
|
||||
bucket_epoch: z.number(),
|
||||
count: z.number(),
|
||||
});
|
||||
|
||||
export type ErrorOccurrencesBucketQueryResult = z.infer<typeof ErrorOccurrencesBucketQueryResult>;
|
||||
|
||||
/**
|
||||
* Creates a query builder for bucketed error occurrence counts.
|
||||
* The `intervalExpr` is a ClickHouse INTERVAL literal (e.g. "INTERVAL 1 HOUR").
|
||||
* Returns a builder directly since the base query varies with each granularity.
|
||||
*/
|
||||
export function createErrorOccurrencesQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
intervalExpr: string,
|
||||
settings?: ClickHouseSettings
|
||||
): ClickhouseQueryBuilder<ErrorOccurrencesBucketQueryResult> {
|
||||
return new ClickhouseQueryBuilder(
|
||||
"getErrorOccurrencesBucketed",
|
||||
`
|
||||
SELECT
|
||||
error_fingerprint,
|
||||
toUnixTimestamp(toStartOfInterval(minute, ${intervalExpr})) as bucket_epoch,
|
||||
sum(count) as count
|
||||
FROM trigger_dev.error_occurrences_v1
|
||||
`,
|
||||
ch,
|
||||
ErrorOccurrencesBucketQueryResult,
|
||||
settings
|
||||
);
|
||||
}
|
||||
|
||||
export const ErrorOccurrencesByVersionQueryResult = z.object({
|
||||
error_fingerprint: z.string(),
|
||||
task_version: z.string(),
|
||||
bucket_epoch: z.number(),
|
||||
count: z.number(),
|
||||
});
|
||||
|
||||
export type ErrorOccurrencesByVersionQueryResult = z.infer<
|
||||
typeof ErrorOccurrencesByVersionQueryResult
|
||||
>;
|
||||
|
||||
/**
|
||||
* Creates a query builder for bucketed error occurrence counts grouped by task_version.
|
||||
* Used for stacked-by-version activity charts on the error detail page.
|
||||
*/
|
||||
export function createErrorOccurrencesByVersionQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
intervalExpr: string,
|
||||
settings?: ClickHouseSettings
|
||||
): ClickhouseQueryBuilder<ErrorOccurrencesByVersionQueryResult> {
|
||||
return new ClickhouseQueryBuilder(
|
||||
"getErrorOccurrencesByVersion",
|
||||
`
|
||||
SELECT
|
||||
error_fingerprint,
|
||||
task_version,
|
||||
toUnixTimestamp(toStartOfInterval(minute, ${intervalExpr})) as bucket_epoch,
|
||||
sum(count) as count
|
||||
FROM trigger_dev.error_occurrences_v1
|
||||
`,
|
||||
ch,
|
||||
ErrorOccurrencesByVersionQueryResult,
|
||||
settings
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Alert evaluator – active errors since a timestamp
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const ActiveErrorsSinceQueryResult = z.object({
|
||||
environment_id: z.string(),
|
||||
task_identifier: z.string(),
|
||||
error_fingerprint: z.string(),
|
||||
error_type: z.string(),
|
||||
error_message: z.string(),
|
||||
sample_stack_trace: z.string(),
|
||||
first_seen: z.string(),
|
||||
last_seen: z.string(),
|
||||
occurrence_count: z.number(),
|
||||
});
|
||||
|
||||
export type ActiveErrorsSinceQueryResult = z.infer<typeof ActiveErrorsSinceQueryResult>;
|
||||
|
||||
/**
|
||||
* Query builder for fetching all errors active since a given timestamp.
|
||||
* Returns errors with last_seen > scheduledAt, grouped by env/task/fingerprint.
|
||||
* Used by the error alert evaluator to find new issues, regressions, and un-ignored errors.
|
||||
*/
|
||||
export function getActiveErrorsSinceQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilder({
|
||||
name: "getActiveErrorsSince",
|
||||
baseQuery: `
|
||||
SELECT
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
any(error_type) as error_type,
|
||||
any(error_message) as error_message,
|
||||
any(sample_stack_trace) as sample_stack_trace,
|
||||
toString(toUnixTimestamp64Milli(min(first_seen))) as first_seen,
|
||||
toString(toUnixTimestamp64Milli(max(last_seen))) as last_seen,
|
||||
toUInt64(sumMerge(occurrence_count)) as occurrence_count
|
||||
FROM trigger_dev.errors_v1
|
||||
`,
|
||||
schema: ActiveErrorsSinceQueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const OccurrenceCountsSinceQueryResult = z.object({
|
||||
environment_id: z.string(),
|
||||
task_identifier: z.string(),
|
||||
error_fingerprint: z.string(),
|
||||
occurrences_since: z.number(),
|
||||
});
|
||||
|
||||
export type OccurrenceCountsSinceQueryResult = z.infer<typeof OccurrenceCountsSinceQueryResult>;
|
||||
|
||||
/**
|
||||
* Query builder for occurrence counts since a given timestamp, grouped by error.
|
||||
* Used by the alert evaluator to check ignore thresholds.
|
||||
*/
|
||||
export function getOccurrenceCountsSinceQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilder({
|
||||
name: "getOccurrenceCountsSince",
|
||||
baseQuery: `
|
||||
SELECT
|
||||
environment_id,
|
||||
task_identifier,
|
||||
error_fingerprint,
|
||||
sum(count) as occurrences_since
|
||||
FROM trigger_dev.error_occurrences_v1
|
||||
`,
|
||||
schema: OccurrenceCountsSinceQueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Alert evaluator helpers – occurrence rate & count since timestamp
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const ErrorOccurrenceTotalCountResult = z.object({
|
||||
total_count: z.number(),
|
||||
});
|
||||
|
||||
export type ErrorOccurrenceTotalCountResult = z.infer<typeof ErrorOccurrenceTotalCountResult>;
|
||||
|
||||
/**
|
||||
* Query builder for summing occurrences since a given timestamp.
|
||||
* Used by the alert evaluator to check total-count-based ignore thresholds.
|
||||
*/
|
||||
export function getOccurrenceCountSinceQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilder({
|
||||
name: "getOccurrenceCountSince",
|
||||
baseQuery: `
|
||||
SELECT
|
||||
sum(count) as total_count
|
||||
FROM trigger_dev.error_occurrences_v1
|
||||
`,
|
||||
schema: ErrorOccurrenceTotalCountResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
import type { ClickHouseSettings } from "@clickhouse/client";
|
||||
export type { ClickHouseSettings };
|
||||
import { ClickhouseClient } from "./client/client.js";
|
||||
import type { ClickhouseReader, ClickhouseWriter } from "./client/types.js";
|
||||
import { NoopClient } from "./client/noop.js";
|
||||
import {
|
||||
insertTaskRunsCompactArrays,
|
||||
insertRawTaskRunPayloadsCompactArrays,
|
||||
getTaskRunsQueryBuilder,
|
||||
getTaskActivityQueryBuilder,
|
||||
getCurrentRunningStats,
|
||||
getChildRunStatusCounts,
|
||||
getAverageDurations,
|
||||
getTaskUsageByOrganization,
|
||||
getTaskRunsCountQueryBuilder,
|
||||
getTaskRunTagsQueryBuilder,
|
||||
getPendingVersionIdsQueryBuilder,
|
||||
getTaskRunExistsQueryBuilder,
|
||||
} from "./taskRuns.js";
|
||||
import {
|
||||
getSpanDetailsQueryBuilder,
|
||||
getSpanDetailsQueryBuilderV2,
|
||||
getTraceDetailedSummaryQueryBuilder,
|
||||
getTraceDetailedSummaryQueryBuilderV2,
|
||||
getTraceEventsForExportQueryBuilder,
|
||||
getTraceEventsForExportQueryBuilderV2,
|
||||
getTraceSummaryQueryBuilder,
|
||||
getTraceSummaryQueryBuilderV2,
|
||||
insertTaskEvents,
|
||||
insertTaskEventsV2,
|
||||
getLogDetailQueryBuilderV2,
|
||||
getLogsSearchListQueryBuilder,
|
||||
} from "./taskEvents.js";
|
||||
import { insertMetrics } from "./metrics.js";
|
||||
import { insertLlmMetrics } from "./llmMetrics.js";
|
||||
import {
|
||||
getSessionTagsQueryBuilder,
|
||||
getSessionsCountQueryBuilder,
|
||||
getSessionsQueryBuilder,
|
||||
insertSessionsCompactArrays,
|
||||
} from "./sessions.js";
|
||||
import {
|
||||
getGlobalModelMetrics,
|
||||
getGlobalModelComparison,
|
||||
getPopularModels,
|
||||
} from "./llmModelAggregates.js";
|
||||
import {
|
||||
getErrorGroups,
|
||||
getErrorInstances,
|
||||
getErrorGroupsListQueryBuilder,
|
||||
getErrorHourlyOccurrences,
|
||||
getErrorOccurrencesListQueryBuilder,
|
||||
createErrorOccurrencesQueryBuilder,
|
||||
createErrorOccurrencesByVersionQueryBuilder,
|
||||
getErrorAffectedVersionsQueryBuilder,
|
||||
getOccurrenceCountSinceQueryBuilder,
|
||||
getActiveErrorsSinceQueryBuilder,
|
||||
getOccurrenceCountsSinceQueryBuilder,
|
||||
} from "./errors.js";
|
||||
export { msToClickHouseInterval } from "./intervals.js";
|
||||
import { Logger, type LogLevel } from "@trigger.dev/core/logger";
|
||||
import type { Agent as HttpAgent } from "http";
|
||||
import type { Agent as HttpsAgent } from "https";
|
||||
|
||||
export type * from "./taskRuns.js";
|
||||
export type * from "./taskEvents.js";
|
||||
export type * from "./metrics.js";
|
||||
export type * from "./llmMetrics.js";
|
||||
export type * from "./llmModelAggregates.js";
|
||||
export type * from "./errors.js";
|
||||
export type * from "./sessions.js";
|
||||
export type * from "./client/queryBuilder.js";
|
||||
|
||||
// Re-export column constants, indices, and type-safe accessors
|
||||
export {
|
||||
TASK_RUN_COLUMNS,
|
||||
TASK_RUN_INDEX,
|
||||
PAYLOAD_COLUMNS,
|
||||
PAYLOAD_INDEX,
|
||||
getTaskRunField,
|
||||
getPayloadField,
|
||||
composeTaskRunVersion,
|
||||
} from "./taskRuns.js";
|
||||
|
||||
export { SESSION_COLUMNS, SESSION_INDEX, getSessionField } from "./sessions.js";
|
||||
|
||||
// TSQL query execution
|
||||
export {
|
||||
executeTSQL,
|
||||
createTSQLExecutor,
|
||||
type ExecuteTSQLOptions,
|
||||
type TableSchema,
|
||||
type TSQLQueryResult,
|
||||
type TSQLQuerySuccess,
|
||||
type QueryStats,
|
||||
type FieldMappings,
|
||||
type WhereClauseCondition,
|
||||
} from "./client/tsql.js";
|
||||
export type { ColumnFormatType, OutputColumnMetadata } from "@internal/tsql";
|
||||
|
||||
// Errors
|
||||
export { QueryError } from "./client/errors.js";
|
||||
|
||||
export type ClickhouseCommonConfig = {
|
||||
keepAlive?: {
|
||||
enabled?: boolean;
|
||||
idleSocketTtl?: number;
|
||||
};
|
||||
httpAgent?: HttpAgent | HttpsAgent;
|
||||
clickhouseSettings?: ClickHouseSettings;
|
||||
logger?: Logger;
|
||||
logLevel?: LogLevel;
|
||||
compression?: {
|
||||
request?: boolean;
|
||||
response?: boolean;
|
||||
};
|
||||
maxOpenConnections?: number;
|
||||
};
|
||||
|
||||
export type ClickHouseConfig =
|
||||
| ({
|
||||
name?: string;
|
||||
url?: string;
|
||||
writerUrl?: never;
|
||||
readerUrl?: never;
|
||||
} & ClickhouseCommonConfig)
|
||||
| ({
|
||||
name?: never;
|
||||
url?: never;
|
||||
writerName?: string;
|
||||
writerUrl: string;
|
||||
readerName?: string;
|
||||
readerUrl: string;
|
||||
} & ClickhouseCommonConfig);
|
||||
|
||||
export class ClickHouse {
|
||||
public readonly reader: ClickhouseReader;
|
||||
public readonly writer: ClickhouseWriter;
|
||||
private readonly logger: Logger;
|
||||
private _splitClients: boolean;
|
||||
|
||||
constructor(config: ClickHouseConfig) {
|
||||
this.logger = config.logger ?? new Logger("ClickHouse", config.logLevel ?? "debug");
|
||||
|
||||
if (config.url) {
|
||||
const url = new URL(config.url);
|
||||
url.password = "redacted";
|
||||
|
||||
this.logger.info("🏠 Initializing ClickHouse client with url", { url: url.toString() });
|
||||
|
||||
const client = new ClickhouseClient({
|
||||
name: config.name ?? "clickhouse",
|
||||
url: config.url,
|
||||
clickhouseSettings: config.clickhouseSettings,
|
||||
logger: this.logger,
|
||||
logLevel: config.logLevel,
|
||||
keepAlive: config.keepAlive,
|
||||
httpAgent: config.httpAgent,
|
||||
maxOpenConnections: config.maxOpenConnections,
|
||||
compression: config.compression,
|
||||
});
|
||||
this.reader = client;
|
||||
this.writer = client;
|
||||
|
||||
this._splitClients = false;
|
||||
} else if (config.writerUrl && config.readerUrl) {
|
||||
this.reader = new ClickhouseClient({
|
||||
name: config.readerName ?? "clickhouse-reader",
|
||||
url: config.readerUrl,
|
||||
clickhouseSettings: config.clickhouseSettings,
|
||||
logger: this.logger,
|
||||
logLevel: config.logLevel,
|
||||
keepAlive: config.keepAlive,
|
||||
httpAgent: config.httpAgent,
|
||||
maxOpenConnections: config.maxOpenConnections,
|
||||
compression: config.compression,
|
||||
});
|
||||
this.writer = new ClickhouseClient({
|
||||
name: config.writerName ?? "clickhouse-writer",
|
||||
url: config.writerUrl,
|
||||
clickhouseSettings: config.clickhouseSettings,
|
||||
logger: this.logger,
|
||||
logLevel: config.logLevel,
|
||||
keepAlive: config.keepAlive,
|
||||
httpAgent: config.httpAgent,
|
||||
maxOpenConnections: config.maxOpenConnections,
|
||||
compression: config.compression,
|
||||
});
|
||||
|
||||
this._splitClients = true;
|
||||
} else {
|
||||
this.reader = new NoopClient();
|
||||
this.writer = new NoopClient();
|
||||
|
||||
this._splitClients = true;
|
||||
}
|
||||
}
|
||||
|
||||
static fromEnv(): ClickHouse {
|
||||
if (
|
||||
typeof process.env.CLICKHOUSE_WRITER_URL === "string" &&
|
||||
typeof process.env.CLICKHOUSE_READER_URL === "string"
|
||||
) {
|
||||
return new ClickHouse({
|
||||
writerUrl: process.env.CLICKHOUSE_WRITER_URL,
|
||||
readerUrl: process.env.CLICKHOUSE_READER_URL,
|
||||
writerName: process.env.CLICKHOUSE_WRITER_NAME,
|
||||
readerName: process.env.CLICKHOUSE_READER_NAME,
|
||||
});
|
||||
}
|
||||
|
||||
return new ClickHouse({
|
||||
url: process.env.CLICKHOUSE_URL,
|
||||
name: process.env.CLICKHOUSE_NAME,
|
||||
});
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this._splitClients) {
|
||||
await Promise.all([this.reader.close(), this.writer.close()]);
|
||||
} else {
|
||||
await this.reader.close();
|
||||
}
|
||||
}
|
||||
|
||||
get taskRuns() {
|
||||
return {
|
||||
insertCompactArrays: insertTaskRunsCompactArrays(this.writer),
|
||||
insertPayloadsCompactArrays: insertRawTaskRunPayloadsCompactArrays(this.writer),
|
||||
queryBuilder: getTaskRunsQueryBuilder(this.reader),
|
||||
countQueryBuilder: getTaskRunsCountQueryBuilder(this.reader),
|
||||
existsQueryBuilder: getTaskRunExistsQueryBuilder(this.reader, { max_execution_time: 10 }),
|
||||
tagQueryBuilder: getTaskRunTagsQueryBuilder(this.reader),
|
||||
pendingVersionIdsQueryBuilder: getPendingVersionIdsQueryBuilder(this.reader),
|
||||
getTaskActivity: getTaskActivityQueryBuilder(this.reader),
|
||||
getCurrentRunningStats: getCurrentRunningStats(this.reader),
|
||||
getChildRunStatusCounts: getChildRunStatusCounts(this.reader),
|
||||
getAverageDurations: getAverageDurations(this.reader),
|
||||
getTaskUsageByOrganization: getTaskUsageByOrganization(this.reader),
|
||||
};
|
||||
}
|
||||
|
||||
get taskEvents() {
|
||||
return {
|
||||
insert: insertTaskEvents(this.writer),
|
||||
traceSummaryQueryBuilder: getTraceSummaryQueryBuilder(this.reader),
|
||||
traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilder(this.reader),
|
||||
traceEventsForExportQueryBuilder: getTraceEventsForExportQueryBuilder(this.reader),
|
||||
spanDetailsQueryBuilder: getSpanDetailsQueryBuilder(this.reader),
|
||||
};
|
||||
}
|
||||
|
||||
get metrics() {
|
||||
return {
|
||||
insert: insertMetrics(this.writer),
|
||||
};
|
||||
}
|
||||
|
||||
get llmMetrics() {
|
||||
return {
|
||||
insert: insertLlmMetrics(this.writer),
|
||||
};
|
||||
}
|
||||
|
||||
get llmModelAggregates() {
|
||||
return {
|
||||
globalMetrics: getGlobalModelMetrics(this.reader),
|
||||
comparison: getGlobalModelComparison(this.reader),
|
||||
popular: getPopularModels(this.reader),
|
||||
};
|
||||
}
|
||||
|
||||
get sessions() {
|
||||
return {
|
||||
insertCompactArrays: insertSessionsCompactArrays(this.writer),
|
||||
queryBuilder: getSessionsQueryBuilder(this.reader),
|
||||
countQueryBuilder: getSessionsCountQueryBuilder(this.reader),
|
||||
tagQueryBuilder: getSessionTagsQueryBuilder(this.reader),
|
||||
};
|
||||
}
|
||||
|
||||
get taskEventsV2() {
|
||||
return {
|
||||
insert: insertTaskEventsV2(this.writer),
|
||||
traceSummaryQueryBuilder: getTraceSummaryQueryBuilderV2(this.reader),
|
||||
traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilderV2(this.reader),
|
||||
traceEventsForExportQueryBuilder: getTraceEventsForExportQueryBuilderV2(this.reader),
|
||||
spanDetailsQueryBuilder: getSpanDetailsQueryBuilderV2(this.reader),
|
||||
logDetailQueryBuilder: getLogDetailQueryBuilderV2(this.reader),
|
||||
};
|
||||
}
|
||||
|
||||
get taskEventsSearch() {
|
||||
return {
|
||||
logsListQueryBuilder: getLogsSearchListQueryBuilder(this.reader),
|
||||
};
|
||||
}
|
||||
|
||||
get errors() {
|
||||
return {
|
||||
getGroups: getErrorGroups(this.reader),
|
||||
getInstances: getErrorInstances(this.reader),
|
||||
getHourlyOccurrences: getErrorHourlyOccurrences(this.reader),
|
||||
affectedVersionsQueryBuilder: getErrorAffectedVersionsQueryBuilder(this.reader),
|
||||
listQueryBuilder: getErrorGroupsListQueryBuilder(this.reader),
|
||||
occurrencesListQueryBuilder: getErrorOccurrencesListQueryBuilder(this.reader),
|
||||
createOccurrencesQueryBuilder: (intervalExpr: string) =>
|
||||
createErrorOccurrencesQueryBuilder(this.reader, intervalExpr),
|
||||
createOccurrencesByVersionQueryBuilder: (intervalExpr: string) =>
|
||||
createErrorOccurrencesByVersionQueryBuilder(this.reader, intervalExpr),
|
||||
occurrenceCountSinceQueryBuilder: getOccurrenceCountSinceQueryBuilder(this.reader),
|
||||
activeErrorsSinceQueryBuilder: getActiveErrorsSinceQueryBuilder(this.reader),
|
||||
occurrenceCountsSinceQueryBuilder: getOccurrenceCountsSinceQueryBuilder(this.reader),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
/** Converts a granularity in milliseconds to a ClickHouse INTERVAL expression. */
|
||||
export function msToClickHouseInterval(ms: number): string {
|
||||
const seconds = Math.round(ms / 1000);
|
||||
return `INTERVAL ${seconds} SECOND`;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { z } from "zod";
|
||||
import type { ClickhouseWriter } from "./client/types.js";
|
||||
|
||||
export const LlmMetricsV1Input = z.object({
|
||||
organization_id: z.string(),
|
||||
project_id: z.string(),
|
||||
environment_id: z.string(),
|
||||
run_id: z.string(),
|
||||
task_identifier: z.string(),
|
||||
trace_id: z.string(),
|
||||
span_id: z.string(),
|
||||
|
||||
gen_ai_system: z.string(),
|
||||
request_model: z.string(),
|
||||
response_model: z.string(),
|
||||
base_response_model: z.string(),
|
||||
matched_model_id: z.string(),
|
||||
operation_id: z.string(),
|
||||
finish_reason: z.string(),
|
||||
cost_source: z.string(),
|
||||
|
||||
pricing_tier_id: z.string(),
|
||||
pricing_tier_name: z.string(),
|
||||
|
||||
input_tokens: z.number(),
|
||||
output_tokens: z.number(),
|
||||
total_tokens: z.number(),
|
||||
usage_details: z.record(z.string(), z.number()),
|
||||
|
||||
input_cost: z.number(),
|
||||
output_cost: z.number(),
|
||||
total_cost: z.number(),
|
||||
cost_details: z.record(z.string(), z.number()),
|
||||
provider_cost: z.number(),
|
||||
|
||||
ms_to_first_chunk: z.number(),
|
||||
tokens_per_second: z.number(),
|
||||
|
||||
metadata: z.record(z.string(), z.string()),
|
||||
|
||||
prompt_slug: z.string(),
|
||||
prompt_version: z.number(),
|
||||
|
||||
start_time: z.string(),
|
||||
duration: z.string(),
|
||||
});
|
||||
|
||||
export type LlmMetricsV1Input = z.input<typeof LlmMetricsV1Input>;
|
||||
|
||||
export function insertLlmMetrics(ch: ClickhouseWriter) {
|
||||
return ch.insertUnsafe<LlmMetricsV1Input>({
|
||||
name: "insertLlmMetrics",
|
||||
table: "trigger_dev.llm_metrics_v1",
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { z } from "zod";
|
||||
import { ClickhouseQueryBuilder } from "./client/queryBuilder.js";
|
||||
import type { ClickhouseReader } from "./client/types.js";
|
||||
|
||||
// --- Schemas ---
|
||||
|
||||
const ModelMetricsRow = z.object({
|
||||
response_model: z.string(),
|
||||
gen_ai_system: z.string(),
|
||||
minute: z.string(),
|
||||
call_count: z.coerce.number(),
|
||||
total_input_tokens: z.coerce.number(),
|
||||
total_output_tokens: z.coerce.number(),
|
||||
total_cost: z.coerce.number(),
|
||||
ttfc_p50: z.coerce.number(),
|
||||
ttfc_p90: z.coerce.number(),
|
||||
ttfc_p95: z.coerce.number(),
|
||||
ttfc_p99: z.coerce.number(),
|
||||
tps_p50: z.coerce.number(),
|
||||
tps_p90: z.coerce.number(),
|
||||
tps_p95: z.coerce.number(),
|
||||
tps_p99: z.coerce.number(),
|
||||
duration_p50: z.coerce.number(),
|
||||
duration_p90: z.coerce.number(),
|
||||
duration_p95: z.coerce.number(),
|
||||
duration_p99: z.coerce.number(),
|
||||
});
|
||||
|
||||
const ModelSummaryRow = z.object({
|
||||
response_model: z.string(),
|
||||
gen_ai_system: z.string(),
|
||||
call_count: z.coerce.number(),
|
||||
total_input_tokens: z.coerce.number(),
|
||||
total_output_tokens: z.coerce.number(),
|
||||
total_cost: z.coerce.number(),
|
||||
ttfc_p50: z.coerce.number(),
|
||||
ttfc_p90: z.coerce.number(),
|
||||
tps_p50: z.coerce.number(),
|
||||
tps_p90: z.coerce.number(),
|
||||
});
|
||||
|
||||
const PopularModelRow = z.object({
|
||||
response_model: z.string(),
|
||||
gen_ai_system: z.string(),
|
||||
call_count: z.coerce.number(),
|
||||
total_cost: z.coerce.number(),
|
||||
ttfc_p50: z.coerce.number(),
|
||||
});
|
||||
|
||||
// --- Query builders ---
|
||||
|
||||
/** Get per-minute metrics for a specific model over a date range. */
|
||||
export function getGlobalModelMetrics(reader: ClickhouseReader) {
|
||||
return new ClickhouseQueryBuilder(
|
||||
"getGlobalModelMetrics",
|
||||
`SELECT
|
||||
response_model,
|
||||
gen_ai_system,
|
||||
minute,
|
||||
sum(call_count) AS call_count,
|
||||
sum(total_input_tokens) AS total_input_tokens,
|
||||
sum(total_output_tokens) AS total_output_tokens,
|
||||
sum(total_cost) AS total_cost,
|
||||
quantilesMerge(0.5, 0.9, 0.95, 0.99)(ttfc_quantiles) AS ttfc_arr,
|
||||
ttfc_arr[1] AS ttfc_p50,
|
||||
ttfc_arr[2] AS ttfc_p90,
|
||||
ttfc_arr[3] AS ttfc_p95,
|
||||
ttfc_arr[4] AS ttfc_p99,
|
||||
quantilesMerge(0.5, 0.9, 0.95, 0.99)(tps_quantiles) AS tps_arr,
|
||||
tps_arr[1] AS tps_p50,
|
||||
tps_arr[2] AS tps_p90,
|
||||
tps_arr[3] AS tps_p95,
|
||||
tps_arr[4] AS tps_p99,
|
||||
quantilesMerge(0.5, 0.9, 0.95, 0.99)(duration_quantiles) AS dur_arr,
|
||||
dur_arr[1] AS duration_p50,
|
||||
dur_arr[2] AS duration_p90,
|
||||
dur_arr[3] AS duration_p95,
|
||||
dur_arr[4] AS duration_p99
|
||||
FROM trigger_dev.llm_model_aggregates_v1
|
||||
WHERE response_model = {responseModel: String}
|
||||
AND minute >= {startTime: DateTime}
|
||||
AND minute <= {endTime: DateTime}
|
||||
GROUP BY response_model, gen_ai_system, minute
|
||||
ORDER BY minute`,
|
||||
reader,
|
||||
ModelMetricsRow
|
||||
);
|
||||
}
|
||||
|
||||
/** Get summary metrics for multiple models (for comparison). */
|
||||
export function getGlobalModelComparison(reader: ClickhouseReader) {
|
||||
return new ClickhouseQueryBuilder(
|
||||
"getGlobalModelComparison",
|
||||
`SELECT
|
||||
response_model,
|
||||
gen_ai_system,
|
||||
sum(call_count) AS call_count,
|
||||
sum(total_input_tokens) AS total_input_tokens,
|
||||
sum(total_output_tokens) AS total_output_tokens,
|
||||
sum(total_cost) AS total_cost,
|
||||
quantilesMerge(0.5, 0.9)(ttfc_quantiles) AS ttfc_arr,
|
||||
ttfc_arr[1] AS ttfc_p50,
|
||||
ttfc_arr[2] AS ttfc_p90,
|
||||
quantilesMerge(0.5, 0.9)(tps_quantiles) AS tps_arr,
|
||||
tps_arr[1] AS tps_p50,
|
||||
tps_arr[2] AS tps_p90
|
||||
FROM trigger_dev.llm_model_aggregates_v1
|
||||
WHERE response_model IN {responseModels: Array(String)}
|
||||
AND minute >= {startTime: DateTime}
|
||||
AND minute <= {endTime: DateTime}
|
||||
GROUP BY response_model, gen_ai_system
|
||||
ORDER BY call_count DESC`,
|
||||
reader,
|
||||
ModelSummaryRow
|
||||
);
|
||||
}
|
||||
|
||||
/** Get the most popular models by call count. */
|
||||
export function getPopularModels(reader: ClickhouseReader) {
|
||||
return new ClickhouseQueryBuilder(
|
||||
"getPopularModels",
|
||||
`SELECT
|
||||
response_model,
|
||||
gen_ai_system,
|
||||
sum(call_count) AS call_count,
|
||||
sum(total_cost) AS total_cost,
|
||||
quantilesMerge(0.5)(ttfc_quantiles) AS ttfc_arr,
|
||||
ttfc_arr[1] AS ttfc_p50
|
||||
FROM trigger_dev.llm_model_aggregates_v1
|
||||
WHERE minute >= {startTime: DateTime}
|
||||
AND minute <= {endTime: DateTime}
|
||||
GROUP BY response_model, gen_ai_system
|
||||
ORDER BY call_count DESC
|
||||
LIMIT {limit: UInt32}`,
|
||||
reader,
|
||||
PopularModelRow
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { z } from "zod";
|
||||
import type { ClickhouseWriter } from "./client/types.js";
|
||||
|
||||
export const MetricsV1Input = z.object({
|
||||
organization_id: z.string(),
|
||||
project_id: z.string(),
|
||||
environment_id: z.string(),
|
||||
metric_name: z.string(),
|
||||
metric_type: z.string(),
|
||||
metric_subject: z.string(),
|
||||
bucket_start: z.string(),
|
||||
value: z.number(),
|
||||
attributes: z.unknown(),
|
||||
});
|
||||
|
||||
export type MetricsV1Input = z.input<typeof MetricsV1Input>;
|
||||
|
||||
export function insertMetrics(ch: ClickhouseWriter) {
|
||||
return ch.insertUnsafe<MetricsV1Input>({
|
||||
name: "insertMetrics",
|
||||
table: "trigger_dev.metrics_v1",
|
||||
settings: {
|
||||
enable_json_type: 1,
|
||||
type_json_skip_duplicated_paths: 1,
|
||||
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
|
||||
input_format_json_throw_on_bad_escape_sequence: 0,
|
||||
input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import type { ClickHouseSettings } from "@clickhouse/client";
|
||||
import { z } from "zod";
|
||||
import type { ClickhouseReader, ClickhouseWriter } from "./client/types.js";
|
||||
|
||||
export const SessionV1 = z.object({
|
||||
environment_id: z.string(),
|
||||
organization_id: z.string(),
|
||||
project_id: z.string(),
|
||||
session_id: z.string(),
|
||||
environment_type: z.string(),
|
||||
friendly_id: z.string(),
|
||||
external_id: z.string().default(""),
|
||||
type: z.string(),
|
||||
task_identifier: z.string().default(""),
|
||||
tags: z.array(z.string()).default([]),
|
||||
metadata: z.unknown(),
|
||||
closed_at: z.number().int().nullish(),
|
||||
closed_reason: z.string().default(""),
|
||||
expires_at: z.number().int().nullish(),
|
||||
created_at: z.number().int(),
|
||||
updated_at: z.number().int(),
|
||||
is_test: z.boolean().default(false),
|
||||
_version: z.string(),
|
||||
_is_deleted: z.number().int().default(0),
|
||||
});
|
||||
|
||||
export type SessionV1 = z.input<typeof SessionV1>;
|
||||
|
||||
// Column order for compact format - must match ClickHouse table schema
|
||||
export const SESSION_COLUMNS = [
|
||||
"environment_id",
|
||||
"organization_id",
|
||||
"project_id",
|
||||
"session_id",
|
||||
"environment_type",
|
||||
"friendly_id",
|
||||
"external_id",
|
||||
"type",
|
||||
"task_identifier",
|
||||
"tags",
|
||||
"metadata",
|
||||
"closed_at",
|
||||
"closed_reason",
|
||||
"expires_at",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"is_test",
|
||||
"_version",
|
||||
"_is_deleted",
|
||||
] as const;
|
||||
|
||||
export type SessionColumnName = (typeof SESSION_COLUMNS)[number];
|
||||
|
||||
export const SESSION_INDEX = Object.fromEntries(SESSION_COLUMNS.map((col, idx) => [col, idx])) as {
|
||||
readonly [K in SessionColumnName]: number;
|
||||
};
|
||||
|
||||
export type SessionFieldTypes = {
|
||||
environment_id: string;
|
||||
organization_id: string;
|
||||
project_id: string;
|
||||
session_id: string;
|
||||
environment_type: string;
|
||||
friendly_id: string;
|
||||
external_id: string;
|
||||
type: string;
|
||||
task_identifier: string;
|
||||
tags: string[];
|
||||
metadata: { data: unknown };
|
||||
closed_at: number | null;
|
||||
closed_reason: string;
|
||||
expires_at: number | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
is_test: boolean;
|
||||
_version: string;
|
||||
_is_deleted: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Type-safe tuple representing a Session insert array.
|
||||
* Order matches {@link SESSION_COLUMNS} exactly.
|
||||
*/
|
||||
export type SessionInsertArray = [
|
||||
environment_id: string,
|
||||
organization_id: string,
|
||||
project_id: string,
|
||||
session_id: string,
|
||||
environment_type: string,
|
||||
friendly_id: string,
|
||||
external_id: string,
|
||||
type: string,
|
||||
task_identifier: string,
|
||||
tags: string[],
|
||||
metadata: { data: unknown },
|
||||
closed_at: number | null,
|
||||
closed_reason: string,
|
||||
expires_at: number | null,
|
||||
created_at: number,
|
||||
updated_at: number,
|
||||
is_test: boolean,
|
||||
_version: string,
|
||||
_is_deleted: number,
|
||||
];
|
||||
|
||||
export function getSessionField<K extends SessionColumnName>(
|
||||
session: SessionInsertArray,
|
||||
field: K
|
||||
): SessionFieldTypes[K] {
|
||||
return session[SESSION_INDEX[field]] as SessionFieldTypes[K];
|
||||
}
|
||||
|
||||
export function insertSessionsCompactArrays(ch: ClickhouseWriter, settings?: ClickHouseSettings) {
|
||||
return ch.insertCompactRaw({
|
||||
name: "insertSessionsCompactArrays",
|
||||
table: "trigger_dev.sessions_v1",
|
||||
columns: SESSION_COLUMNS,
|
||||
settings: {
|
||||
enable_json_type: 1,
|
||||
type_json_skip_duplicated_paths: 1,
|
||||
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
|
||||
...settings,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function insertSessions(ch: ClickhouseWriter, settings?: ClickHouseSettings) {
|
||||
return ch.insert({
|
||||
name: "insertSessions",
|
||||
table: "trigger_dev.sessions_v1",
|
||||
schema: SessionV1,
|
||||
settings: {
|
||||
enable_json_type: 1,
|
||||
type_json_skip_duplicated_paths: 1,
|
||||
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
|
||||
...settings,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ─── read path ───────────────────────────────────────────────────
|
||||
|
||||
export const SessionV1QueryResult = z.object({
|
||||
session_id: z.string(),
|
||||
});
|
||||
|
||||
export type SessionV1QueryResult = z.infer<typeof SessionV1QueryResult>;
|
||||
|
||||
/**
|
||||
* Base query builder for listing Sessions. Filters + pagination are composed
|
||||
* on top of this; callers can chain `.where(...).orderBy(...).limit(...)`.
|
||||
*/
|
||||
export function getSessionsQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilder({
|
||||
name: "getSessions",
|
||||
baseQuery: "SELECT session_id FROM trigger_dev.sessions_v1 FINAL",
|
||||
schema: SessionV1QueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export function getSessionsCountQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilder({
|
||||
name: "getSessionsCount",
|
||||
baseQuery: "SELECT count() as count FROM trigger_dev.sessions_v1 FINAL",
|
||||
schema: z.object({ count: z.number().int() }),
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const SessionTagsQueryResult = z.object({
|
||||
tag: z.string(),
|
||||
});
|
||||
|
||||
export type SessionTagsQueryResult = z.infer<typeof SessionTagsQueryResult>;
|
||||
|
||||
export function getSessionTagsQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilder({
|
||||
name: "getSessionTags",
|
||||
baseQuery: "SELECT DISTINCT arrayJoin(tags) as tag FROM trigger_dev.sessions_v1",
|
||||
schema: SessionTagsQueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,374 @@
|
||||
import type { ClickHouseSettings } from "@clickhouse/client";
|
||||
import { z } from "zod";
|
||||
import type { ClickhouseReader, ClickhouseWriter } from "./client/types.js";
|
||||
|
||||
export const TaskEventV1Input = z.object({
|
||||
environment_id: z.string(),
|
||||
organization_id: z.string(),
|
||||
project_id: z.string(),
|
||||
task_identifier: z.string(),
|
||||
run_id: z.string(),
|
||||
start_time: z.string(),
|
||||
duration: z.string(),
|
||||
trace_id: z.string(),
|
||||
span_id: z.string(),
|
||||
parent_span_id: z.string(),
|
||||
message: z.string(),
|
||||
kind: z.string(),
|
||||
status: z.string(),
|
||||
attributes: z.unknown(),
|
||||
metadata: z.string(),
|
||||
expires_at: z.string(),
|
||||
machine_id: z.string().optional(),
|
||||
});
|
||||
|
||||
export type TaskEventV1Input = z.input<typeof TaskEventV1Input>;
|
||||
|
||||
export function insertTaskEvents(ch: ClickhouseWriter, settings?: ClickHouseSettings) {
|
||||
return ch.insertUnsafe<TaskEventV1Input>({
|
||||
name: "insertTaskEvents",
|
||||
table: "trigger_dev.task_events_v1",
|
||||
settings: {
|
||||
enable_json_type: 1,
|
||||
type_json_skip_duplicated_paths: 1,
|
||||
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
|
||||
input_format_json_throw_on_bad_escape_sequence: 0,
|
||||
input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
|
||||
...settings,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export const TaskEventSummaryV1Result = z.object({
|
||||
span_id: z.string(),
|
||||
parent_span_id: z.string(),
|
||||
run_id: z.string(),
|
||||
start_time: z.string(),
|
||||
duration: z.number().or(z.string()),
|
||||
status: z.string(),
|
||||
kind: z.string(),
|
||||
metadata: z.string(),
|
||||
message: z.string(),
|
||||
});
|
||||
|
||||
export type TaskEventSummaryV1Result = z.output<typeof TaskEventSummaryV1Result>;
|
||||
|
||||
export function getTraceSummaryQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilderFast<TaskEventSummaryV1Result>({
|
||||
name: "getTraceEvents",
|
||||
table: "trigger_dev.task_events_v1",
|
||||
columns: [
|
||||
"span_id",
|
||||
"parent_span_id",
|
||||
"run_id",
|
||||
"start_time",
|
||||
"duration",
|
||||
"status",
|
||||
"kind",
|
||||
"metadata",
|
||||
{ name: "message", expression: "LEFT(message, 256)" },
|
||||
],
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const TaskEventDetailedSummaryV1Result = z.object({
|
||||
span_id: z.string(),
|
||||
parent_span_id: z.string(),
|
||||
run_id: z.string(),
|
||||
start_time: z.string(),
|
||||
duration: z.number().or(z.string()),
|
||||
status: z.string(),
|
||||
kind: z.string(),
|
||||
metadata: z.string(),
|
||||
message: z.string(),
|
||||
attributes_text: z.string(),
|
||||
});
|
||||
|
||||
export type TaskEventDetailedSummaryV1Result = z.output<typeof TaskEventDetailedSummaryV1Result>;
|
||||
|
||||
export function getTraceDetailedSummaryQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilderFast<TaskEventDetailedSummaryV1Result>({
|
||||
name: "getTaskEventDetailedSummary",
|
||||
table: "trigger_dev.task_events_v1",
|
||||
columns: [
|
||||
"span_id",
|
||||
"parent_span_id",
|
||||
"run_id",
|
||||
"start_time",
|
||||
"duration",
|
||||
"status",
|
||||
"kind",
|
||||
"metadata",
|
||||
{ name: "message", expression: "LEFT(message, 256)" },
|
||||
"attributes_text",
|
||||
],
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
// Row shape for streaming a whole trace out for export (the "Download trace"
|
||||
// feature). Unlike the detailed-summary builders this keeps the FULL message
|
||||
// (not LEFT(message, 256)) since the export is the source of truth, and it's
|
||||
// consumed via executeStream() so the trace is never fully materialised.
|
||||
export type TaskEventExportRow = {
|
||||
span_id: string;
|
||||
parent_span_id: string;
|
||||
start_time: string;
|
||||
duration: number | string;
|
||||
status: string;
|
||||
kind: string;
|
||||
message: string;
|
||||
attributes_text: string;
|
||||
};
|
||||
|
||||
const TASK_EVENT_EXPORT_COLUMNS = [
|
||||
"span_id",
|
||||
"parent_span_id",
|
||||
"start_time",
|
||||
"duration",
|
||||
"status",
|
||||
"kind",
|
||||
"message",
|
||||
"attributes_text",
|
||||
] as const;
|
||||
|
||||
export function getTraceEventsForExportQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilderFast<TaskEventExportRow>({
|
||||
name: "getTraceEventsForExport",
|
||||
table: "trigger_dev.task_events_v1",
|
||||
columns: [...TASK_EVENT_EXPORT_COLUMNS],
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const TaskEventDetailsV1Result = z.object({
|
||||
span_id: z.string(),
|
||||
parent_span_id: z.string(),
|
||||
start_time: z.string(),
|
||||
duration: z.number().or(z.string()),
|
||||
status: z.string(),
|
||||
kind: z.string(),
|
||||
metadata: z.string(),
|
||||
message: z.string(),
|
||||
attributes_text: z.string(),
|
||||
});
|
||||
|
||||
export type TaskEventDetailsV1Result = z.input<typeof TaskEventDetailsV1Result>;
|
||||
|
||||
export function getSpanDetailsQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilder({
|
||||
name: "getSpanDetails",
|
||||
baseQuery:
|
||||
"SELECT span_id, parent_span_id, start_time, duration, status, kind, metadata, message, attributes_text FROM trigger_dev.task_events_v1",
|
||||
schema: TaskEventDetailsV1Result,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// V2 Table Functions (partitioned by inserted_at instead of start_time)
|
||||
// ============================================================================
|
||||
|
||||
export const TaskEventV2Input = z.object({
|
||||
environment_id: z.string(),
|
||||
organization_id: z.string(),
|
||||
project_id: z.string(),
|
||||
task_identifier: z.string(),
|
||||
run_id: z.string(),
|
||||
start_time: z.string(),
|
||||
duration: z.string(),
|
||||
trace_id: z.string(),
|
||||
span_id: z.string(),
|
||||
parent_span_id: z.string(),
|
||||
message: z.string(),
|
||||
kind: z.string(),
|
||||
status: z.string(),
|
||||
attributes: z.unknown(),
|
||||
metadata: z.string(),
|
||||
expires_at: z.string(),
|
||||
machine_id: z.string().optional(),
|
||||
// inserted_at has a default value in the table, so it's optional for inserts
|
||||
inserted_at: z.string().optional(),
|
||||
});
|
||||
|
||||
export type TaskEventV2Input = z.input<typeof TaskEventV2Input>;
|
||||
|
||||
export function insertTaskEventsV2(ch: ClickhouseWriter, settings?: ClickHouseSettings) {
|
||||
return ch.insertUnsafe<TaskEventV2Input>({
|
||||
name: "insertTaskEventsV2",
|
||||
table: "trigger_dev.task_events_v2",
|
||||
settings: {
|
||||
enable_json_type: 1,
|
||||
type_json_skip_duplicated_paths: 1,
|
||||
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
|
||||
input_format_json_throw_on_bad_escape_sequence: 0,
|
||||
input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
|
||||
...settings,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function getTraceSummaryQueryBuilderV2(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilderFast<TaskEventSummaryV1Result>({
|
||||
name: "getTraceEventsV2",
|
||||
table: "trigger_dev.task_events_v2",
|
||||
columns: [
|
||||
"span_id",
|
||||
"parent_span_id",
|
||||
"run_id",
|
||||
"start_time",
|
||||
"duration",
|
||||
"status",
|
||||
"kind",
|
||||
"metadata",
|
||||
{ name: "message", expression: "LEFT(message, 256)" },
|
||||
],
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export function getTraceDetailedSummaryQueryBuilderV2(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilderFast<TaskEventDetailedSummaryV1Result>({
|
||||
name: "getTaskEventDetailedSummaryV2",
|
||||
table: "trigger_dev.task_events_v2",
|
||||
columns: [
|
||||
"span_id",
|
||||
"parent_span_id",
|
||||
"run_id",
|
||||
"start_time",
|
||||
"duration",
|
||||
"status",
|
||||
"kind",
|
||||
"metadata",
|
||||
{ name: "message", expression: "LEFT(message, 256)" },
|
||||
"attributes_text",
|
||||
],
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export function getSpanDetailsQueryBuilderV2(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilder({
|
||||
name: "getSpanDetailsV2",
|
||||
baseQuery:
|
||||
"SELECT span_id, parent_span_id, start_time, duration, status, kind, metadata, message, attributes_text FROM trigger_dev.task_events_v2",
|
||||
schema: TaskEventDetailsV1Result,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export function getTraceEventsForExportQueryBuilderV2(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilderFast<TaskEventExportRow>({
|
||||
name: "getTraceEventsForExportV2",
|
||||
table: "trigger_dev.task_events_v2",
|
||||
columns: [...TASK_EVENT_EXPORT_COLUMNS],
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Search Table Query Builders (for logs page, using task_events_search_v1)
|
||||
// ============================================================================
|
||||
|
||||
export const LogsSearchListResult = z.object({
|
||||
environment_id: z.string(),
|
||||
organization_id: z.string(),
|
||||
project_id: z.string(),
|
||||
task_identifier: z.string(),
|
||||
run_id: z.string(),
|
||||
start_time: z.string(),
|
||||
trace_id: z.string(),
|
||||
span_id: z.string(),
|
||||
parent_span_id: z.string(),
|
||||
message: z.string(),
|
||||
kind: z.string(),
|
||||
status: z.string(),
|
||||
duration: z.number().or(z.string()),
|
||||
attributes_text: z.string(),
|
||||
triggered_timestamp: z.string(),
|
||||
});
|
||||
|
||||
export type LogsSearchListResult = z.output<typeof LogsSearchListResult>;
|
||||
|
||||
export function getLogsSearchListQueryBuilder(ch: ClickhouseReader) {
|
||||
return ch.queryBuilderFast<LogsSearchListResult>({
|
||||
name: "getLogsSearchList",
|
||||
table: "trigger_dev.task_events_search_v1",
|
||||
columns: [
|
||||
"environment_id",
|
||||
"organization_id",
|
||||
"project_id",
|
||||
"task_identifier",
|
||||
"run_id",
|
||||
"start_time",
|
||||
"trace_id",
|
||||
"span_id",
|
||||
"parent_span_id",
|
||||
{ name: "message", expression: "LEFT(message, 512)" },
|
||||
"kind",
|
||||
"status",
|
||||
"duration",
|
||||
"attributes_text",
|
||||
"triggered_timestamp",
|
||||
],
|
||||
settings: {
|
||||
use_query_condition_cache: 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Single log detail query builder (for side panel)
|
||||
export const LogDetailV2Result = z.object({
|
||||
environment_id: z.string(),
|
||||
organization_id: z.string(),
|
||||
project_id: z.string(),
|
||||
task_identifier: z.string(),
|
||||
run_id: z.string(),
|
||||
start_time: z.string(),
|
||||
trace_id: z.string(),
|
||||
span_id: z.string(),
|
||||
parent_span_id: z.string(),
|
||||
message: z.string(),
|
||||
kind: z.string(),
|
||||
status: z.string(),
|
||||
duration: z.number().or(z.string()),
|
||||
attributes_text: z.string(),
|
||||
});
|
||||
|
||||
export type LogDetailV2Result = z.output<typeof LogDetailV2Result>;
|
||||
|
||||
export function getLogDetailQueryBuilderV2(ch: ClickhouseReader) {
|
||||
return ch.queryBuilderFast<LogDetailV2Result>({
|
||||
name: "getLogDetail",
|
||||
table: "trigger_dev.task_events_v2",
|
||||
columns: [
|
||||
"environment_id",
|
||||
"organization_id",
|
||||
"project_id",
|
||||
"task_identifier",
|
||||
"run_id",
|
||||
"start_time",
|
||||
"trace_id",
|
||||
"span_id",
|
||||
"parent_span_id",
|
||||
"message",
|
||||
"kind",
|
||||
"status",
|
||||
"duration",
|
||||
"attributes_text",
|
||||
],
|
||||
});
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,715 @@
|
||||
import type { ClickHouseSettings } from "@clickhouse/client";
|
||||
import { z } from "zod";
|
||||
import type { ClickhouseReader, ClickhouseWriter } from "./client/types.js";
|
||||
|
||||
export const TaskRunV2 = z.object({
|
||||
environment_id: z.string(),
|
||||
organization_id: z.string(),
|
||||
project_id: z.string(),
|
||||
run_id: z.string(),
|
||||
updated_at: z.number().int(),
|
||||
created_at: z.number().int(),
|
||||
status: z.string(),
|
||||
environment_type: z.string(),
|
||||
friendly_id: z.string(),
|
||||
attempt: z.number().int().default(1),
|
||||
engine: z.string(),
|
||||
task_identifier: z.string(),
|
||||
queue: z.string(),
|
||||
schedule_id: z.string(),
|
||||
batch_id: z.string(),
|
||||
completed_at: z.number().int().nullish(),
|
||||
started_at: z.number().int().nullish(),
|
||||
executed_at: z.number().int().nullish(),
|
||||
delay_until: z.number().int().nullish(),
|
||||
queued_at: z.number().int().nullish(),
|
||||
expired_at: z.number().int().nullish(),
|
||||
usage_duration_ms: z.number().int().default(0),
|
||||
cost_in_cents: z.number().default(0),
|
||||
base_cost_in_cents: z.number().default(0),
|
||||
output: z.unknown(),
|
||||
error: z.unknown(),
|
||||
error_fingerprint: z.string().default(""),
|
||||
tags: z.array(z.string()).default([]),
|
||||
task_version: z.string(),
|
||||
sdk_version: z.string(),
|
||||
cli_version: z.string(),
|
||||
machine_preset: z.string(),
|
||||
root_run_id: z.string(),
|
||||
parent_run_id: z.string(),
|
||||
depth: z.number().int().default(0),
|
||||
span_id: z.string(),
|
||||
trace_id: z.string(),
|
||||
idempotency_key: z.string(),
|
||||
idempotency_key_user: z.string().default(""),
|
||||
idempotency_key_scope: z.string().default(""),
|
||||
expiration_ttl: z.string(),
|
||||
is_test: z.boolean().default(false),
|
||||
concurrency_key: z.string().default(""),
|
||||
bulk_action_group_ids: z.array(z.string()).default([]),
|
||||
worker_queue: z.string().default(""),
|
||||
region: z.string().default(""),
|
||||
plan_type: z.string().default(""),
|
||||
max_duration_in_seconds: z.number().int().nullish(),
|
||||
trigger_source: z.string().default(""),
|
||||
root_trigger_source: z.string().default(""),
|
||||
task_kind: z.string().default(""),
|
||||
is_warm_start: z.boolean().nullish(),
|
||||
_version: z.string(),
|
||||
_is_deleted: z.number().int().default(0),
|
||||
});
|
||||
|
||||
export type TaskRunV2 = z.input<typeof TaskRunV2>;
|
||||
|
||||
// Column order for compact format - must match ClickHouse table schema
|
||||
export const TASK_RUN_COLUMNS = [
|
||||
"environment_id",
|
||||
"organization_id",
|
||||
"project_id",
|
||||
"run_id",
|
||||
"updated_at",
|
||||
"created_at",
|
||||
"status",
|
||||
"environment_type",
|
||||
"friendly_id",
|
||||
"attempt",
|
||||
"engine",
|
||||
"task_identifier",
|
||||
"queue",
|
||||
"schedule_id",
|
||||
"batch_id",
|
||||
"completed_at",
|
||||
"started_at",
|
||||
"executed_at",
|
||||
"delay_until",
|
||||
"queued_at",
|
||||
"expired_at",
|
||||
"usage_duration_ms",
|
||||
"cost_in_cents",
|
||||
"base_cost_in_cents",
|
||||
"output",
|
||||
"error",
|
||||
"error_fingerprint",
|
||||
"tags",
|
||||
"task_version",
|
||||
"sdk_version",
|
||||
"cli_version",
|
||||
"machine_preset",
|
||||
"root_run_id",
|
||||
"parent_run_id",
|
||||
"depth",
|
||||
"span_id",
|
||||
"trace_id",
|
||||
"idempotency_key",
|
||||
"idempotency_key_user",
|
||||
"idempotency_key_scope",
|
||||
"expiration_ttl",
|
||||
"is_test",
|
||||
"_version",
|
||||
"_is_deleted",
|
||||
"concurrency_key",
|
||||
"bulk_action_group_ids",
|
||||
"worker_queue",
|
||||
"region",
|
||||
"plan_type",
|
||||
"max_duration_in_seconds",
|
||||
"trigger_source",
|
||||
"root_trigger_source",
|
||||
"task_kind",
|
||||
"is_warm_start",
|
||||
] as const;
|
||||
|
||||
export type TaskRunColumnName = (typeof TASK_RUN_COLUMNS)[number];
|
||||
|
||||
// Type-safe column indices generated from TASK_RUN_COLUMNS
|
||||
// This ensures indices stay in sync with column order automatically
|
||||
export const TASK_RUN_INDEX = Object.fromEntries(
|
||||
TASK_RUN_COLUMNS.map((col, idx) => [col, idx])
|
||||
) as { readonly [K in TaskRunColumnName]: number };
|
||||
|
||||
/**
|
||||
* Type mapping from column name to its type in TaskRunInsertArray.
|
||||
* This enables type-safe field access without manual casting.
|
||||
*/
|
||||
export type TaskRunFieldTypes = {
|
||||
environment_id: string;
|
||||
organization_id: string;
|
||||
project_id: string;
|
||||
run_id: string;
|
||||
updated_at: number;
|
||||
created_at: number;
|
||||
status: string;
|
||||
environment_type: string;
|
||||
friendly_id: string;
|
||||
attempt: number;
|
||||
engine: string;
|
||||
task_identifier: string;
|
||||
queue: string;
|
||||
schedule_id: string;
|
||||
batch_id: string;
|
||||
completed_at: number | null;
|
||||
started_at: number | null;
|
||||
executed_at: number | null;
|
||||
delay_until: number | null;
|
||||
queued_at: number | null;
|
||||
expired_at: number | null;
|
||||
usage_duration_ms: number;
|
||||
cost_in_cents: number;
|
||||
base_cost_in_cents: number;
|
||||
output: { data: unknown };
|
||||
error: { data: unknown };
|
||||
error_fingerprint: string;
|
||||
tags: string[];
|
||||
task_version: string;
|
||||
sdk_version: string;
|
||||
cli_version: string;
|
||||
machine_preset: string;
|
||||
root_run_id: string;
|
||||
parent_run_id: string;
|
||||
depth: number;
|
||||
span_id: string;
|
||||
trace_id: string;
|
||||
idempotency_key: string;
|
||||
idempotency_key_user: string;
|
||||
idempotency_key_scope: string;
|
||||
expiration_ttl: string;
|
||||
is_test: boolean;
|
||||
_version: string;
|
||||
_is_deleted: number;
|
||||
concurrency_key: string;
|
||||
bulk_action_group_ids: string[];
|
||||
worker_queue: string;
|
||||
region: string;
|
||||
plan_type: string;
|
||||
max_duration_in_seconds: number | null;
|
||||
trigger_source: string;
|
||||
root_trigger_source: string;
|
||||
task_kind: string;
|
||||
is_warm_start: boolean | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Type-safe accessor for TaskRunInsertArray fields.
|
||||
* Returns the correct type for each field without manual casting.
|
||||
*
|
||||
* @example
|
||||
* const orgId = getTaskRunField(run, "organization_id"); // type: string
|
||||
* const createdAt = getTaskRunField(run, "created_at"); // type: number
|
||||
*/
|
||||
export function getTaskRunField<K extends TaskRunColumnName>(
|
||||
run: TaskRunInsertArray,
|
||||
field: K
|
||||
): TaskRunFieldTypes[K] {
|
||||
return run[TASK_RUN_INDEX[field]] as TaskRunFieldTypes[K];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose a globally-comparable ReplacingMergeTree version for task_runs_v2
|
||||
* when the same run can be replicated from more than one Postgres producer.
|
||||
*
|
||||
* Each producer has its own, mutually-incomparable LSN space, so the raw
|
||||
* LSN-derived version cannot be compared across producers. We reserve the top
|
||||
* 8 bits for an `originGeneration` epoch (monotonic across producers: the more
|
||||
* authoritative / later-cutover producer gets the higher generation) and keep
|
||||
* the producer's own LSN in the low 56 bits to preserve in-producer ordering.
|
||||
*
|
||||
* Self-host single-DB never calls this (one producer => generation is constant
|
||||
* and the existing raw LSN path is sufficient); the split gate skips it.
|
||||
*/
|
||||
export function composeTaskRunVersion(opts: {
|
||||
originGeneration: number;
|
||||
lsnVersion: bigint;
|
||||
}): bigint {
|
||||
const gen = BigInt(opts.originGeneration);
|
||||
if (gen < BigInt(0) || gen > BigInt(0xff)) {
|
||||
throw new Error(`originGeneration out of range (0-255): ${opts.originGeneration}`);
|
||||
}
|
||||
const LSN_BITS = BigInt(56);
|
||||
const LSN_MASK = (BigInt(1) << LSN_BITS) - BigInt(1); // low 56 bits
|
||||
return (gen << LSN_BITS) | (opts.lsnVersion & LSN_MASK);
|
||||
}
|
||||
|
||||
export function insertTaskRunsCompactArrays(ch: ClickhouseWriter, settings?: ClickHouseSettings) {
|
||||
return ch.insertCompactRaw({
|
||||
name: "insertTaskRunsCompactArrays",
|
||||
table: "trigger_dev.task_runs_v2",
|
||||
columns: TASK_RUN_COLUMNS,
|
||||
settings: {
|
||||
enable_json_type: 1,
|
||||
type_json_skip_duplicated_paths: 1,
|
||||
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
|
||||
...settings,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Object-based insert function for tests and non-performance-critical code
|
||||
export function insertTaskRuns(ch: ClickhouseWriter, settings?: ClickHouseSettings) {
|
||||
return ch.insert({
|
||||
name: "insertTaskRuns",
|
||||
table: "trigger_dev.task_runs_v2",
|
||||
schema: TaskRunV2,
|
||||
settings: {
|
||||
enable_json_type: 1,
|
||||
type_json_skip_duplicated_paths: 1,
|
||||
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
|
||||
...settings,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export const RawTaskRunPayloadV1 = z.object({
|
||||
run_id: z.string(),
|
||||
created_at: z.number().int(),
|
||||
payload: z.unknown(),
|
||||
});
|
||||
|
||||
export type RawTaskRunPayloadV1 = z.infer<typeof RawTaskRunPayloadV1>;
|
||||
|
||||
export const PAYLOAD_COLUMNS = ["run_id", "created_at", "payload"] as const;
|
||||
|
||||
export type PayloadColumnName = (typeof PAYLOAD_COLUMNS)[number];
|
||||
|
||||
// Type-safe column indices generated from PAYLOAD_COLUMNS
|
||||
export const PAYLOAD_INDEX = Object.fromEntries(PAYLOAD_COLUMNS.map((col, idx) => [col, idx])) as {
|
||||
readonly [K in PayloadColumnName]: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Type mapping from column name to its type in PayloadInsertArray.
|
||||
*/
|
||||
export type PayloadFieldTypes = {
|
||||
run_id: string;
|
||||
created_at: number;
|
||||
payload: { data: unknown };
|
||||
};
|
||||
|
||||
/**
|
||||
* Type-safe accessor for PayloadInsertArray fields.
|
||||
* Returns the correct type for each field without manual casting.
|
||||
*/
|
||||
export function getPayloadField<K extends PayloadColumnName>(
|
||||
payload: PayloadInsertArray,
|
||||
field: K
|
||||
): PayloadFieldTypes[K] {
|
||||
return payload[PAYLOAD_INDEX[field]] as PayloadFieldTypes[K];
|
||||
}
|
||||
|
||||
/**
|
||||
* Type-safe tuple representing a task run insert array.
|
||||
* Order matches TASK_RUN_COLUMNS exactly.
|
||||
*/
|
||||
export type TaskRunInsertArray = [
|
||||
environment_id: string,
|
||||
organization_id: string,
|
||||
project_id: string,
|
||||
run_id: string,
|
||||
updated_at: number,
|
||||
created_at: number,
|
||||
status: string,
|
||||
environment_type: string,
|
||||
friendly_id: string,
|
||||
attempt: number,
|
||||
engine: string,
|
||||
task_identifier: string,
|
||||
queue: string,
|
||||
schedule_id: string,
|
||||
batch_id: string,
|
||||
completed_at: number | null,
|
||||
started_at: number | null,
|
||||
executed_at: number | null,
|
||||
delay_until: number | null,
|
||||
queued_at: number | null,
|
||||
expired_at: number | null,
|
||||
usage_duration_ms: number,
|
||||
cost_in_cents: number,
|
||||
base_cost_in_cents: number,
|
||||
output: { data: unknown },
|
||||
error: { data: unknown },
|
||||
error_fingerprint: string,
|
||||
tags: string[],
|
||||
task_version: string,
|
||||
sdk_version: string,
|
||||
cli_version: string,
|
||||
machine_preset: string,
|
||||
root_run_id: string,
|
||||
parent_run_id: string,
|
||||
depth: number,
|
||||
span_id: string,
|
||||
trace_id: string,
|
||||
idempotency_key: string,
|
||||
idempotency_key_user: string,
|
||||
idempotency_key_scope: string,
|
||||
expiration_ttl: string,
|
||||
is_test: boolean,
|
||||
_version: string,
|
||||
_is_deleted: number,
|
||||
concurrency_key: string,
|
||||
bulk_action_group_ids: string[],
|
||||
worker_queue: string,
|
||||
region: string,
|
||||
plan_type: string,
|
||||
max_duration_in_seconds: number | null,
|
||||
trigger_source: string,
|
||||
root_trigger_source: string,
|
||||
task_kind: string,
|
||||
is_warm_start: boolean | null,
|
||||
];
|
||||
|
||||
/**
|
||||
* Type-safe tuple representing a payload insert array.
|
||||
* Order matches PAYLOAD_COLUMNS exactly.
|
||||
*/
|
||||
export type PayloadInsertArray = [run_id: string, created_at: number, payload: { data: unknown }];
|
||||
|
||||
export function insertRawTaskRunPayloadsCompactArrays(
|
||||
ch: ClickhouseWriter,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.insertCompactRaw({
|
||||
name: "insertRawTaskRunPayloadsCompactArrays",
|
||||
table: "trigger_dev.raw_task_runs_payload_v1",
|
||||
columns: PAYLOAD_COLUMNS,
|
||||
settings: {
|
||||
async_insert: 1,
|
||||
wait_for_async_insert: 0,
|
||||
async_insert_max_data_size: "1000000",
|
||||
async_insert_busy_timeout_ms: 1000,
|
||||
enable_json_type: 1,
|
||||
type_json_skip_duplicated_paths: 1,
|
||||
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
|
||||
...settings,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Object-based insert function for tests and non-performance-critical code
|
||||
export function insertRawTaskRunPayloads(ch: ClickhouseWriter, settings?: ClickHouseSettings) {
|
||||
return ch.insert({
|
||||
name: "insertRawTaskRunPayloads",
|
||||
table: "trigger_dev.raw_task_runs_payload_v1",
|
||||
schema: RawTaskRunPayloadV1,
|
||||
settings: {
|
||||
async_insert: 1,
|
||||
wait_for_async_insert: 0,
|
||||
async_insert_max_data_size: "1000000",
|
||||
async_insert_busy_timeout_ms: 1000,
|
||||
enable_json_type: 1,
|
||||
type_json_skip_duplicated_paths: 1,
|
||||
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
|
||||
...settings,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export const TaskRunV2QueryResult = z.object({
|
||||
run_id: z.string(),
|
||||
});
|
||||
|
||||
export type TaskRunV2QueryResult = z.infer<typeof TaskRunV2QueryResult>;
|
||||
|
||||
// Adds the created_at timestamp (ms since epoch) needed to build composite
|
||||
// keyset cursors over (created_at, run_id) — see runsRepository.server.ts.
|
||||
// Returned as a JSON number because the client sets
|
||||
// output_format_json_quote_64bit_integers: 0. Kept separate from
|
||||
// TaskRunV2QueryResult so run_id-only consumers (e.g. the pending-version
|
||||
// lookup) aren't forced to select a column they don't need.
|
||||
export const TaskRunListQueryResult = z.object({
|
||||
run_id: z.string(),
|
||||
created_at_ms: z.number().int(),
|
||||
});
|
||||
|
||||
export type TaskRunListQueryResult = z.infer<typeof TaskRunListQueryResult>;
|
||||
|
||||
export function getTaskRunsQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilder({
|
||||
name: "getTaskRuns",
|
||||
baseQuery:
|
||||
"SELECT run_id, toUnixTimestamp64Milli(created_at) AS created_at_ms FROM trigger_dev.task_runs_v2 FINAL",
|
||||
schema: TaskRunListQueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookup builder for the run-engine `PendingVersionSystem`. Returns just
|
||||
* `run_id` from `task_runs_v2`. No `FINAL` — the run-engine re-validates
|
||||
* each candidate against Postgres by primary key, so a stale
|
||||
* `PENDING_VERSION` row from a not-yet-merged part is harmless and
|
||||
* `FINAL` would be too expensive for this hot path.
|
||||
*/
|
||||
export function getPendingVersionIdsQueryBuilder(
|
||||
ch: ClickhouseReader,
|
||||
settings?: ClickHouseSettings
|
||||
) {
|
||||
return ch.queryBuilder({
|
||||
name: "getPendingVersionIds",
|
||||
baseQuery: "SELECT run_id FROM trigger_dev.task_runs_v2",
|
||||
schema: TaskRunV2QueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export function getTaskRunsCountQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilder({
|
||||
name: "getTaskRunsCount",
|
||||
baseQuery: "SELECT count() as count FROM trigger_dev.task_runs_v2 FINAL",
|
||||
schema: z.object({
|
||||
count: z.number().int(),
|
||||
}),
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const TaskRunExistsQueryResult = z.object({
|
||||
run_exists: z.number().int(),
|
||||
});
|
||||
|
||||
export type TaskRunExistsQueryResult = z.infer<typeof TaskRunExistsQueryResult>;
|
||||
|
||||
// Empty-state existence probe. No FINAL (a stale/dup row still answers "a run exists").
|
||||
// Callers must filter organization_id + project_id + environment_id (the sort-key prefix).
|
||||
export function getTaskRunExistsQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilder({
|
||||
name: "getTaskRunExists",
|
||||
baseQuery: "SELECT 1 AS run_exists FROM trigger_dev.task_runs_v2",
|
||||
schema: TaskRunExistsQueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const TaskRunTagsQueryResult = z.object({
|
||||
tag: z.string(),
|
||||
});
|
||||
|
||||
export type TaskRunTagsQueryResult = z.infer<typeof TaskRunTagsQueryResult>;
|
||||
|
||||
export function getTaskRunTagsQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.queryBuilder({
|
||||
name: "getTaskRunTags",
|
||||
baseQuery: "SELECT DISTINCT arrayJoin(tags) as tag FROM trigger_dev.task_runs_v2",
|
||||
schema: TaskRunTagsQueryResult,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const TaskActivityQueryResult = z.object({
|
||||
task_identifier: z.string(),
|
||||
status: z.string(),
|
||||
day: z.string(),
|
||||
count: z.number().int(),
|
||||
});
|
||||
|
||||
export type TaskActivityQueryResult = z.infer<typeof TaskActivityQueryResult>;
|
||||
|
||||
export const TaskActivityQueryParams = z.object({
|
||||
organizationId: z.string(),
|
||||
projectId: z.string(),
|
||||
environmentId: z.string(),
|
||||
days: z.number().int(),
|
||||
});
|
||||
|
||||
export function getTaskActivityQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.query({
|
||||
name: "getTaskActivity",
|
||||
query: `
|
||||
SELECT
|
||||
task_identifier,
|
||||
status,
|
||||
toDate(created_at) as day,
|
||||
count() as count
|
||||
FROM trigger_dev.task_runs_v2 FINAL
|
||||
WHERE
|
||||
organization_id = {organizationId: String}
|
||||
AND project_id = {projectId: String}
|
||||
AND environment_id = {environmentId: String}
|
||||
AND created_at >= today() - {days: Int64}
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
task_identifier,
|
||||
status,
|
||||
day
|
||||
ORDER BY
|
||||
task_identifier ASC,
|
||||
day ASC,
|
||||
status ASC
|
||||
`,
|
||||
schema: TaskActivityQueryResult,
|
||||
params: TaskActivityQueryParams,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const CurrentRunningStatsQueryResult = z.object({
|
||||
task_identifier: z.string(),
|
||||
status: z.string(),
|
||||
count: z.number().int(),
|
||||
});
|
||||
|
||||
export type CurrentRunningStatsQueryResult = z.infer<typeof CurrentRunningStatsQueryResult>;
|
||||
|
||||
export const CurrentRunningStatsQueryParams = z.object({
|
||||
organizationId: z.string(),
|
||||
projectId: z.string(),
|
||||
environmentId: z.string(),
|
||||
days: z.number().int(),
|
||||
});
|
||||
|
||||
export function getCurrentRunningStats(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.query({
|
||||
name: "getCurrentRunningStats",
|
||||
query: `
|
||||
SELECT
|
||||
task_identifier,
|
||||
status,
|
||||
count() as count
|
||||
FROM trigger_dev.task_runs_v2 FINAL
|
||||
WHERE
|
||||
organization_id = {organizationId: String}
|
||||
AND project_id = {projectId: String}
|
||||
AND environment_id = {environmentId: String}
|
||||
AND status IN ('PENDING', 'WAITING_FOR_DEPLOY', 'WAITING_TO_RESUME', 'QUEUED', 'EXECUTING', 'DELAYED')
|
||||
AND _is_deleted = 0
|
||||
AND created_at >= now() - INTERVAL {days: Int64} DAY
|
||||
GROUP BY
|
||||
task_identifier,
|
||||
status
|
||||
ORDER BY
|
||||
task_identifier ASC
|
||||
`,
|
||||
schema: CurrentRunningStatsQueryResult,
|
||||
params: CurrentRunningStatsQueryParams,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const ChildRunStatusCountsQueryResult = z.object({
|
||||
root_run_id: z.string(),
|
||||
status: z.string(),
|
||||
count: z.number().int(),
|
||||
});
|
||||
|
||||
export type ChildRunStatusCountsQueryResult = z.infer<typeof ChildRunStatusCountsQueryResult>;
|
||||
|
||||
export const ChildRunStatusCountsQueryParams = z.object({
|
||||
organizationId: z.string(),
|
||||
projectId: z.string(),
|
||||
environmentId: z.string(),
|
||||
rootRunIds: z.array(z.string()).min(1),
|
||||
since: z.number().int(),
|
||||
});
|
||||
|
||||
export function getChildRunStatusCounts(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.query({
|
||||
name: "getChildRunStatusCounts",
|
||||
query: `
|
||||
SELECT
|
||||
root_run_id,
|
||||
status,
|
||||
count() as count
|
||||
FROM trigger_dev.task_runs_v2 FINAL
|
||||
WHERE
|
||||
organization_id = {organizationId: String}
|
||||
AND project_id = {projectId: String}
|
||||
AND environment_id = {environmentId: String}
|
||||
AND root_run_id IN {rootRunIds: Array(String)}
|
||||
AND created_at >= fromUnixTimestamp64Milli({since: Int64})
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
root_run_id,
|
||||
status
|
||||
ORDER BY
|
||||
root_run_id ASC,
|
||||
status ASC
|
||||
`,
|
||||
schema: ChildRunStatusCountsQueryResult,
|
||||
params: ChildRunStatusCountsQueryParams,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const AverageDurationsQueryResult = z.object({
|
||||
task_identifier: z.string(),
|
||||
duration: z.number(),
|
||||
});
|
||||
|
||||
export type AverageDurationsQueryResult = z.infer<typeof AverageDurationsQueryResult>;
|
||||
|
||||
export const AverageDurationsQueryParams = z.object({
|
||||
organizationId: z.string(),
|
||||
projectId: z.string(),
|
||||
environmentId: z.string(),
|
||||
days: z.number().int(),
|
||||
});
|
||||
|
||||
export function getAverageDurations(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.query({
|
||||
name: "getAverageDurations",
|
||||
query: `
|
||||
SELECT
|
||||
task_identifier,
|
||||
avg(toUnixTimestamp(completed_at) - toUnixTimestamp(started_at)) as duration
|
||||
FROM trigger_dev.task_runs_v2 FINAL
|
||||
WHERE
|
||||
organization_id = {organizationId: String}
|
||||
AND project_id = {projectId: String}
|
||||
AND environment_id = {environmentId: String}
|
||||
AND created_at >= today() - {days: Int64}
|
||||
AND status IN ('COMPLETED_SUCCESSFULLY', 'COMPLETED_WITH_ERRORS')
|
||||
AND started_at IS NOT NULL
|
||||
AND completed_at IS NOT NULL
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
task_identifier
|
||||
`,
|
||||
schema: AverageDurationsQueryResult,
|
||||
params: AverageDurationsQueryParams,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
||||
export const TaskUsageByOrganizationQueryResult = z.object({
|
||||
task_identifier: z.string(),
|
||||
run_count: z.number(),
|
||||
average_duration: z.number(),
|
||||
total_duration: z.number(),
|
||||
average_cost: z.number(),
|
||||
total_cost: z.number(),
|
||||
total_base_cost: z.number(),
|
||||
});
|
||||
|
||||
export const TaskUsageByOrganizationQueryParams = z.object({
|
||||
startTime: z.number().int(),
|
||||
endTime: z.number().int(),
|
||||
organizationId: z.string(),
|
||||
});
|
||||
|
||||
export function getTaskUsageByOrganization(ch: ClickhouseReader, settings?: ClickHouseSettings) {
|
||||
return ch.query({
|
||||
name: "getTaskUsageByOrganization",
|
||||
query: `
|
||||
SELECT
|
||||
task_identifier,
|
||||
count() AS run_count,
|
||||
avg(usage_duration_ms) AS average_duration,
|
||||
sum(usage_duration_ms) AS total_duration,
|
||||
avg(cost_in_cents) / 100.0 AS average_cost,
|
||||
sum(cost_in_cents) / 100.0 AS total_cost,
|
||||
sum(base_cost_in_cents) / 100.0 AS total_base_cost
|
||||
FROM trigger_dev.task_runs_v2 FINAL
|
||||
WHERE
|
||||
environment_type != 'DEVELOPMENT'
|
||||
AND created_at >= fromUnixTimestamp64Milli({startTime: Int64})
|
||||
AND created_at < fromUnixTimestamp64Milli({endTime: Int64})
|
||||
AND organization_id = {organizationId: String}
|
||||
AND _is_deleted = 0
|
||||
GROUP BY
|
||||
task_identifier
|
||||
ORDER BY
|
||||
total_cost DESC
|
||||
`,
|
||||
schema: TaskUsageByOrganizationQueryResult,
|
||||
params: TaskUsageByOrganizationQueryParams,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,864 @@
|
||||
import { clickhouseTest } from "@internal/testcontainers";
|
||||
import { z } from "zod";
|
||||
import { ClickhouseClient } from "./client/client.js";
|
||||
import { executeTSQL, type TableSchema } from "./client/tsql.js";
|
||||
import { insertTaskRuns } from "./taskRuns.js";
|
||||
import { column } from "@internal/tsql";
|
||||
|
||||
/**
|
||||
* Schema definition for task_runs table used in function tests.
|
||||
* Includes numeric, string, datetime, and array columns for exercising all function categories.
|
||||
*/
|
||||
const taskRunsSchema: TableSchema = {
|
||||
name: "task_runs",
|
||||
clickhouseName: "trigger_dev.task_runs_v2",
|
||||
columns: {
|
||||
run_id: { name: "run_id", ...column("String") },
|
||||
friendly_id: { name: "friendly_id", ...column("String") },
|
||||
status: { name: "status", ...column("String") },
|
||||
task_identifier: { name: "task_identifier", ...column("String") },
|
||||
queue: { name: "queue", ...column("String") },
|
||||
environment_id: { name: "environment_id", ...column("String") },
|
||||
environment_type: { name: "environment_type", ...column("String") },
|
||||
organization_id: { name: "organization_id", ...column("String") },
|
||||
project_id: { name: "project_id", ...column("String") },
|
||||
created_at: { name: "created_at", ...column("DateTime64") },
|
||||
updated_at: { name: "updated_at", ...column("DateTime64") },
|
||||
started_at: { name: "started_at", ...column("Nullable(DateTime64)") },
|
||||
completed_at: { name: "completed_at", ...column("Nullable(DateTime64)") },
|
||||
is_test: { name: "is_test", ...column("UInt8") },
|
||||
tags: { name: "tags", ...column("Array(String)") },
|
||||
output: {
|
||||
name: "output",
|
||||
...column("JSON"),
|
||||
nullValue: "'{}'",
|
||||
textColumn: "output_text",
|
||||
dataPrefix: "data",
|
||||
},
|
||||
usage_duration_ms: { name: "usage_duration_ms", ...column("UInt32") },
|
||||
cost_in_cents: { name: "cost_in_cents", ...column("Float64") },
|
||||
attempt: { name: "attempt", ...column("UInt8") },
|
||||
depth: { name: "depth", ...column("UInt8") },
|
||||
},
|
||||
tenantColumns: {
|
||||
organizationId: "organization_id",
|
||||
projectId: "project_id",
|
||||
environmentId: "environment_id",
|
||||
},
|
||||
};
|
||||
|
||||
const enforcedWhereClause = {
|
||||
organization_id: { op: "eq" as const, value: "org_tenant1" },
|
||||
project_id: { op: "eq" as const, value: "proj_tenant1" },
|
||||
environment_id: { op: "eq" as const, value: "env_tenant1" },
|
||||
};
|
||||
|
||||
const defaultTaskRun = {
|
||||
environment_id: "env_tenant1",
|
||||
environment_type: "DEVELOPMENT",
|
||||
organization_id: "org_tenant1",
|
||||
project_id: "proj_tenant1",
|
||||
run_id: "run_func_test_1",
|
||||
friendly_id: "friendly_func_test_1",
|
||||
attempt: 1,
|
||||
engine: "V2",
|
||||
status: "COMPLETED_SUCCESSFULLY",
|
||||
task_identifier: "my-task",
|
||||
queue: "my-queue",
|
||||
schedule_id: "",
|
||||
batch_id: "",
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
started_at: Date.now() - 5000,
|
||||
completed_at: Date.now(),
|
||||
tags: ["tag-a", "tag-b"],
|
||||
output: { data: { count: 42, label: "ok", ratio: 1.5, enabled: true, items: [1, 2, 3] } },
|
||||
error: null,
|
||||
usage_duration_ms: 4500,
|
||||
cost_in_cents: 1.5,
|
||||
base_cost_in_cents: 0.5,
|
||||
task_version: "1.0.0",
|
||||
sdk_version: "4.0.0",
|
||||
cli_version: "4.0.0",
|
||||
machine_preset: "small-1x",
|
||||
is_test: false,
|
||||
span_id: "span_123",
|
||||
trace_id: "trace_123",
|
||||
idempotency_key: "idem_123",
|
||||
expiration_ttl: "",
|
||||
root_run_id: "",
|
||||
parent_run_id: "",
|
||||
depth: 2,
|
||||
concurrency_key: "",
|
||||
bulk_action_group_ids: [] as string[],
|
||||
_version: "1",
|
||||
};
|
||||
|
||||
/**
|
||||
* Helper: execute a TSQL query and assert no errors.
|
||||
*/
|
||||
async function assertQueryExecutes(client: ClickhouseClient, tsqlQuery: string): Promise<void> {
|
||||
const [error] = await executeTSQL(client, {
|
||||
name: "func-test",
|
||||
query: tsqlQuery,
|
||||
schema: z.record(z.any()),
|
||||
enforcedWhereClause,
|
||||
tableSchema: [taskRunsSchema],
|
||||
});
|
||||
|
||||
if (error) {
|
||||
throw new Error(`Query failed: ${tsqlQuery}\n\nError: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: set up a client with test data inserted.
|
||||
*/
|
||||
async function setupClient(clickhouseContainer: { getConnectionUrl(): string }) {
|
||||
const client = new ClickhouseClient({
|
||||
name: "func-test",
|
||||
url: clickhouseContainer.getConnectionUrl(),
|
||||
});
|
||||
|
||||
const insert = insertTaskRuns(client, { async_insert: 0 });
|
||||
const [insertError] = await insert([defaultTaskRun]);
|
||||
expect(insertError).toBeNull();
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: run all test cases in a single ClickHouse container.
|
||||
* Each case is a [name, tsqlQuery] tuple.
|
||||
*/
|
||||
async function runCases(client: ClickhouseClient, cases: [string, string][]): Promise<void> {
|
||||
const failures: string[] = [];
|
||||
|
||||
for (const [name, query] of cases) {
|
||||
try {
|
||||
await assertQueryExecutes(client, query);
|
||||
} catch (e) {
|
||||
failures.push(` ${name}: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
throw new Error(
|
||||
`${failures.length}/${cases.length} function(s) failed:\n${failures.join("\n")}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const url = "https://user:pass@www.example.com:8080/path/page?q=1&r=2#frag";
|
||||
|
||||
describe("TSQL Function Smoke Tests", () => {
|
||||
// ─── Arithmetic functions ─────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Arithmetic functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["plus", "SELECT plus(usage_duration_ms, 1) AS r FROM task_runs"],
|
||||
["minus", "SELECT minus(usage_duration_ms, 1) AS r FROM task_runs"],
|
||||
["multiply", "SELECT multiply(usage_duration_ms, 2) AS r FROM task_runs"],
|
||||
["divide", "SELECT divide(usage_duration_ms, 2) AS r FROM task_runs"],
|
||||
["intDiv", "SELECT intDiv(usage_duration_ms, 2) AS r FROM task_runs"],
|
||||
["intDivOrZero", "SELECT intDivOrZero(usage_duration_ms, 0) AS r FROM task_runs"],
|
||||
["modulo", "SELECT modulo(usage_duration_ms, 3) AS r FROM task_runs"],
|
||||
["moduloOrZero", "SELECT moduloOrZero(usage_duration_ms, 0) AS r FROM task_runs"],
|
||||
["positiveModulo", "SELECT positiveModulo(usage_duration_ms, 3) AS r FROM task_runs"],
|
||||
["negate", "SELECT negate(cost_in_cents) AS r FROM task_runs"],
|
||||
["abs", "SELECT abs(cost_in_cents) AS r FROM task_runs"],
|
||||
["gcd", "SELECT gcd(12, 8) AS r FROM task_runs"],
|
||||
["lcm", "SELECT lcm(12, 8) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Mathematical functions ───────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Mathematical functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["exp", "SELECT exp(1) AS r FROM task_runs"],
|
||||
["log", "SELECT log(2.718) AS r FROM task_runs"],
|
||||
["ln", "SELECT ln(2.718) AS r FROM task_runs"],
|
||||
["exp2", "SELECT exp2(3) AS r FROM task_runs"],
|
||||
["log2", "SELECT log2(8) AS r FROM task_runs"],
|
||||
["exp10", "SELECT exp10(2) AS r FROM task_runs"],
|
||||
["log10", "SELECT log10(100) AS r FROM task_runs"],
|
||||
["sqrt", "SELECT sqrt(16) AS r FROM task_runs"],
|
||||
["cbrt", "SELECT cbrt(27) AS r FROM task_runs"],
|
||||
["erf", "SELECT erf(1) AS r FROM task_runs"],
|
||||
["erfc", "SELECT erfc(1) AS r FROM task_runs"],
|
||||
["lgamma", "SELECT lgamma(5) AS r FROM task_runs"],
|
||||
["tgamma", "SELECT tgamma(5) AS r FROM task_runs"],
|
||||
["sin", "SELECT sin(1) AS r FROM task_runs"],
|
||||
["cos", "SELECT cos(1) AS r FROM task_runs"],
|
||||
["tan", "SELECT tan(1) AS r FROM task_runs"],
|
||||
["asin", "SELECT asin(0.5) AS r FROM task_runs"],
|
||||
["acos", "SELECT acos(0.5) AS r FROM task_runs"],
|
||||
["atan", "SELECT atan(1) AS r FROM task_runs"],
|
||||
["pow", "SELECT pow(2, 3) AS r FROM task_runs"],
|
||||
["power", "SELECT power(2, 3) AS r FROM task_runs"],
|
||||
["round", "SELECT round(3.14159, 2) AS r FROM task_runs"],
|
||||
["floor", "SELECT floor(3.7) AS r FROM task_runs"],
|
||||
["ceil", "SELECT ceil(3.2) AS r FROM task_runs"],
|
||||
["ceiling", "SELECT ceiling(3.2) AS r FROM task_runs"],
|
||||
["trunc", "SELECT trunc(3.7) AS r FROM task_runs"],
|
||||
["truncate", "SELECT truncate(3.7) AS r FROM task_runs"],
|
||||
["sign", "SELECT sign(-5) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── String functions ─────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("String functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["empty", "SELECT empty(status) AS r FROM task_runs"],
|
||||
["notEmpty", "SELECT notEmpty(status) AS r FROM task_runs"],
|
||||
["length", "SELECT length(status) AS r FROM task_runs"],
|
||||
["lengthUTF8", "SELECT lengthUTF8(status) AS r FROM task_runs"],
|
||||
["char_length", "SELECT char_length(status) AS r FROM task_runs"],
|
||||
["character_length", "SELECT character_length(status) AS r FROM task_runs"],
|
||||
["lower", "SELECT lower(status) AS r FROM task_runs"],
|
||||
["upper", "SELECT upper(status) AS r FROM task_runs"],
|
||||
["lowerUTF8", "SELECT lowerUTF8(status) AS r FROM task_runs"],
|
||||
["upperUTF8", "SELECT upperUTF8(status) AS r FROM task_runs"],
|
||||
["reverse", "SELECT reverse(status) AS r FROM task_runs"],
|
||||
["reverseUTF8", "SELECT reverseUTF8(status) AS r FROM task_runs"],
|
||||
["concat", "SELECT concat(status, '-', run_id) AS r FROM task_runs"],
|
||||
["substring", "SELECT substring(status, 1, 3) AS r FROM task_runs"],
|
||||
["substr", "SELECT substr(status, 1, 3) AS r FROM task_runs"],
|
||||
["mid", "SELECT mid(status, 1, 3) AS r FROM task_runs"],
|
||||
["substringUTF8", "SELECT substringUTF8(status, 1, 3) AS r FROM task_runs"],
|
||||
[
|
||||
"appendTrailingCharIfAbsent",
|
||||
"SELECT appendTrailingCharIfAbsent(status, '!') AS r FROM task_runs",
|
||||
],
|
||||
["base64Encode", "SELECT base64Encode(status) AS r FROM task_runs"],
|
||||
["base64Decode", "SELECT base64Decode(base64Encode(status)) AS r FROM task_runs"],
|
||||
["tryBase64Decode", "SELECT tryBase64Decode('aGVsbG8=') AS r FROM task_runs"],
|
||||
["endsWith", "SELECT endsWith(status, 'LY') AS r FROM task_runs"],
|
||||
["startsWith", "SELECT startsWith(status, 'COM') AS r FROM task_runs"],
|
||||
["trim", "SELECT trim(status) AS r FROM task_runs"],
|
||||
["trimLeft", "SELECT trimLeft(status) AS r FROM task_runs"],
|
||||
["trimRight", "SELECT trimRight(status) AS r FROM task_runs"],
|
||||
["ltrim", "SELECT ltrim(status) AS r FROM task_runs"],
|
||||
["rtrim", "SELECT rtrim(status) AS r FROM task_runs"],
|
||||
["leftPad", "SELECT leftPad(status, 30, '*') AS r FROM task_runs"],
|
||||
["rightPad", "SELECT rightPad(status, 30, '*') AS r FROM task_runs"],
|
||||
["leftPadUTF8", "SELECT leftPadUTF8(status, 30, '*') AS r FROM task_runs"],
|
||||
["rightPadUTF8", "SELECT rightPadUTF8(status, 30, '*') AS r FROM task_runs"],
|
||||
["left", "SELECT left(status, 3) AS r FROM task_runs"],
|
||||
["right", "SELECT right(status, 3) AS r FROM task_runs"],
|
||||
["repeat", "SELECT repeat(status, 2) AS r FROM task_runs"],
|
||||
["space", "SELECT space(5) AS r FROM task_runs"],
|
||||
["replace", "SELECT replace(status, 'COMPLETED', 'DONE') AS r FROM task_runs"],
|
||||
["replaceOne", "SELECT replaceOne(status, 'COMPLETED', 'DONE') AS r FROM task_runs"],
|
||||
["replaceAll", "SELECT replaceAll(status, 'COMPLETED', 'DONE') AS r FROM task_runs"],
|
||||
["replaceRegexpOne", "SELECT replaceRegexpOne(status, '[A-Z]+', 'X') AS r FROM task_runs"],
|
||||
["replaceRegexpAll", "SELECT replaceRegexpAll(status, '[A-Z]', 'x') AS r FROM task_runs"],
|
||||
["position", "SELECT position(status, 'COM') AS r FROM task_runs"],
|
||||
[
|
||||
"positionCaseInsensitive",
|
||||
"SELECT positionCaseInsensitive(status, 'com') AS r FROM task_runs",
|
||||
],
|
||||
["positionUTF8", "SELECT positionUTF8(status, 'COM') AS r FROM task_runs"],
|
||||
[
|
||||
"positionCaseInsensitiveUTF8",
|
||||
"SELECT positionCaseInsensitiveUTF8(status, 'com') AS r FROM task_runs",
|
||||
],
|
||||
["locate", "SELECT locate(status, 'COM') AS r FROM task_runs"],
|
||||
["match", "SELECT match(status, 'COMPLETED.*') AS r FROM task_runs"],
|
||||
["like", "SELECT like(status, '%COMPLETED%') AS r FROM task_runs"],
|
||||
["ilike", "SELECT ilike(status, '%completed%') AS r FROM task_runs"],
|
||||
["notLike", "SELECT notLike(status, '%PENDING%') AS r FROM task_runs"],
|
||||
["notILike", "SELECT notILike(status, '%pending%') AS r FROM task_runs"],
|
||||
["splitByChar", "SELECT splitByChar('_', status) AS r FROM task_runs"],
|
||||
["splitByString", "SELECT splitByString('_', status) AS r FROM task_runs"],
|
||||
["splitByRegexp", "SELECT splitByRegexp('_', status) AS r FROM task_runs"],
|
||||
["arrayStringConcat", "SELECT arrayStringConcat(tags, ',') AS r FROM task_runs"],
|
||||
["format", "SELECT format('{0}-{1}', status, run_id) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Null functions ───────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Null functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["coalesce", "SELECT coalesce(started_at, now()) AS r FROM task_runs"],
|
||||
["ifNull", "SELECT ifNull(started_at, now()) AS r FROM task_runs"],
|
||||
["nullIf", "SELECT nullIf(status, 'PENDING') AS r FROM task_runs"],
|
||||
["assumeNotNull", "SELECT assumeNotNull(started_at) AS r FROM task_runs"],
|
||||
["toNullable", "SELECT toNullable(status) AS r FROM task_runs"],
|
||||
["isNull", "SELECT isNull(started_at) AS r FROM task_runs"],
|
||||
["isNotNull", "SELECT isNotNull(started_at) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Conditional functions ────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Conditional functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["if", "SELECT if(usage_duration_ms > 1000, 'slow', 'fast') AS r FROM task_runs"],
|
||||
[
|
||||
"multiIf",
|
||||
"SELECT multiIf(usage_duration_ms > 5000, 'slow', usage_duration_ms > 1000, 'medium', 'fast') AS r FROM task_runs",
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Comparison functions ─────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Comparison functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["equals", "SELECT equals(status, 'PENDING') AS r FROM task_runs"],
|
||||
["notEquals", "SELECT notEquals(status, 'PENDING') AS r FROM task_runs"],
|
||||
["less", "SELECT less(usage_duration_ms, 9999) AS r FROM task_runs"],
|
||||
["greater", "SELECT greater(usage_duration_ms, 0) AS r FROM task_runs"],
|
||||
["lessOrEquals", "SELECT lessOrEquals(usage_duration_ms, 9999) AS r FROM task_runs"],
|
||||
["greaterOrEquals", "SELECT greaterOrEquals(usage_duration_ms, 0) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Logical functions ────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Logical functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["and", "SELECT and(usage_duration_ms > 0, is_test = 0) AS r FROM task_runs"],
|
||||
["or", "SELECT or(usage_duration_ms > 9999, is_test = 0) AS r FROM task_runs"],
|
||||
["xor", "SELECT xor(usage_duration_ms > 0, is_test = 1) AS r FROM task_runs"],
|
||||
["not", "SELECT not(is_test) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Type conversion functions ────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Type conversion functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["toString", "SELECT toString(usage_duration_ms) AS r FROM task_runs"],
|
||||
["toFixedString", "SELECT toFixedString(status, 30) AS r FROM task_runs"],
|
||||
["toUInt8", "SELECT toUInt8(is_test) AS r FROM task_runs"],
|
||||
["toUInt16", "SELECT toUInt16(usage_duration_ms) AS r FROM task_runs"],
|
||||
["toUInt32", "SELECT toUInt32(usage_duration_ms) AS r FROM task_runs"],
|
||||
["toUInt64", "SELECT toUInt64(usage_duration_ms) AS r FROM task_runs"],
|
||||
["toInt8", "SELECT toInt8(1) AS r FROM task_runs"],
|
||||
["toInt16", "SELECT toInt16(1) AS r FROM task_runs"],
|
||||
["toInt32", "SELECT toInt32(1) AS r FROM task_runs"],
|
||||
["toInt64", "SELECT toInt64(usage_duration_ms) AS r FROM task_runs"],
|
||||
["toInt128", "SELECT toInt128(1) AS r FROM task_runs"],
|
||||
["toInt256", "SELECT toInt256(1) AS r FROM task_runs"],
|
||||
["toUInt128", "SELECT toUInt128(1) AS r FROM task_runs"],
|
||||
["toUInt256", "SELECT toUInt256(1) AS r FROM task_runs"],
|
||||
["toFloat32", "SELECT toFloat32(cost_in_cents) AS r FROM task_runs"],
|
||||
["toFloat64", "SELECT toFloat64(cost_in_cents) AS r FROM task_runs"],
|
||||
["toDecimal32", "SELECT toDecimal32(cost_in_cents, 2) AS r FROM task_runs"],
|
||||
["toDecimal64", "SELECT toDecimal64(cost_in_cents, 2) AS r FROM task_runs"],
|
||||
["toDecimal128", "SELECT toDecimal128(cost_in_cents, 2) AS r FROM task_runs"],
|
||||
["toDecimal256", "SELECT toDecimal256(cost_in_cents, 2) AS r FROM task_runs"],
|
||||
["toDate", "SELECT toDate(created_at) AS r FROM task_runs"],
|
||||
["toDateOrNull", "SELECT toDateOrNull('2024-01-01') AS r FROM task_runs"],
|
||||
["toDateOrZero", "SELECT toDateOrZero('invalid') AS r FROM task_runs"],
|
||||
["toDate32", "SELECT toDate32(created_at) AS r FROM task_runs"],
|
||||
["toDate32OrNull", "SELECT toDate32OrNull('2024-01-01') AS r FROM task_runs"],
|
||||
["toDate32OrZero", "SELECT toDate32OrZero('invalid') AS r FROM task_runs"],
|
||||
["toDateTime", "SELECT toDateTime(created_at) AS r FROM task_runs"],
|
||||
["toDateTimeOrNull", "SELECT toDateTimeOrNull('2024-01-01 00:00:00') AS r FROM task_runs"],
|
||||
["toDateTimeOrZero", "SELECT toDateTimeOrZero('invalid') AS r FROM task_runs"],
|
||||
["toDateTime64", "SELECT toDateTime64(created_at, 3) AS r FROM task_runs"],
|
||||
[
|
||||
"toDateTime64OrNull",
|
||||
"SELECT toDateTime64OrNull('2024-01-01 00:00:00.000', 3) AS r FROM task_runs",
|
||||
],
|
||||
["toDateTime64OrZero", "SELECT toDateTime64OrZero('invalid', 3) AS r FROM task_runs"],
|
||||
["toTypeName", "SELECT toTypeName(status) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Date/time functions ──────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Date/time functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["now", "SELECT now() AS r FROM task_runs"],
|
||||
["now64", "SELECT now64() AS r FROM task_runs"],
|
||||
["today", "SELECT today() AS r FROM task_runs"],
|
||||
["yesterday", "SELECT yesterday() AS r FROM task_runs"],
|
||||
["toYear", "SELECT toYear(created_at) AS r FROM task_runs"],
|
||||
["toQuarter", "SELECT toQuarter(created_at) AS r FROM task_runs"],
|
||||
["toMonth", "SELECT toMonth(created_at) AS r FROM task_runs"],
|
||||
["toDayOfYear", "SELECT toDayOfYear(created_at) AS r FROM task_runs"],
|
||||
["toDayOfMonth", "SELECT toDayOfMonth(created_at) AS r FROM task_runs"],
|
||||
["toDayOfWeek", "SELECT toDayOfWeek(created_at) AS r FROM task_runs"],
|
||||
["toHour", "SELECT toHour(created_at) AS r FROM task_runs"],
|
||||
["toMinute", "SELECT toMinute(created_at) AS r FROM task_runs"],
|
||||
["toSecond", "SELECT toSecond(created_at) AS r FROM task_runs"],
|
||||
["toUnixTimestamp", "SELECT toUnixTimestamp(created_at) AS r FROM task_runs"],
|
||||
["toStartOfYear", "SELECT toStartOfYear(created_at) AS r FROM task_runs"],
|
||||
["toStartOfQuarter", "SELECT toStartOfQuarter(created_at) AS r FROM task_runs"],
|
||||
["toStartOfMonth", "SELECT toStartOfMonth(created_at) AS r FROM task_runs"],
|
||||
["toMonday", "SELECT toMonday(created_at) AS r FROM task_runs"],
|
||||
["toStartOfWeek", "SELECT toStartOfWeek(created_at) AS r FROM task_runs"],
|
||||
["toStartOfDay", "SELECT toStartOfDay(created_at) AS r FROM task_runs"],
|
||||
["toStartOfHour", "SELECT toStartOfHour(created_at) AS r FROM task_runs"],
|
||||
["toStartOfMinute", "SELECT toStartOfMinute(created_at) AS r FROM task_runs"],
|
||||
["toStartOfSecond", "SELECT toStartOfSecond(created_at) AS r FROM task_runs"],
|
||||
["toStartOfFiveMinutes", "SELECT toStartOfFiveMinutes(created_at) AS r FROM task_runs"],
|
||||
["toStartOfTenMinutes", "SELECT toStartOfTenMinutes(created_at) AS r FROM task_runs"],
|
||||
["toStartOfFifteenMinutes", "SELECT toStartOfFifteenMinutes(created_at) AS r FROM task_runs"],
|
||||
[
|
||||
"toStartOfInterval",
|
||||
"SELECT toStartOfInterval(created_at, INTERVAL 1 hour) AS r FROM task_runs",
|
||||
],
|
||||
["toTime", "SELECT toTime(created_at) AS r FROM task_runs"],
|
||||
["toISOYear", "SELECT toISOYear(created_at) AS r FROM task_runs"],
|
||||
["toISOWeek", "SELECT toISOWeek(created_at) AS r FROM task_runs"],
|
||||
["toWeek", "SELECT toWeek(created_at) AS r FROM task_runs"],
|
||||
["toYearWeek", "SELECT toYearWeek(created_at) AS r FROM task_runs"],
|
||||
["dateAdd (string unit)", "SELECT dateAdd('day', 7, created_at) AS r FROM task_runs"],
|
||||
["dateAdd (keyword unit)", "SELECT dateAdd(day, 7, created_at) AS r FROM task_runs"],
|
||||
["dateSub (string unit)", "SELECT dateSub('hour', 1, created_at) AS r FROM task_runs"],
|
||||
[
|
||||
"dateDiff (string unit)",
|
||||
"SELECT dateDiff('minute', created_at, updated_at) AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"dateDiff (millisecond)",
|
||||
"SELECT dateDiff('millisecond', created_at, updated_at) AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"dateDiff (microsecond)",
|
||||
"SELECT dateDiff('microsecond', created_at, updated_at) AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"dateDiff (nanosecond)",
|
||||
"SELECT dateDiff('nanosecond', created_at, updated_at) AS r FROM task_runs",
|
||||
],
|
||||
["dateTrunc (string unit)", "SELECT dateTrunc('month', created_at) AS r FROM task_runs"],
|
||||
["date_add (string unit)", "SELECT date_add('day', 7, created_at) AS r FROM task_runs"],
|
||||
["date_sub (string unit)", "SELECT date_sub('hour', 1, created_at) AS r FROM task_runs"],
|
||||
[
|
||||
"date_diff (string unit)",
|
||||
"SELECT date_diff('minute', created_at, updated_at) AS r FROM task_runs",
|
||||
],
|
||||
["date_trunc (string unit)", "SELECT date_trunc('month', created_at) AS r FROM task_runs"],
|
||||
["addSeconds", "SELECT addSeconds(created_at, 10) AS r FROM task_runs"],
|
||||
["addMinutes", "SELECT addMinutes(created_at, 10) AS r FROM task_runs"],
|
||||
["addHours", "SELECT addHours(created_at, 1) AS r FROM task_runs"],
|
||||
["addDays", "SELECT addDays(created_at, 1) AS r FROM task_runs"],
|
||||
["addWeeks", "SELECT addWeeks(created_at, 1) AS r FROM task_runs"],
|
||||
["addMonths", "SELECT addMonths(created_at, 1) AS r FROM task_runs"],
|
||||
["addQuarters", "SELECT addQuarters(created_at, 1) AS r FROM task_runs"],
|
||||
["addYears", "SELECT addYears(created_at, 1) AS r FROM task_runs"],
|
||||
["subtractSeconds", "SELECT subtractSeconds(created_at, 10) AS r FROM task_runs"],
|
||||
["subtractMinutes", "SELECT subtractMinutes(created_at, 10) AS r FROM task_runs"],
|
||||
["subtractHours", "SELECT subtractHours(created_at, 1) AS r FROM task_runs"],
|
||||
["subtractDays", "SELECT subtractDays(created_at, 1) AS r FROM task_runs"],
|
||||
["subtractWeeks", "SELECT subtractWeeks(created_at, 1) AS r FROM task_runs"],
|
||||
["subtractMonths", "SELECT subtractMonths(created_at, 1) AS r FROM task_runs"],
|
||||
["subtractQuarters", "SELECT subtractQuarters(created_at, 1) AS r FROM task_runs"],
|
||||
["subtractYears", "SELECT subtractYears(created_at, 1) AS r FROM task_runs"],
|
||||
["toTimeZone", "SELECT toTimeZone(created_at, 'America/New_York') AS r FROM task_runs"],
|
||||
["formatDateTime", "SELECT formatDateTime(created_at, '%Y-%m-%d') AS r FROM task_runs"],
|
||||
["parseDateTime", "SELECT parseDateTime('2024-01-15', '%Y-%m-%d') AS r FROM task_runs"],
|
||||
[
|
||||
"parseDateTimeBestEffort",
|
||||
"SELECT parseDateTimeBestEffort('2024-01-15 10:30:00') AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"parseDateTimeBestEffortOrNull",
|
||||
"SELECT parseDateTimeBestEffortOrNull('invalid') AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"parseDateTimeBestEffortOrZero",
|
||||
"SELECT parseDateTimeBestEffortOrZero('invalid') AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"parseDateTime64BestEffort",
|
||||
"SELECT parseDateTime64BestEffort('2024-01-15 10:30:00.123') AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"parseDateTime64BestEffortOrNull",
|
||||
"SELECT parseDateTime64BestEffortOrNull('invalid') AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"parseDateTime64BestEffortOrZero",
|
||||
"SELECT parseDateTime64BestEffortOrZero('invalid') AS r FROM task_runs",
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Interval functions ───────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Interval functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["toIntervalSecond", "SELECT toIntervalSecond(10) AS r FROM task_runs"],
|
||||
["toIntervalMinute", "SELECT toIntervalMinute(5) AS r FROM task_runs"],
|
||||
["toIntervalHour", "SELECT toIntervalHour(1) AS r FROM task_runs"],
|
||||
["toIntervalDay", "SELECT toIntervalDay(7) AS r FROM task_runs"],
|
||||
["toIntervalWeek", "SELECT toIntervalWeek(2) AS r FROM task_runs"],
|
||||
["toIntervalMonth", "SELECT toIntervalMonth(3) AS r FROM task_runs"],
|
||||
["toIntervalQuarter", "SELECT toIntervalQuarter(1) AS r FROM task_runs"],
|
||||
["toIntervalYear", "SELECT toIntervalYear(1) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Array functions ──────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Array functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["array", "SELECT array(1, 2, 3) AS r FROM task_runs"],
|
||||
["range", "SELECT range(5) AS r FROM task_runs"],
|
||||
["arrayElement", "SELECT arrayElement(tags, 1) AS r FROM task_runs"],
|
||||
["has", "SELECT has(tags, 'tag-a') AS r FROM task_runs"],
|
||||
["hasAll", "SELECT hasAll(tags, array('tag-a')) AS r FROM task_runs"],
|
||||
["hasAny", "SELECT hasAny(tags, array('tag-a', 'tag-c')) AS r FROM task_runs"],
|
||||
["hasSubstr", "SELECT hasSubstr(tags, array('tag-a')) AS r FROM task_runs"],
|
||||
["indexOf", "SELECT indexOf(tags, 'tag-a') AS r FROM task_runs"],
|
||||
["arrayCount", "SELECT arrayCount(array(1, 0, 1, 0)) AS r FROM task_runs"],
|
||||
["countEqual", "SELECT countEqual(tags, 'tag-a') AS r FROM task_runs"],
|
||||
["arrayEnumerate", "SELECT arrayEnumerate(tags) AS r FROM task_runs"],
|
||||
["arrayEnumerateDense", "SELECT arrayEnumerateDense(tags) AS r FROM task_runs"],
|
||||
["arrayEnumerateUniq", "SELECT arrayEnumerateUniq(tags) AS r FROM task_runs"],
|
||||
["arrayPopBack", "SELECT arrayPopBack(tags) AS r FROM task_runs"],
|
||||
["arrayPopFront", "SELECT arrayPopFront(tags) AS r FROM task_runs"],
|
||||
["arrayPushBack", "SELECT arrayPushBack(tags, 'tag-new') AS r FROM task_runs"],
|
||||
["arrayPushFront", "SELECT arrayPushFront(tags, 'tag-new') AS r FROM task_runs"],
|
||||
["arrayResize", "SELECT arrayResize(tags, 5, '') AS r FROM task_runs"],
|
||||
["arraySlice", "SELECT arraySlice(tags, 1, 1) AS r FROM task_runs"],
|
||||
["arraySort", "SELECT arraySort(tags) AS r FROM task_runs"],
|
||||
["arrayReverseSort", "SELECT arrayReverseSort(tags) AS r FROM task_runs"],
|
||||
["arrayShuffle", "SELECT arrayShuffle(tags) AS r FROM task_runs"],
|
||||
["arrayUniq", "SELECT arrayUniq(tags) AS r FROM task_runs"],
|
||||
["arrayDifference", "SELECT arrayDifference(array(1, 2, 5)) AS r FROM task_runs"],
|
||||
["arrayDistinct", "SELECT arrayDistinct(tags) AS r FROM task_runs"],
|
||||
["arrayIntersect", "SELECT arrayIntersect(tags, array('tag-a')) AS r FROM task_runs"],
|
||||
["arrayReduce", "SELECT arrayReduce('sum', array(1, 2, 3)) AS r FROM task_runs"],
|
||||
["arrayReverse", "SELECT arrayReverse(tags) AS r FROM task_runs"],
|
||||
["arrayFlatten", "SELECT arrayFlatten(array(array(1, 2), array(3))) AS r FROM task_runs"],
|
||||
["arrayCompact", "SELECT arrayCompact(array(1, 1, 2, 3, 3)) AS r FROM task_runs"],
|
||||
["arrayZip", "SELECT arrayZip(array(1, 2), array('a', 'b')) AS r FROM task_runs"],
|
||||
["arrayMin", "SELECT arrayMin(array(1, 2, 3)) AS r FROM task_runs"],
|
||||
["arrayMax", "SELECT arrayMax(array(1, 2, 3)) AS r FROM task_runs"],
|
||||
["arraySum", "SELECT arraySum(array(1, 2, 3)) AS r FROM task_runs"],
|
||||
["arrayAvg", "SELECT arrayAvg(array(1, 2, 3)) AS r FROM task_runs"],
|
||||
["arrayCumSum", "SELECT arrayCumSum(array(1, 2, 3)) AS r FROM task_runs"],
|
||||
[
|
||||
"arrayCumSumNonNegative",
|
||||
"SELECT arrayCumSumNonNegative(array(1, -2, 3)) AS r FROM task_runs",
|
||||
],
|
||||
["arrayProduct", "SELECT arrayProduct(array(1, 2, 3)) AS r FROM task_runs"],
|
||||
["arrayJoin", "SELECT arrayJoin(array(1, 2, 3)) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── JSON functions ───────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("JSON functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["JSONHas", `SELECT JSONHas('{"a": 1}', 'a') AS r FROM task_runs`],
|
||||
["JSONLength", `SELECT JSONLength('{"a": 1, "b": 2}') AS r FROM task_runs`],
|
||||
["JSONType", `SELECT JSONType('{"a": 1}', 'a') AS r FROM task_runs`],
|
||||
["JSONExtractUInt", `SELECT JSONExtractUInt('{"a": 1}', 'a') AS r FROM task_runs`],
|
||||
["JSONExtractInt", `SELECT JSONExtractInt('{"a": -1}', 'a') AS r FROM task_runs`],
|
||||
["JSONExtractFloat", `SELECT JSONExtractFloat('{"a": 1.5}', 'a') AS r FROM task_runs`],
|
||||
["JSONExtractBool", `SELECT JSONExtractBool('{"a": true}', 'a') AS r FROM task_runs`],
|
||||
["JSONExtractString", `SELECT JSONExtractString('{"a": "hello"}', 'a') AS r FROM task_runs`],
|
||||
["JSONExtractRaw", `SELECT JSONExtractRaw('{"a": [1,2]}', 'a') AS r FROM task_runs`],
|
||||
[
|
||||
"JSONExtractArrayRaw",
|
||||
`SELECT JSONExtractArrayRaw('{"a": [1,2]}', 'a') AS r FROM task_runs`,
|
||||
],
|
||||
["JSONExtractKeys", `SELECT JSONExtractKeys('{"a": 1, "b": 2}') AS r FROM task_runs`],
|
||||
["toJSONString", "SELECT toJSONString(map('a', 1)) AS r FROM task_runs"],
|
||||
// The `output` column is a native JSON type, so JSON functions must read its
|
||||
// String companion (output_text). Without the printer fix these fail with
|
||||
// "should be a string containing JSON, illegal type: JSON".
|
||||
["JSONHas(output)", "SELECT JSONHas(output, 'count') AS r FROM task_runs"],
|
||||
["JSONLength(output)", "SELECT JSONLength(output) AS r FROM task_runs"],
|
||||
["JSONType(output)", "SELECT JSONType(output, 'count') AS r FROM task_runs"],
|
||||
["JSONExtractInt(output)", "SELECT JSONExtractInt(output, 'count') AS r FROM task_runs"],
|
||||
["JSONExtractUInt(output)", "SELECT JSONExtractUInt(output, 'count') AS r FROM task_runs"],
|
||||
["JSONExtractFloat(output)", "SELECT JSONExtractFloat(output, 'ratio') AS r FROM task_runs"],
|
||||
["JSONExtractBool(output)", "SELECT JSONExtractBool(output, 'enabled') AS r FROM task_runs"],
|
||||
[
|
||||
"JSONExtractString(output)",
|
||||
"SELECT JSONExtractString(output, 'label') AS r FROM task_runs",
|
||||
],
|
||||
["JSONExtractRaw(output)", "SELECT JSONExtractRaw(output, 'count') AS r FROM task_runs"],
|
||||
["JSONExtractKeys(output)", "SELECT JSONExtractKeys(output) AS r FROM task_runs"],
|
||||
// The JSON field can be wrapped in a passthrough like assumeNotNull; the swap
|
||||
// still has to reach the native column underneath.
|
||||
[
|
||||
"JSONExtractArrayRaw(assumeNotNull(output))",
|
||||
"SELECT JSONExtractArrayRaw(assumeNotNull(output), 'items') AS r FROM task_runs",
|
||||
],
|
||||
// toJSONString(output) is already a String, so it stays on the native column.
|
||||
[
|
||||
"JSONExtractString(toJSONString(output))",
|
||||
"SELECT JSONExtractString(toJSONString(output), 'data', 'label') AS r FROM task_runs",
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Tuple functions ──────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Tuple functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["tuple", "SELECT tuple(1, 'a', 3.14) AS r FROM task_runs"],
|
||||
["tupleElement", "SELECT tupleElement(tuple(1, 'a'), 1) AS r FROM task_runs"],
|
||||
["untuple", "SELECT untuple(tuple(1, 'a')) FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Map functions ────────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Map functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["map", "SELECT map('a', 1, 'b', 2) AS r FROM task_runs"],
|
||||
["mapFromArrays", "SELECT mapFromArrays(array('a', 'b'), array(1, 2)) AS r FROM task_runs"],
|
||||
["mapContains", "SELECT mapContains(map('a', 1), 'a') AS r FROM task_runs"],
|
||||
["mapKeys", "SELECT mapKeys(map('a', 1, 'b', 2)) AS r FROM task_runs"],
|
||||
["mapValues", "SELECT mapValues(map('a', 1, 'b', 2)) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Hash functions ───────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Hash functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["MD5", "SELECT hex(MD5('hello')) AS r FROM task_runs"],
|
||||
["SHA1", "SELECT hex(SHA1('hello')) AS r FROM task_runs"],
|
||||
["SHA224", "SELECT hex(SHA224('hello')) AS r FROM task_runs"],
|
||||
["SHA256", "SELECT hex(SHA256('hello')) AS r FROM task_runs"],
|
||||
["SHA384", "SELECT hex(SHA384('hello')) AS r FROM task_runs"],
|
||||
["SHA512", "SELECT hex(SHA512('hello')) AS r FROM task_runs"],
|
||||
["sipHash64", "SELECT sipHash64('hello') AS r FROM task_runs"],
|
||||
["sipHash128", "SELECT hex(sipHash128('hello')) AS r FROM task_runs"],
|
||||
["cityHash64", "SELECT cityHash64('hello') AS r FROM task_runs"],
|
||||
["intHash32", "SELECT intHash32(42) AS r FROM task_runs"],
|
||||
["intHash64", "SELECT intHash64(42) AS r FROM task_runs"],
|
||||
["farmHash64", "SELECT farmHash64('hello') AS r FROM task_runs"],
|
||||
["farmFingerprint64", "SELECT farmFingerprint64('hello') AS r FROM task_runs"],
|
||||
["xxHash32", "SELECT xxHash32('hello') AS r FROM task_runs"],
|
||||
["xxHash64", "SELECT xxHash64('hello') AS r FROM task_runs"],
|
||||
["murmurHash2_32", "SELECT murmurHash2_32('hello') AS r FROM task_runs"],
|
||||
["murmurHash2_64", "SELECT murmurHash2_64('hello') AS r FROM task_runs"],
|
||||
["murmurHash3_32", "SELECT murmurHash3_32('hello') AS r FROM task_runs"],
|
||||
["murmurHash3_64", "SELECT murmurHash3_64('hello') AS r FROM task_runs"],
|
||||
["murmurHash3_128", "SELECT hex(murmurHash3_128('hello')) AS r FROM task_runs"],
|
||||
["hex", "SELECT hex(255) AS r FROM task_runs"],
|
||||
["unhex", "SELECT unhex('48656C6C6F') AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── URL functions ────────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("URL functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["protocol", `SELECT protocol('${url}') AS r FROM task_runs`],
|
||||
["domain", `SELECT domain('${url}') AS r FROM task_runs`],
|
||||
["domainWithoutWWW", `SELECT domainWithoutWWW('${url}') AS r FROM task_runs`],
|
||||
["topLevelDomain", `SELECT topLevelDomain('${url}') AS r FROM task_runs`],
|
||||
[
|
||||
"firstSignificantSubdomain",
|
||||
`SELECT firstSignificantSubdomain('${url}') AS r FROM task_runs`,
|
||||
],
|
||||
[
|
||||
"cutToFirstSignificantSubdomain",
|
||||
`SELECT cutToFirstSignificantSubdomain('${url}') AS r FROM task_runs`,
|
||||
],
|
||||
[
|
||||
"cutToFirstSignificantSubdomainWithWWW",
|
||||
`SELECT cutToFirstSignificantSubdomainWithWWW('${url}') AS r FROM task_runs`,
|
||||
],
|
||||
["port", `SELECT port('${url}') AS r FROM task_runs`],
|
||||
["path", `SELECT path('${url}') AS r FROM task_runs`],
|
||||
["pathFull", `SELECT pathFull('${url}') AS r FROM task_runs`],
|
||||
["queryString", `SELECT queryString('${url}') AS r FROM task_runs`],
|
||||
["fragment", `SELECT fragment('${url}') AS r FROM task_runs`],
|
||||
["extractURLParameter", `SELECT extractURLParameter('${url}', 'q') AS r FROM task_runs`],
|
||||
["extractURLParameters", `SELECT extractURLParameters('${url}') AS r FROM task_runs`],
|
||||
["encodeURLComponent", "SELECT encodeURLComponent('hello world') AS r FROM task_runs"],
|
||||
["decodeURLComponent", "SELECT decodeURLComponent('hello%20world') AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── UUID functions ───────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("UUID functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["generateUUIDv4", "SELECT generateUUIDv4() AS r FROM task_runs"],
|
||||
[
|
||||
"UUIDStringToNum",
|
||||
"SELECT UUIDStringToNum('00000000-0000-0000-0000-000000000000') AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"UUIDNumToString",
|
||||
"SELECT UUIDNumToString(UUIDStringToNum('00000000-0000-0000-0000-000000000000')) AS r FROM task_runs",
|
||||
],
|
||||
["toUUID", "SELECT toUUID('00000000-0000-0000-0000-000000000000') AS r FROM task_runs"],
|
||||
["toUUIDOrNull", "SELECT toUUIDOrNull('not-a-uuid') AS r FROM task_runs"],
|
||||
["toUUIDOrZero", "SELECT toUUIDOrZero('not-a-uuid') AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Misc functions ───────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Misc functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["isFinite", "SELECT isFinite(1.0) AS r FROM task_runs"],
|
||||
["isInfinite", "SELECT isInfinite(1.0 / 0) AS r FROM task_runs"],
|
||||
["ifNotFinite", "SELECT ifNotFinite(1.0 / 0, 0) AS r FROM task_runs"],
|
||||
["isNaN", "SELECT isNaN(0.0 / 0) AS r FROM task_runs"],
|
||||
["bar", "SELECT bar(usage_duration_ms, 0, 10000, 20) AS r FROM task_runs"],
|
||||
[
|
||||
"transform",
|
||||
"SELECT transform(status, array('PENDING', 'COMPLETED_SUCCESSFULLY'), array('P', 'C'), 'X') AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"formatReadableDecimalSize",
|
||||
"SELECT formatReadableDecimalSize(1000000) AS r FROM task_runs",
|
||||
],
|
||||
["formatReadableSize", "SELECT formatReadableSize(1000000) AS r FROM task_runs"],
|
||||
["formatReadableQuantity", "SELECT formatReadableQuantity(1000000) AS r FROM task_runs"],
|
||||
["formatReadableTimeDelta", "SELECT formatReadableTimeDelta(3661) AS r FROM task_runs"],
|
||||
["least", "SELECT least(1, 2) AS r FROM task_runs"],
|
||||
["greatest", "SELECT greatest(1, 2) AS r FROM task_runs"],
|
||||
["min2", "SELECT min2(1, 2) AS r FROM task_runs"],
|
||||
["max2", "SELECT max2(1, 2) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Aggregate functions ──────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Aggregate functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["count()", "SELECT count() AS r FROM task_runs"],
|
||||
["count(col)", "SELECT count(run_id) AS r FROM task_runs"],
|
||||
["countDistinct", "SELECT countDistinct(status) AS r FROM task_runs"],
|
||||
["min", "SELECT min(usage_duration_ms) AS r FROM task_runs"],
|
||||
["max", "SELECT max(usage_duration_ms) AS r FROM task_runs"],
|
||||
["sum", "SELECT sum(usage_duration_ms) AS r FROM task_runs"],
|
||||
["avg", "SELECT avg(usage_duration_ms) AS r FROM task_runs"],
|
||||
["any", "SELECT any(status) AS r FROM task_runs"],
|
||||
["anyLast", "SELECT anyLast(status) AS r FROM task_runs"],
|
||||
["anyHeavy", "SELECT anyHeavy(status) AS r FROM task_runs"],
|
||||
["argMin", "SELECT argMin(run_id, usage_duration_ms) AS r FROM task_runs"],
|
||||
["argMax", "SELECT argMax(run_id, usage_duration_ms) AS r FROM task_runs"],
|
||||
["stddevPop", "SELECT stddevPop(usage_duration_ms) AS r FROM task_runs"],
|
||||
["stddevSamp", "SELECT stddevSamp(usage_duration_ms) AS r FROM task_runs"],
|
||||
["varPop", "SELECT varPop(usage_duration_ms) AS r FROM task_runs"],
|
||||
["varSamp", "SELECT varSamp(usage_duration_ms) AS r FROM task_runs"],
|
||||
["covarPop", "SELECT covarPop(usage_duration_ms, cost_in_cents) AS r FROM task_runs"],
|
||||
["covarSamp", "SELECT covarSamp(usage_duration_ms, cost_in_cents) AS r FROM task_runs"],
|
||||
["corr", "SELECT corr(usage_duration_ms, cost_in_cents) AS r FROM task_runs"],
|
||||
["groupArray", "SELECT groupArray(status) AS r FROM task_runs"],
|
||||
["groupUniqArray", "SELECT groupUniqArray(status) AS r FROM task_runs"],
|
||||
["groupArrayMovingAvg", "SELECT groupArrayMovingAvg(usage_duration_ms) AS r FROM task_runs"],
|
||||
["groupArrayMovingSum", "SELECT groupArrayMovingSum(usage_duration_ms) AS r FROM task_runs"],
|
||||
["uniq", "SELECT uniq(status) AS r FROM task_runs"],
|
||||
["uniqExact", "SELECT uniqExact(status) AS r FROM task_runs"],
|
||||
["uniqHLL12", "SELECT uniqHLL12(status) AS r FROM task_runs"],
|
||||
["uniqTheta", "SELECT uniqTheta(status) AS r FROM task_runs"],
|
||||
["median", "SELECT median(usage_duration_ms) AS r FROM task_runs"],
|
||||
["medianExact", "SELECT medianExact(usage_duration_ms) AS r FROM task_runs"],
|
||||
["quantile", "SELECT quantile(0.95)(usage_duration_ms) AS r FROM task_runs"],
|
||||
["quantiles", "SELECT quantiles(0.5, 0.9, 0.99)(usage_duration_ms) AS r FROM task_runs"],
|
||||
["topK", "SELECT topK(3)(status) AS r FROM task_runs"],
|
||||
[
|
||||
"simpleLinearRegression",
|
||||
"SELECT simpleLinearRegression(usage_duration_ms, cost_in_cents) AS r FROM task_runs",
|
||||
],
|
||||
["groupArraySample", "SELECT groupArraySample(2)(status) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Conditional aggregate functions ──────────────────────────────────────
|
||||
|
||||
clickhouseTest("Conditional aggregate functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
["countIf", "SELECT countIf(usage_duration_ms > 1000) AS r FROM task_runs"],
|
||||
[
|
||||
"countDistinctIf",
|
||||
"SELECT countDistinctIf(status, usage_duration_ms > 0) AS r FROM task_runs",
|
||||
],
|
||||
["minIf", "SELECT minIf(usage_duration_ms, is_test = 0) AS r FROM task_runs"],
|
||||
["maxIf", "SELECT maxIf(usage_duration_ms, is_test = 0) AS r FROM task_runs"],
|
||||
["sumIf", "SELECT sumIf(usage_duration_ms, is_test = 0) AS r FROM task_runs"],
|
||||
["avgIf", "SELECT avgIf(usage_duration_ms, is_test = 0) AS r FROM task_runs"],
|
||||
["anyIf", "SELECT anyIf(status, usage_duration_ms > 0) AS r FROM task_runs"],
|
||||
["anyLastIf", "SELECT anyLastIf(status, usage_duration_ms > 0) AS r FROM task_runs"],
|
||||
["anyHeavyIf", "SELECT anyHeavyIf(status, usage_duration_ms > 0) AS r FROM task_runs"],
|
||||
["groupArrayIf", "SELECT groupArrayIf(status, usage_duration_ms > 0) AS r FROM task_runs"],
|
||||
[
|
||||
"groupUniqArrayIf",
|
||||
"SELECT groupUniqArrayIf(status, usage_duration_ms > 0) AS r FROM task_runs",
|
||||
],
|
||||
["uniqIf", "SELECT uniqIf(status, usage_duration_ms > 0) AS r FROM task_runs"],
|
||||
["uniqExactIf", "SELECT uniqExactIf(status, usage_duration_ms > 0) AS r FROM task_runs"],
|
||||
["medianIf", "SELECT medianIf(usage_duration_ms, is_test = 0) AS r FROM task_runs"],
|
||||
["quantileIf", "SELECT quantileIf(0.95)(usage_duration_ms, is_test = 0) AS r FROM task_runs"],
|
||||
["argMinIf", "SELECT argMinIf(run_id, usage_duration_ms, is_test = 0) AS r FROM task_runs"],
|
||||
["argMaxIf", "SELECT argMaxIf(run_id, usage_duration_ms, is_test = 0) AS r FROM task_runs"],
|
||||
]);
|
||||
});
|
||||
|
||||
// ─── Search functions ─────────────────────────────────────────────────────
|
||||
|
||||
clickhouseTest("Search functions", async ({ clickhouseContainer }) => {
|
||||
const client = await setupClient(clickhouseContainer);
|
||||
await runCases(client, [
|
||||
[
|
||||
"multiMatchAny",
|
||||
"SELECT multiMatchAny(status, array('COMPLETED.*', 'PENDING')) AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"multiMatchAnyIndex",
|
||||
"SELECT multiMatchAnyIndex(status, array('COMPLETED.*', 'PENDING')) AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"multiMatchAllIndices",
|
||||
"SELECT multiMatchAllIndices(status, array('COMPLETED.*', 'PEND.*')) AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"multiSearchFirstPosition",
|
||||
"SELECT multiSearchFirstPosition(status, array('COMP', 'PEND')) AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"multiSearchFirstIndex",
|
||||
"SELECT multiSearchFirstIndex(status, array('COMP', 'PEND')) AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"multiSearchAny",
|
||||
"SELECT multiSearchAny(status, array('COMP', 'PEND')) AS r FROM task_runs",
|
||||
],
|
||||
["extract", "SELECT extract(status, '[A-Z]+') AS r FROM task_runs"],
|
||||
["extractAll", "SELECT extractAll(status, '[A-Z]+') AS r FROM task_runs"],
|
||||
[
|
||||
"extractAllGroupsHorizontal",
|
||||
"SELECT extractAllGroupsHorizontal(status, '([A-Z]+)') AS r FROM task_runs",
|
||||
],
|
||||
[
|
||||
"extractAllGroupsVertical",
|
||||
"SELECT extractAllGroupsVertical(status, '([A-Z]+)') AS r FROM task_runs",
|
||||
],
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2019",
|
||||
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"outDir": "dist",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"noImplicitAny": false,
|
||||
"declaration": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }],
|
||||
"compilerOptions": {
|
||||
"moduleResolution": "Node16",
|
||||
"module": "Node16",
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "src/**/*.test.ts"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2019",
|
||||
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"include": ["src/**/*.test.ts", "vitest.config.ts"],
|
||||
"references": [{ "path": "./tsconfig.src.json" }],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2019",
|
||||
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"types": ["vitest/globals"],
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
sequence: { sequencer: DurationShardingSequencer },
|
||||
include: ["**/*.test.ts"],
|
||||
globals: true,
|
||||
isolate: true,
|
||||
fileParallelism: false,
|
||||
testTimeout: 60_000,
|
||||
coverage: {
|
||||
provider: "v8",
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "@internal/compute",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"main": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": {
|
||||
"@triggerdotdev/source": "./src/index.ts",
|
||||
"import": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"default": "./dist/src/index.js"
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"zod": "3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
"rimraf": "6.0.1"
|
||||
},
|
||||
"scripts": {
|
||||
"clean": "rimraf dist",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"build": "pnpm run clean && tsc -p tsconfig.build.json",
|
||||
"dev": "tsc --watch -p tsconfig.build.json"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
import type {
|
||||
TemplateCreateRequest,
|
||||
TemplateCreateResponse,
|
||||
InstanceCreateRequest,
|
||||
InstanceCreateResponse,
|
||||
InstanceSnapshotRequest,
|
||||
SnapshotRestoreRequest,
|
||||
} from "./types.js";
|
||||
|
||||
export type ComputeClientOptions = {
|
||||
gatewayUrl: string;
|
||||
authToken?: string;
|
||||
timeoutMs: number;
|
||||
/**
|
||||
* Called once per outbound request to collect cross-service correlation
|
||||
* headers (e.g. `traceparent`, `x-request-id`) from the caller's current
|
||||
* scope. The returned record is merged onto the outbound headers. Return
|
||||
* `{}` (or omit the option) to skip propagation.
|
||||
*/
|
||||
getPropagationHeaders?: () => Record<string, string>;
|
||||
};
|
||||
|
||||
export class ComputeClient {
|
||||
readonly templates: TemplatesNamespace;
|
||||
readonly instances: InstancesNamespace;
|
||||
readonly snapshots: SnapshotsNamespace;
|
||||
|
||||
constructor(private opts: ComputeClientOptions) {
|
||||
const http = new HttpTransport(opts);
|
||||
this.templates = new TemplatesNamespace(http);
|
||||
this.instances = new InstancesNamespace(http);
|
||||
this.snapshots = new SnapshotsNamespace(http);
|
||||
}
|
||||
}
|
||||
|
||||
// ── HTTP transport (shared plumbing) ─────────────────────────────────────────
|
||||
|
||||
type RequestOptions = {
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
class HttpTransport {
|
||||
constructor(private opts: ComputeClientOptions) {}
|
||||
|
||||
private get headers(): Record<string, string> {
|
||||
const h: Record<string, string> = { "Content-Type": "application/json" };
|
||||
if (this.opts.authToken) {
|
||||
h["Authorization"] = `Bearer ${this.opts.authToken}`;
|
||||
}
|
||||
const propagation = this.opts.getPropagationHeaders?.();
|
||||
if (propagation) {
|
||||
for (const [key, value] of Object.entries(propagation)) {
|
||||
if (value) {
|
||||
h[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
private signal(options?: RequestOptions): AbortSignal {
|
||||
return options?.signal ?? AbortSignal.timeout(this.opts.timeoutMs);
|
||||
}
|
||||
|
||||
async post<T = unknown>(
|
||||
path: string,
|
||||
body: unknown,
|
||||
options?: RequestOptions
|
||||
): Promise<T | undefined> {
|
||||
const url = `${this.opts.gatewayUrl}${path}`;
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: this.headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: this.signal(options),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorBody = await response.text().catch(() => "unknown error");
|
||||
throw new ComputeClientError(response.status, errorBody, url);
|
||||
}
|
||||
|
||||
// 202 Accepted or 204 No Content - no body to parse
|
||||
if (response.status === 202 || response.status === 204) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
async delete(path: string, options?: RequestOptions): Promise<void> {
|
||||
const url = `${this.opts.gatewayUrl}${path}`;
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "DELETE",
|
||||
headers: this.headers,
|
||||
signal: this.signal(options),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorBody = await response.text().catch(() => "unknown error");
|
||||
throw new ComputeClientError(response.status, errorBody, url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Error ────────────────────────────────────────────────────────────────────
|
||||
|
||||
export class ComputeClientError extends Error {
|
||||
constructor(
|
||||
public readonly status: number,
|
||||
public readonly body: string,
|
||||
public readonly url: string
|
||||
) {
|
||||
super(`Compute gateway request failed (${status}): ${body}`);
|
||||
this.name = "ComputeClientError";
|
||||
}
|
||||
}
|
||||
|
||||
// ── Namespaces ───────────────────────────────────────────────────────────────
|
||||
|
||||
class TemplatesNamespace {
|
||||
constructor(private http: HttpTransport) {}
|
||||
|
||||
async create(
|
||||
req: TemplateCreateRequest,
|
||||
options?: RequestOptions
|
||||
): Promise<TemplateCreateResponse | undefined> {
|
||||
// Background mode returns 202 with no body; sync/callback mode returns
|
||||
// the full result. Caller decides whether to inspect.
|
||||
return this.http.post<TemplateCreateResponse>("/api/templates", req, options);
|
||||
}
|
||||
}
|
||||
|
||||
class InstancesNamespace {
|
||||
constructor(private http: HttpTransport) {}
|
||||
|
||||
async create(
|
||||
req: InstanceCreateRequest,
|
||||
options?: RequestOptions
|
||||
): Promise<InstanceCreateResponse> {
|
||||
const result = await this.http.post<InstanceCreateResponse>("/api/instances", req, options);
|
||||
if (!result) {
|
||||
throw new Error("Compute gateway returned no instance body");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async delete(runnerId: string, options?: RequestOptions): Promise<void> {
|
||||
return this.http.delete(`/api/instances/${runnerId}`, options);
|
||||
}
|
||||
|
||||
async snapshot(
|
||||
runnerId: string,
|
||||
req: InstanceSnapshotRequest,
|
||||
options?: RequestOptions
|
||||
): Promise<void> {
|
||||
await this.http.post(`/api/instances/${runnerId}/snapshot`, req, options);
|
||||
}
|
||||
}
|
||||
|
||||
class SnapshotsNamespace {
|
||||
constructor(private http: HttpTransport) {}
|
||||
|
||||
async restore(
|
||||
snapshotId: string,
|
||||
req: SnapshotRestoreRequest,
|
||||
options?: RequestOptions
|
||||
): Promise<void> {
|
||||
await this.http.post(`/api/snapshots/${snapshotId}/restore`, req, options);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Strip the digest suffix from a container image reference.
|
||||
* Tags are immutable, so we resolve by tag rather than pinning to a digest.
|
||||
*
|
||||
* "ghcr.io/org/image:tag@sha256:abc..." -> "ghcr.io/org/image:tag"
|
||||
* "ghcr.io/org/image@sha256:abc..." -> "ghcr.io/org/image"
|
||||
* "ghcr.io/org/image:tag" -> "ghcr.io/org/image:tag" (unchanged)
|
||||
*/
|
||||
export function stripImageDigest(imageRef: string): string {
|
||||
return imageRef.split("@")[0] ?? imageRef;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
export { ComputeClient, ComputeClientError } from "./client.js";
|
||||
export type { ComputeClientOptions } from "./client.js";
|
||||
export { stripImageDigest } from "./imageRef.js";
|
||||
export {
|
||||
MachineConfigSchema,
|
||||
TemplateCreateRequestSchema,
|
||||
TemplateCreateResultEntrySchema,
|
||||
TemplateCreateResponseSchema,
|
||||
InstanceCreateRequestSchema,
|
||||
InstanceCreateResponseSchema,
|
||||
InstanceSnapshotRequestSchema,
|
||||
SnapshotRestoreRequestSchema,
|
||||
SnapshotCallbackPayloadSchema,
|
||||
} from "./types.js";
|
||||
export type {
|
||||
MachineConfig,
|
||||
TemplateCreateRequest,
|
||||
TemplateCreateResultEntry,
|
||||
TemplateCreateResponse,
|
||||
InstanceCreateRequest,
|
||||
InstanceCreateResponse,
|
||||
InstanceSnapshotRequest,
|
||||
SnapshotRestoreRequest,
|
||||
SnapshotCallbackPayload,
|
||||
} from "./types.js";
|
||||
@@ -0,0 +1,96 @@
|
||||
import { z } from "zod";
|
||||
|
||||
// ── Templates ────────────────────────────────────────────────────────────────
|
||||
|
||||
export const MachineConfigSchema = z.object({
|
||||
cpu: z.number(),
|
||||
memory_gb: z.number(),
|
||||
});
|
||||
export type MachineConfig = z.infer<typeof MachineConfigSchema>;
|
||||
|
||||
export const TemplateCreateRequestSchema = z.object({
|
||||
image: z.string(),
|
||||
machine_configs: z.array(MachineConfigSchema),
|
||||
background: z.boolean().optional(),
|
||||
callback: z
|
||||
.object({
|
||||
url: z.string(),
|
||||
metadata: z.record(z.string()).optional(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
export type TemplateCreateRequest = z.infer<typeof TemplateCreateRequestSchema>;
|
||||
|
||||
export const TemplateCreateResultEntrySchema = z.object({
|
||||
machine_config: MachineConfigSchema,
|
||||
error: z.string().optional(),
|
||||
});
|
||||
export type TemplateCreateResultEntry = z.infer<typeof TemplateCreateResultEntrySchema>;
|
||||
|
||||
export const TemplateCreateResponseSchema = z.object({
|
||||
results: z.array(TemplateCreateResultEntrySchema),
|
||||
error: z.string().optional(),
|
||||
});
|
||||
export type TemplateCreateResponse = z.infer<typeof TemplateCreateResponseSchema>;
|
||||
|
||||
// ── Instances ────────────────────────────────────────────────────────────────
|
||||
|
||||
export const InstanceCreateRequestSchema = z.object({
|
||||
name: z.string(),
|
||||
image: z.string(),
|
||||
env: z.record(z.string()),
|
||||
cpu: z.number(),
|
||||
memory_gb: z.number(),
|
||||
metadata: z.record(z.unknown()).optional(),
|
||||
// Per-instance identity labels; the provider promotes a configured subset
|
||||
// to network-policy selection. Distinct from metadata, which is
|
||||
// observability-only and never selected on.
|
||||
labels: z.record(z.string()).optional(),
|
||||
});
|
||||
export type InstanceCreateRequest = z.infer<typeof InstanceCreateRequestSchema>;
|
||||
|
||||
export const InstanceCreateResponseSchema = z.object({
|
||||
id: z.string(),
|
||||
_timing: z.unknown().optional(),
|
||||
});
|
||||
export type InstanceCreateResponse = z.infer<typeof InstanceCreateResponseSchema>;
|
||||
|
||||
export const InstanceSnapshotRequestSchema = z.object({
|
||||
callback: z.object({
|
||||
url: z.string(),
|
||||
metadata: z.record(z.string()),
|
||||
}),
|
||||
});
|
||||
export type InstanceSnapshotRequest = z.infer<typeof InstanceSnapshotRequestSchema>;
|
||||
|
||||
// ── Snapshots ────────────────────────────────────────────────────────────────
|
||||
|
||||
export const SnapshotRestoreRequestSchema = z.object({
|
||||
name: z.string(),
|
||||
metadata: z.record(z.string()),
|
||||
cpu: z.number(),
|
||||
memory_gb: z.number(),
|
||||
// Per-instance identity labels; the caller must resupply the same set as on
|
||||
// create. The provider doesn't persist them across a snapshot, so omitting
|
||||
// them drops the restored run's policy-based network selection.
|
||||
labels: z.record(z.string()).optional(),
|
||||
});
|
||||
export type SnapshotRestoreRequest = z.infer<typeof SnapshotRestoreRequestSchema>;
|
||||
|
||||
export const SnapshotCallbackPayloadSchema = z.discriminatedUnion("status", [
|
||||
z.object({
|
||||
status: z.literal("completed"),
|
||||
snapshot_id: z.string(),
|
||||
instance_id: z.string(),
|
||||
metadata: z.record(z.string()).optional(),
|
||||
duration_ms: z.number().optional(),
|
||||
}),
|
||||
z.object({
|
||||
status: z.literal("failed"),
|
||||
instance_id: z.string(),
|
||||
error: z.string().optional(),
|
||||
metadata: z.record(z.string()).optional(),
|
||||
duration_ms: z.number().optional(),
|
||||
}),
|
||||
]);
|
||||
export type SnapshotCallbackPayload = z.infer<typeof SnapshotCallbackPayloadSchema>;
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"outDir": "dist",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"declaration": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2019",
|
||||
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"strict": true
|
||||
},
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
# @internal/dashboard-agent-db
|
||||
|
||||
The conversation datastore for the in-dashboard agent, isolated from the main
|
||||
Prisma database. Drizzle (postgres-js) over a dedicated `trigger_dashboard_agent`
|
||||
Postgres schema.
|
||||
|
||||
- **Cloud:** a separate PlanetScale Postgres database. The app connects over a
|
||||
pooled connection (`DASHBOARD_AGENT_DATABASE_URL`); migrations run over a direct
|
||||
(non-pooler) connection (`DASHBOARD_AGENT_DIRECT_URL`), since a transaction-mode
|
||||
pooler can't run the migrator.
|
||||
- **OSS / self-host:** falls back to the main `DATABASE_URL` (and `DIRECT_URL` for
|
||||
migrations); the tables live in the dedicated `trigger_dashboard_agent` schema,
|
||||
isolated from Prisma's `public`.
|
||||
|
||||
The schema is **foreign-key-free** — it references main entities (`organizationId`,
|
||||
`userId`) by id only, because in cloud it lives in a different database.
|
||||
|
||||
## Why a separate store
|
||||
|
||||
The agent runs as an ephemeral Trigger task and must have **no access to the main
|
||||
database or ClickHouse** (those go through the API). This is its own low-blast-radius
|
||||
store: the agent connects directly here to persist conversations, and the webapp
|
||||
connects here for the History tab. Conversation history *correctness* is owned by
|
||||
`chat.agent`'s built-in object-store snapshot — this DB is a display read-model
|
||||
(list chats, render a past chat, resume the transport), never the model's source
|
||||
of truth.
|
||||
|
||||
## Tables
|
||||
|
||||
- `chats` — one row per conversation: org/user scope, title, a `messages` JSONB
|
||||
display copy of the transcript, and `metadata` (the project/env context the chat
|
||||
ran in). Soft-deleted via `deleted_at`, pinned via `pinned_at`.
|
||||
- `chat_sessions` — live transport state keyed by `chat_id`: the session-scoped
|
||||
`public_access_token` and `last_event_id` for resume. Separate table so the
|
||||
secret token is isolated from list queries and the hot per-turn write stays off
|
||||
the conversation row's indexes.
|
||||
|
||||
## Migrations
|
||||
|
||||
```bash
|
||||
pnpm run db:generate # generate SQL migration from src/schema.ts (offline)
|
||||
pnpm run db:migrate # apply migrations (direct url: DASHBOARD_AGENT_DIRECT_URL, falling back to DASHBOARD_AGENT_DATABASE_URL / DIRECT_URL / DATABASE_URL)
|
||||
```
|
||||
|
||||
drizzle-kit is scoped to the `trigger_dashboard_agent` schema (`schemaFilter`), so
|
||||
pointing it at the main OSS database never touches Prisma's tables.
|
||||
@@ -0,0 +1,25 @@
|
||||
import { defineConfig } from "drizzle-kit";
|
||||
|
||||
// Migrations need a direct (non-pooler) connection; a transaction-mode pooler
|
||||
// can't run the migrator. Prefer the agent's direct url, then its pooled url,
|
||||
// then the main DIRECT_URL/DATABASE_URL (OSS single-database fallback; tables
|
||||
// still land in the trigger_dashboard_agent schema).
|
||||
const url =
|
||||
process.env.DASHBOARD_AGENT_DIRECT_URL ??
|
||||
process.env.DASHBOARD_AGENT_DATABASE_URL ??
|
||||
process.env.DIRECT_URL ??
|
||||
process.env.DATABASE_URL ??
|
||||
"postgres://placeholder"; // generate is offline; a real url is only needed for migrate/studio
|
||||
|
||||
export default defineConfig({
|
||||
schema: "./src/schema.ts",
|
||||
out: "./drizzle",
|
||||
dialect: "postgresql",
|
||||
// Only manage our schema — never introspect or diff Prisma's `public` schema.
|
||||
schemaFilter: ["trigger_dashboard_agent"],
|
||||
// Own journal table so dev (`drizzle-kit migrate`) and deploy (migrate.mjs)
|
||||
// share one history and we don't cross-poison the default
|
||||
// drizzle.__drizzle_migrations when the DB is shared. See migrate.mjs.
|
||||
migrations: { table: "__dashboard_agent_migrations", schema: "drizzle" },
|
||||
dbCredentials: { url },
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
CREATE SCHEMA IF NOT EXISTS "trigger_dashboard_agent";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE IF NOT EXISTS "trigger_dashboard_agent"."chat_sessions" (
|
||||
"chat_id" text PRIMARY KEY NOT NULL,
|
||||
"public_access_token" text NOT NULL,
|
||||
"last_event_id" text,
|
||||
"run_id" text,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE IF NOT EXISTS "trigger_dashboard_agent"."chats" (
|
||||
"id" text PRIMARY KEY NOT NULL,
|
||||
"organization_id" text NOT NULL,
|
||||
"user_id" text NOT NULL,
|
||||
"title" text DEFAULT 'New chat' NOT NULL,
|
||||
"messages" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
||||
"metadata" jsonb DEFAULT '{}'::jsonb NOT NULL,
|
||||
"pinned_at" timestamp with time zone,
|
||||
"deleted_at" timestamp with time zone,
|
||||
"last_message_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "chats_org_user_last_msg_idx" ON "trigger_dashboard_agent"."chats" USING btree ("organization_id","user_id","last_message_at" DESC NULLS LAST) WHERE "trigger_dashboard_agent"."chats"."deleted_at" is null;
|
||||
@@ -0,0 +1,38 @@
|
||||
CREATE TABLE IF NOT EXISTS "trigger_dashboard_agent"."chat_turn_evals" (
|
||||
"chat_id" text NOT NULL,
|
||||
"turn" integer NOT NULL,
|
||||
"organization_id" text NOT NULL,
|
||||
"user_id" text NOT NULL,
|
||||
"agent_run_id" text,
|
||||
"eval_run_id" text,
|
||||
"project_ref" text,
|
||||
"environment" text,
|
||||
"current_page" text,
|
||||
"model" text,
|
||||
"prompt_slug" text,
|
||||
"prompt_version" integer,
|
||||
"tools_used" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
||||
"tool_error" boolean DEFAULT false NOT NULL,
|
||||
"judge_model" text,
|
||||
"score_grounded" smallint,
|
||||
"score_answered" smallint,
|
||||
"score_concise" smallint,
|
||||
"passed" boolean,
|
||||
"intent_category" text,
|
||||
"outcome" text,
|
||||
"sentiment" text,
|
||||
"capability_gap" boolean DEFAULT false NOT NULL,
|
||||
"docs_gap" boolean DEFAULT false NOT NULL,
|
||||
"support_opportunity" boolean DEFAULT false NOT NULL,
|
||||
"feature_request" boolean DEFAULT false NOT NULL,
|
||||
"topics" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
||||
"signals" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
||||
"summary" text,
|
||||
"user_text" text,
|
||||
"judge" jsonb,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "chat_turn_evals_chat_id_turn_pk" PRIMARY KEY("chat_id","turn")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "chat_turn_evals_org_created_idx" ON "trigger_dashboard_agent"."chat_turn_evals" USING btree ("organization_id","created_at" DESC NULLS LAST);--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "chat_turn_evals_org_opps_idx" ON "trigger_dashboard_agent"."chat_turn_evals" USING btree ("organization_id","created_at" DESC NULLS LAST) WHERE "trigger_dashboard_agent"."chat_turn_evals"."capability_gap" or "trigger_dashboard_agent"."chat_turn_evals"."docs_gap" or "trigger_dashboard_agent"."chat_turn_evals"."support_opportunity" or "trigger_dashboard_agent"."chat_turn_evals"."feature_request";
|
||||
@@ -0,0 +1,178 @@
|
||||
{
|
||||
"id": "512ce1a7-b31d-4644-9639-3e124b22e52e",
|
||||
"prevId": "00000000-0000-0000-0000-000000000000",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
"trigger_dashboard_agent.chat_sessions": {
|
||||
"name": "chat_sessions",
|
||||
"schema": "trigger_dashboard_agent",
|
||||
"columns": {
|
||||
"chat_id": {
|
||||
"name": "chat_id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true
|
||||
},
|
||||
"public_access_token": {
|
||||
"name": "public_access_token",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"last_event_id": {
|
||||
"name": "last_event_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"run_id": {
|
||||
"name": "run_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"trigger_dashboard_agent.chats": {
|
||||
"name": "chats",
|
||||
"schema": "trigger_dashboard_agent",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true
|
||||
},
|
||||
"organization_id": {
|
||||
"name": "organization_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"user_id": {
|
||||
"name": "user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"title": {
|
||||
"name": "title",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'New chat'"
|
||||
},
|
||||
"messages": {
|
||||
"name": "messages",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'[]'::jsonb"
|
||||
},
|
||||
"metadata": {
|
||||
"name": "metadata",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'{}'::jsonb"
|
||||
},
|
||||
"pinned_at": {
|
||||
"name": "pinned_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"deleted_at": {
|
||||
"name": "deleted_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"last_message_at": {
|
||||
"name": "last_message_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"chats_org_user_last_msg_idx": {
|
||||
"name": "chats_org_user_last_msg_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "organization_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "user_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "last_message_at",
|
||||
"isExpression": false,
|
||||
"asc": false,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"trigger_dashboard_agent\".\"chats\".\"deleted_at\" is null",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
}
|
||||
},
|
||||
"enums": {},
|
||||
"schemas": {
|
||||
"trigger_dashboard_agent": "trigger_dashboard_agent"
|
||||
},
|
||||
"sequences": {},
|
||||
"roles": {},
|
||||
"policies": {},
|
||||
"views": {},
|
||||
"_meta": {
|
||||
"columns": {},
|
||||
"schemas": {},
|
||||
"tables": {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
{
|
||||
"id": "7a42a0cf-9933-4381-adad-98c25105465b",
|
||||
"prevId": "512ce1a7-b31d-4644-9639-3e124b22e52e",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
"trigger_dashboard_agent.chat_sessions": {
|
||||
"name": "chat_sessions",
|
||||
"schema": "trigger_dashboard_agent",
|
||||
"columns": {
|
||||
"chat_id": {
|
||||
"name": "chat_id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true
|
||||
},
|
||||
"public_access_token": {
|
||||
"name": "public_access_token",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"last_event_id": {
|
||||
"name": "last_event_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"run_id": {
|
||||
"name": "run_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"trigger_dashboard_agent.chat_turn_evals": {
|
||||
"name": "chat_turn_evals",
|
||||
"schema": "trigger_dashboard_agent",
|
||||
"columns": {
|
||||
"chat_id": {
|
||||
"name": "chat_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"turn": {
|
||||
"name": "turn",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"organization_id": {
|
||||
"name": "organization_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"user_id": {
|
||||
"name": "user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"agent_run_id": {
|
||||
"name": "agent_run_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"eval_run_id": {
|
||||
"name": "eval_run_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"project_ref": {
|
||||
"name": "project_ref",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"environment": {
|
||||
"name": "environment",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"current_page": {
|
||||
"name": "current_page",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"model": {
|
||||
"name": "model",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"prompt_slug": {
|
||||
"name": "prompt_slug",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"prompt_version": {
|
||||
"name": "prompt_version",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"tools_used": {
|
||||
"name": "tools_used",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'[]'::jsonb"
|
||||
},
|
||||
"tool_error": {
|
||||
"name": "tool_error",
|
||||
"type": "boolean",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": false
|
||||
},
|
||||
"judge_model": {
|
||||
"name": "judge_model",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"score_grounded": {
|
||||
"name": "score_grounded",
|
||||
"type": "smallint",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"score_answered": {
|
||||
"name": "score_answered",
|
||||
"type": "smallint",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"score_concise": {
|
||||
"name": "score_concise",
|
||||
"type": "smallint",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"passed": {
|
||||
"name": "passed",
|
||||
"type": "boolean",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"intent_category": {
|
||||
"name": "intent_category",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"outcome": {
|
||||
"name": "outcome",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"sentiment": {
|
||||
"name": "sentiment",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"capability_gap": {
|
||||
"name": "capability_gap",
|
||||
"type": "boolean",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": false
|
||||
},
|
||||
"docs_gap": {
|
||||
"name": "docs_gap",
|
||||
"type": "boolean",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": false
|
||||
},
|
||||
"support_opportunity": {
|
||||
"name": "support_opportunity",
|
||||
"type": "boolean",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": false
|
||||
},
|
||||
"feature_request": {
|
||||
"name": "feature_request",
|
||||
"type": "boolean",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": false
|
||||
},
|
||||
"topics": {
|
||||
"name": "topics",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'[]'::jsonb"
|
||||
},
|
||||
"signals": {
|
||||
"name": "signals",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'[]'::jsonb"
|
||||
},
|
||||
"summary": {
|
||||
"name": "summary",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"user_text": {
|
||||
"name": "user_text",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"judge": {
|
||||
"name": "judge",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"chat_turn_evals_org_created_idx": {
|
||||
"name": "chat_turn_evals_org_created_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "organization_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "created_at",
|
||||
"isExpression": false,
|
||||
"asc": false,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"chat_turn_evals_org_opps_idx": {
|
||||
"name": "chat_turn_evals_org_opps_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "organization_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "created_at",
|
||||
"isExpression": false,
|
||||
"asc": false,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"trigger_dashboard_agent\".\"chat_turn_evals\".\"capability_gap\" or \"trigger_dashboard_agent\".\"chat_turn_evals\".\"docs_gap\" or \"trigger_dashboard_agent\".\"chat_turn_evals\".\"support_opportunity\" or \"trigger_dashboard_agent\".\"chat_turn_evals\".\"feature_request\"",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {
|
||||
"chat_turn_evals_chat_id_turn_pk": {
|
||||
"name": "chat_turn_evals_chat_id_turn_pk",
|
||||
"columns": ["chat_id", "turn"]
|
||||
}
|
||||
},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"trigger_dashboard_agent.chats": {
|
||||
"name": "chats",
|
||||
"schema": "trigger_dashboard_agent",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true
|
||||
},
|
||||
"organization_id": {
|
||||
"name": "organization_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"user_id": {
|
||||
"name": "user_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"title": {
|
||||
"name": "title",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'New chat'"
|
||||
},
|
||||
"messages": {
|
||||
"name": "messages",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'[]'::jsonb"
|
||||
},
|
||||
"metadata": {
|
||||
"name": "metadata",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'{}'::jsonb"
|
||||
},
|
||||
"pinned_at": {
|
||||
"name": "pinned_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"deleted_at": {
|
||||
"name": "deleted_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"last_message_at": {
|
||||
"name": "last_message_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"chats_org_user_last_msg_idx": {
|
||||
"name": "chats_org_user_last_msg_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "organization_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "user_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "last_message_at",
|
||||
"isExpression": false,
|
||||
"asc": false,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"trigger_dashboard_agent\".\"chats\".\"deleted_at\" is null",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
}
|
||||
},
|
||||
"enums": {},
|
||||
"schemas": {
|
||||
"trigger_dashboard_agent": "trigger_dashboard_agent"
|
||||
},
|
||||
"sequences": {},
|
||||
"roles": {},
|
||||
"policies": {},
|
||||
"views": {},
|
||||
"_meta": {
|
||||
"columns": {},
|
||||
"schemas": {},
|
||||
"tables": {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"entries": [
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "7",
|
||||
"when": 1781711036274,
|
||||
"tag": "0000_magenta_lilandra",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 1,
|
||||
"version": "7",
|
||||
"when": 1781990914081,
|
||||
"tag": "0001_slimy_living_tribunal",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// Pending-status check for the `trigger_dashboard_agent` schema (sibling of
|
||||
// migrate.mjs). Exit 0 = up to date, 1 = pending, 2 = error.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import postgres from "postgres";
|
||||
|
||||
const MIGRATIONS_SCHEMA = "drizzle";
|
||||
const MIGRATIONS_TABLE = "__dashboard_agent_migrations";
|
||||
|
||||
// Match migrate.mjs: same precedence, and expand `${VAR}` refs (see migrate.mjs).
|
||||
const connectionString = (
|
||||
process.env.DASHBOARD_AGENT_DIRECT_URL ??
|
||||
process.env.DASHBOARD_AGENT_DATABASE_URL ??
|
||||
process.env.DIRECT_URL ??
|
||||
process.env.DATABASE_URL
|
||||
)?.replace(/\$\{(\w+)\}/g, (_, k) => process.env[k] ?? "");
|
||||
|
||||
if (!connectionString) {
|
||||
console.error(
|
||||
"[dashboard-agent-db] No database url set (DASHBOARD_AGENT_DIRECT_URL / DASHBOARD_AGENT_DATABASE_URL / DIRECT_URL / DATABASE_URL); cannot check status."
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// Match migrate.mjs: drop the Prisma-style `?schema=` param postgres.js forwards.
|
||||
function normalizeConnectionString(value) {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
url.searchParams.delete("schema");
|
||||
return url.toString();
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
const journalPath = join(dirname(fileURLToPath(import.meta.url)), "drizzle/meta/_journal.json");
|
||||
const sql = postgres(normalizeConnectionString(connectionString), {
|
||||
max: 1,
|
||||
prepare: false,
|
||||
onnotice: () => {},
|
||||
});
|
||||
|
||||
async function main() {
|
||||
const journal = JSON.parse(readFileSync(journalPath, "utf-8"));
|
||||
const entries = [...journal.entries].sort((a, b) => a.when - b.when);
|
||||
|
||||
let lastAppliedAt = -1;
|
||||
try {
|
||||
const rows = await sql`SELECT MAX(created_at)::bigint AS last FROM ${sql(
|
||||
MIGRATIONS_SCHEMA
|
||||
)}.${sql(MIGRATIONS_TABLE)}`;
|
||||
lastAppliedAt = rows[0].last === null ? -1 : Number(rows[0].last);
|
||||
} catch (err) {
|
||||
// 42P01: journal table absent (fresh database), so nothing is applied.
|
||||
if (err.code !== "42P01") throw err;
|
||||
}
|
||||
|
||||
const pending = entries.filter((e) => e.when > lastAppliedAt);
|
||||
console.log(`${entries.length} migration(s) found, ${entries.length - pending.length} applied`);
|
||||
|
||||
if (pending.length > 0) {
|
||||
console.log(`${pending.length} pending migration(s):`);
|
||||
for (const e of pending) console.log(` - ${e.tag}`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
console.log("Dashboard agent schema is up to date");
|
||||
return 0;
|
||||
}
|
||||
|
||||
main()
|
||||
.then((code) => sql.end({ timeout: 5 }).then(() => process.exit(code)))
|
||||
.catch((err) => {
|
||||
console.error(err);
|
||||
sql.end({ timeout: 5 }).finally(() => process.exit(2));
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
// Production migration runner for the `trigger_dashboard_agent` schema.
|
||||
//
|
||||
// Runs under plain `node migrate.mjs` in the built image: `drizzle-orm` and
|
||||
// `postgres` are runtime dependencies, so this needs no `drizzle-kit`, `tsx`,
|
||||
// or build step (keeps the image lean). The OSS container runs this from its
|
||||
// entrypoint; cloud runs it out-of-band against its own database.
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { migrate } from "drizzle-orm/postgres-js/migrator";
|
||||
import postgres from "postgres";
|
||||
|
||||
// Migrations need a direct (non-pooler) connection; a transaction-mode pooler
|
||||
// can't run the migrator. Prefer the agent's direct url, then its pooled url,
|
||||
// then the main DIRECT_URL/DATABASE_URL (OSS single-database fallback; tables
|
||||
// still land in the `trigger_dashboard_agent` schema).
|
||||
// `.replace` expands `${VAR}` refs (e.g. the repo .env's DIRECT_URL=${DATABASE_URL});
|
||||
// node's --env-file loads them literally, unlike Prisma's dotenv-expand.
|
||||
const connectionString = (
|
||||
process.env.DASHBOARD_AGENT_DIRECT_URL ??
|
||||
process.env.DASHBOARD_AGENT_DATABASE_URL ??
|
||||
process.env.DIRECT_URL ??
|
||||
process.env.DATABASE_URL
|
||||
)?.replace(/\$\{(\w+)\}/g, (_, k) => process.env[k] ?? "");
|
||||
|
||||
if (!connectionString) {
|
||||
console.error(
|
||||
"[dashboard-agent-db] No database url set (DASHBOARD_AGENT_DIRECT_URL / DASHBOARD_AGENT_DATABASE_URL / DIRECT_URL / DATABASE_URL); cannot migrate."
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Prisma-style URLs carry `?schema=...`; postgres.js forwards unknown query
|
||||
// params as server startup config and Postgres rejects `schema`. Our tables are
|
||||
// schema-qualified, so the param is unnecessary — drop it.
|
||||
function normalizeConnectionString(value) {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
url.searchParams.delete("schema");
|
||||
return url.toString();
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
const migrationsFolder = join(dirname(fileURLToPath(import.meta.url)), "drizzle");
|
||||
const sql = postgres(normalizeConnectionString(connectionString), {
|
||||
max: 1,
|
||||
prepare: false,
|
||||
// Silence the "schema/relation already exists, skipping" notices the journal's
|
||||
// idempotent CREATE IF NOT EXISTS emits on every re-run, so restart logs stay clean.
|
||||
onnotice: () => {},
|
||||
});
|
||||
|
||||
try {
|
||||
// Track our history in a DEDICATED journal table. Drizzle's migrator reads the
|
||||
// latest row from <schema>.<table> by created_at and skips any journal entry
|
||||
// dated at or before it. The default `drizzle.__drizzle_migrations` is shared
|
||||
// by every Drizzle app, so when this DB is shared (OSS single-database fallback
|
||||
// and the enterprise-image E2E gate) billing's journal rows poison ours: a
|
||||
// billing row dated between our 0000 and 0001 makes the migrator skip 0000 (the
|
||||
// CREATE SCHEMA) and run 0001 against a schema that never got created. An own
|
||||
// table keeps our history independent (enterprise/db does the same with
|
||||
// __enterprise_migrations; billing keeps the default).
|
||||
//
|
||||
// The table stays in the `drizzle` schema, not our data schema, so 0000's
|
||||
// `CREATE SCHEMA "trigger_dashboard_agent"` doesn't collide with the schema the
|
||||
// migrator pre-creates for its journal.
|
||||
await migrate(drizzle(sql), {
|
||||
migrationsFolder,
|
||||
migrationsTable: "__dashboard_agent_migrations",
|
||||
});
|
||||
console.log("[dashboard-agent-db] migrations complete");
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "@internal/dashboard-agent-db",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"drizzle-orm": "^0.45.0",
|
||||
"postgres": "^3.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"drizzle-kit": "^0.31.0"
|
||||
},
|
||||
"scripts": {
|
||||
"typecheck": "tsc --noEmit",
|
||||
"db:generate": "drizzle-kit generate",
|
||||
"db:migrate": "drizzle-kit migrate",
|
||||
"db:migrate:deploy": "node --env-file-if-exists=../../.env migrate.mjs",
|
||||
"db:migrate:status": "node --env-file-if-exists=../../.env migrate-status.mjs",
|
||||
"db:studio": "drizzle-kit studio"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { drizzle, type PostgresJsDatabase } from "drizzle-orm/postgres-js";
|
||||
import postgres, { type Sql } from "postgres";
|
||||
import * as schema from "./schema.js";
|
||||
|
||||
export type DashboardAgentSchema = typeof schema;
|
||||
export type DashboardAgentDb = PostgresJsDatabase<DashboardAgentSchema>;
|
||||
|
||||
export interface DashboardAgentDbClient {
|
||||
db: DashboardAgentDb;
|
||||
sql: Sql;
|
||||
/** Close the underlying connection pool. Call on agent run shutdown. */
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
export interface CreateDashboardAgentDbOptions {
|
||||
/**
|
||||
* Max client-side pool size. Keep small — the agent runs in many short-lived
|
||||
* task containers and PlanetScale's pooler does the real connection pooling.
|
||||
*/
|
||||
max?: number;
|
||||
/** Idle timeout (seconds) so suspended agent runs release connections. */
|
||||
idleTimeoutSeconds?: number;
|
||||
/** Connection timeout (seconds). */
|
||||
connectTimeoutSeconds?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a Drizzle client for the dashboard-agent datastore. Shared by the agent
|
||||
* task (its own persistence) and the webapp (History tab + frontend actions).
|
||||
*
|
||||
* Connections go through a transaction-mode pooler (PlanetScale / PgBouncer-style),
|
||||
* so prepared statements are disabled — they don't survive a connection being
|
||||
* handed to a different client between checkouts.
|
||||
*/
|
||||
// Prisma-style URLs carry `?schema=...`; postgres.js forwards unknown query
|
||||
// params as server startup config and Postgres rejects `schema`. Our tables are
|
||||
// schema-qualified, so the param is unnecessary — drop it. Matters for the OSS
|
||||
// fallback to the main DATABASE_URL.
|
||||
function normalizeConnectionString(connectionString: string): string {
|
||||
try {
|
||||
const url = new URL(connectionString);
|
||||
url.searchParams.delete("schema");
|
||||
return url.toString();
|
||||
} catch {
|
||||
return connectionString;
|
||||
}
|
||||
}
|
||||
|
||||
export function createDashboardAgentDb(
|
||||
connectionString: string,
|
||||
options: CreateDashboardAgentDbOptions = {}
|
||||
): DashboardAgentDbClient {
|
||||
const sql = postgres(normalizeConnectionString(connectionString), {
|
||||
max: options.max ?? 5,
|
||||
idle_timeout: options.idleTimeoutSeconds ?? 20,
|
||||
connect_timeout: options.connectTimeoutSeconds ?? 10,
|
||||
prepare: false,
|
||||
});
|
||||
|
||||
const db = drizzle(sql, { schema });
|
||||
|
||||
return {
|
||||
db,
|
||||
sql,
|
||||
close: () => sql.end(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from "./schema.js";
|
||||
export * from "./client.js";
|
||||
export * from "./queries.js";
|
||||
@@ -0,0 +1,271 @@
|
||||
import { and, desc, eq, isNull, sql } from "drizzle-orm";
|
||||
import type { DashboardAgentDb } from "./client.js";
|
||||
import {
|
||||
chats,
|
||||
chatSessions,
|
||||
chatTurnEvals,
|
||||
type ChatSession,
|
||||
type NewChatTurnEval,
|
||||
} from "./schema.js";
|
||||
|
||||
/**
|
||||
* The access-pattern layer. Every query that touches user data is scoped by
|
||||
* `organizationId` and/or `userId` so tenant isolation lives in one place —
|
||||
* callers can't forget the `where`. Shared by the agent task and the webapp.
|
||||
*/
|
||||
|
||||
/** Placeholder title for a chat with no generated or user-set title yet. */
|
||||
export const DEFAULT_CHAT_TITLE = "New chat";
|
||||
|
||||
export interface ChatListItem {
|
||||
id: string;
|
||||
title: string;
|
||||
pinnedAt: Date | null;
|
||||
lastMessageAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* #1 History tab: a user's chats within an org, recent first, pinned on top.
|
||||
* Deliberately selects metadata columns only — never `messages` (large blob) or
|
||||
* the session token. Covered by `chats_org_user_last_msg_idx`.
|
||||
*/
|
||||
export async function listChats(
|
||||
db: DashboardAgentDb,
|
||||
params: { organizationId: string; userId: string; limit?: number }
|
||||
): Promise<ChatListItem[]> {
|
||||
return db
|
||||
.select({
|
||||
id: chats.id,
|
||||
title: chats.title,
|
||||
pinnedAt: chats.pinnedAt,
|
||||
lastMessageAt: chats.lastMessageAt,
|
||||
createdAt: chats.createdAt,
|
||||
updatedAt: chats.updatedAt,
|
||||
metadata: chats.metadata,
|
||||
})
|
||||
.from(chats)
|
||||
.where(
|
||||
and(
|
||||
eq(chats.organizationId, params.organizationId),
|
||||
eq(chats.userId, params.userId),
|
||||
isNull(chats.deletedAt)
|
||||
)
|
||||
)
|
||||
.orderBy(sql`${chats.pinnedAt} desc nulls last`, desc(chats.lastMessageAt))
|
||||
.limit(params.limit ?? 50);
|
||||
}
|
||||
|
||||
/**
|
||||
* #2 Open a chat: the stored transcript for `useChat`'s initialMessages.
|
||||
* Scoped to the owner; returns null if missing/deleted/not theirs.
|
||||
*/
|
||||
export async function getChatMessages(
|
||||
db: DashboardAgentDb,
|
||||
params: { chatId: string; userId: string }
|
||||
): Promise<unknown[] | null> {
|
||||
const rows = await db
|
||||
.select({ messages: chats.messages })
|
||||
.from(chats)
|
||||
.where(
|
||||
and(eq(chats.id, params.chatId), eq(chats.userId, params.userId), isNull(chats.deletedAt))
|
||||
)
|
||||
.limit(1);
|
||||
return rows[0]?.messages ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* #3 Resume the transport on first paint: the session-scoped token + stream
|
||||
* cursor. Joins `chats` to scope by owner (chat_sessions has no userId).
|
||||
*/
|
||||
export async function getSession(
|
||||
db: DashboardAgentDb,
|
||||
params: { chatId: string; userId: string }
|
||||
): Promise<ChatSession | null> {
|
||||
const rows = await db
|
||||
.select({
|
||||
chatId: chatSessions.chatId,
|
||||
publicAccessToken: chatSessions.publicAccessToken,
|
||||
lastEventId: chatSessions.lastEventId,
|
||||
runId: chatSessions.runId,
|
||||
updatedAt: chatSessions.updatedAt,
|
||||
})
|
||||
.from(chatSessions)
|
||||
.innerJoin(chats, eq(chats.id, chatSessions.chatId))
|
||||
.where(and(eq(chatSessions.chatId, params.chatId), eq(chats.userId, params.userId)))
|
||||
.limit(1);
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owner check: true when a non-deleted chat with this id belongs to the user.
|
||||
* Used to authorize chat-scoped actions (e.g. minting a session token) before
|
||||
* a session row necessarily exists.
|
||||
*/
|
||||
export async function chatExists(
|
||||
db: DashboardAgentDb,
|
||||
params: { chatId: string; userId: string; organizationId: string }
|
||||
): Promise<boolean> {
|
||||
const rows = await db
|
||||
.select({ id: chats.id })
|
||||
.from(chats)
|
||||
.where(
|
||||
and(
|
||||
eq(chats.id, params.chatId),
|
||||
eq(chats.organizationId, params.organizationId),
|
||||
eq(chats.userId, params.userId),
|
||||
isNull(chats.deletedAt)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* #4 Create a chat. Idempotent (`onConflictDoNothing`) so the webapp's "new
|
||||
* chat" insert and the agent's defensive `onChatStart` ensure can't race into a
|
||||
* duplicate-key error.
|
||||
*/
|
||||
export async function createChat(
|
||||
db: DashboardAgentDb,
|
||||
params: {
|
||||
id: string;
|
||||
organizationId: string;
|
||||
userId: string;
|
||||
title?: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
): Promise<void> {
|
||||
await db
|
||||
.insert(chats)
|
||||
.values({
|
||||
id: params.id,
|
||||
organizationId: params.organizationId,
|
||||
userId: params.userId,
|
||||
title: params.title ?? DEFAULT_CHAT_TITLE,
|
||||
metadata: params.metadata ?? {},
|
||||
})
|
||||
.onConflictDoNothing();
|
||||
}
|
||||
|
||||
/** The agent's defensive ensure-exists in `onChatStart` / `onPreload`. */
|
||||
export const ensureChat = createChat;
|
||||
|
||||
/** #5 Rename. */
|
||||
export async function renameChat(
|
||||
db: DashboardAgentDb,
|
||||
params: { chatId: string; userId: string; title: string }
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(chats)
|
||||
.set({ title: params.title, updatedAt: sql`now()` })
|
||||
.where(and(eq(chats.id, params.chatId), eq(chats.userId, params.userId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* #5 Set an auto-generated title, but only while the chat still has the default
|
||||
* title. Conditional on `DEFAULT_CHAT_TITLE` so the background title write can't
|
||||
* clobber a user rename, and so it's a safe no-op if it runs more than once.
|
||||
*/
|
||||
export async function setChatTitleIfDefault(
|
||||
db: DashboardAgentDb,
|
||||
params: { chatId: string; title: string }
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(chats)
|
||||
.set({ title: params.title, updatedAt: sql`now()` })
|
||||
.where(
|
||||
and(eq(chats.id, params.chatId), eq(chats.title, DEFAULT_CHAT_TITLE), isNull(chats.deletedAt))
|
||||
);
|
||||
}
|
||||
|
||||
/** #5 Pin / unpin. */
|
||||
export async function setChatPinned(
|
||||
db: DashboardAgentDb,
|
||||
params: { chatId: string; userId: string; pinned: boolean }
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(chats)
|
||||
.set({ pinnedAt: params.pinned ? sql`now()` : null, updatedAt: sql`now()` })
|
||||
.where(and(eq(chats.id, params.chatId), eq(chats.userId, params.userId)));
|
||||
}
|
||||
|
||||
/** #5 Soft-delete. */
|
||||
export async function softDeleteChat(
|
||||
db: DashboardAgentDb,
|
||||
params: { chatId: string; userId: string }
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(chats)
|
||||
.set({ deletedAt: sql`now()`, updatedAt: sql`now()` })
|
||||
.where(and(eq(chats.id, params.chatId), eq(chats.userId, params.userId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* #6a Persist messages only (agent `onTurnStart` — make the user's message
|
||||
* durable in the display copy before the model starts streaming).
|
||||
*/
|
||||
export async function persistMessages(
|
||||
db: DashboardAgentDb,
|
||||
params: { chatId: string; messages: unknown[] }
|
||||
): Promise<void> {
|
||||
await db
|
||||
.update(chats)
|
||||
.set({ messages: params.messages, lastMessageAt: sql`now()`, updatedAt: sql`now()` })
|
||||
.where(eq(chats.id, params.chatId));
|
||||
}
|
||||
|
||||
/**
|
||||
* #6b Persist a completed turn (agent `onTurnComplete`): the finalized transcript
|
||||
* and the refreshed session state, in one transaction. Atomicity matters — on
|
||||
* the next page load the frontend reads `messages` and `lastEventId` in parallel;
|
||||
* a torn write can resume from a stale cursor and double-render the last turn.
|
||||
*/
|
||||
export async function persistTurn(
|
||||
db: DashboardAgentDb,
|
||||
params: {
|
||||
chatId: string;
|
||||
messages: unknown[];
|
||||
session: {
|
||||
publicAccessToken: string;
|
||||
lastEventId?: string | null;
|
||||
runId?: string | null;
|
||||
};
|
||||
}
|
||||
): Promise<void> {
|
||||
await db.transaction(async (tx) => {
|
||||
await tx
|
||||
.update(chats)
|
||||
.set({ messages: params.messages, lastMessageAt: sql`now()`, updatedAt: sql`now()` })
|
||||
.where(eq(chats.id, params.chatId));
|
||||
|
||||
await tx
|
||||
.insert(chatSessions)
|
||||
.values({
|
||||
chatId: params.chatId,
|
||||
publicAccessToken: params.session.publicAccessToken,
|
||||
lastEventId: params.session.lastEventId ?? null,
|
||||
runId: params.session.runId ?? null,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: chatSessions.chatId,
|
||||
set: {
|
||||
publicAccessToken: params.session.publicAccessToken,
|
||||
lastEventId: params.session.lastEventId ?? null,
|
||||
runId: params.session.runId ?? null,
|
||||
updatedAt: sql`now()`,
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* #11 Record a turn eval. Idempotent on `(chatId, turn)` so a re-delivered turn
|
||||
* (the eval task is triggered with an idempotency key, and may still retry) can
|
||||
* never write a second row.
|
||||
*/
|
||||
export async function insertTurnEval(db: DashboardAgentDb, row: NewChatTurnEval): Promise<void> {
|
||||
await db.insert(chatTurnEvals).values(row).onConflictDoNothing();
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import {
|
||||
boolean,
|
||||
index,
|
||||
integer,
|
||||
jsonb,
|
||||
pgSchema,
|
||||
primaryKey,
|
||||
smallint,
|
||||
text,
|
||||
timestamp,
|
||||
} from "drizzle-orm/pg-core";
|
||||
|
||||
/**
|
||||
* All dashboard-agent tables live in a dedicated Postgres schema. In cloud this
|
||||
* is a separate PlanetScale database; in OSS it isolates the agent's tables from
|
||||
* Prisma's `public` schema inside the main database. Tables are schema-qualified
|
||||
* explicitly, so no `search_path` configuration is required on the connection.
|
||||
*/
|
||||
export const dashboardAgentSchema = pgSchema("trigger_dashboard_agent");
|
||||
|
||||
/**
|
||||
* One row per conversation. Scope is **org + user** — a chat is not bound to a
|
||||
* single project/env; the project/env it ran in (and any extra ones the user
|
||||
* adds to context) live in `metadata`, because one conversation can range over
|
||||
* several projects/envs.
|
||||
*
|
||||
* `messages` is a display copy of the `UIMessage[]` transcript. The model's
|
||||
* source of truth for history is chat.agent's built-in object-store snapshot,
|
||||
* not this column — a stale write here can make the History view lag a turn but
|
||||
* can never corrupt what the model sees.
|
||||
*
|
||||
* Foreign-key-free: `organizationId` / `userId` are main-DB ids with no FK,
|
||||
* because in cloud this table lives in a different database.
|
||||
*/
|
||||
export const chats = dashboardAgentSchema.table(
|
||||
"chats",
|
||||
{
|
||||
// = chatId = the Session externalId. Stable for the life of the thread.
|
||||
id: text("id").primaryKey(),
|
||||
organizationId: text("organization_id").notNull(),
|
||||
userId: text("user_id").notNull(),
|
||||
title: text("title").notNull().default("New chat"),
|
||||
// UIMessage[] display copy — never read to rebuild model context.
|
||||
messages: jsonb("messages").$type<unknown[]>().notNull().default([]),
|
||||
// Project/env context + model choice + page snapshot. Flexible by design.
|
||||
metadata: jsonb("metadata").$type<Record<string, unknown>>().notNull().default({}),
|
||||
pinnedAt: timestamp("pinned_at", { withTimezone: true }),
|
||||
deletedAt: timestamp("deleted_at", { withTimezone: true }),
|
||||
lastMessageAt: timestamp("last_message_at", { withTimezone: true }),
|
||||
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
// History tab: "my chats in this org, recent first". Partial index keeps
|
||||
// soft-deleted rows out of the hot path.
|
||||
index("chats_org_user_last_msg_idx")
|
||||
.on(t.organizationId, t.userId, t.lastMessageAt.desc())
|
||||
.where(sql`${t.deletedAt} is null`),
|
||||
]
|
||||
);
|
||||
|
||||
/**
|
||||
* Live transport state the frontend needs to resume a chat on first paint,
|
||||
* keyed by chatId. Separate from `chats` so the secret token is isolated from
|
||||
* list queries and the hot per-turn write stays off the conversation row.
|
||||
*
|
||||
* No `userId` here on purpose: the agent's `onTurnComplete` event doesn't carry
|
||||
* `clientData`, and ownership is already enforced via the `chats` row — the
|
||||
* resume query joins `chats` to scope by owner (see `getSession`).
|
||||
*/
|
||||
export const chatSessions = dashboardAgentSchema.table("chat_sessions", {
|
||||
chatId: text("chat_id").primaryKey(), // = chats.id (FK-free, cross-db)
|
||||
publicAccessToken: text("public_access_token").notNull(),
|
||||
lastEventId: text("last_event_id"),
|
||||
runId: text("run_id"), // telemetry / "view this run"
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
});
|
||||
|
||||
/**
|
||||
* One row per evaluated turn, written by the `dashboard-agent-eval-turn` task
|
||||
* that the agent triggers from `onTurnComplete`. Two kinds of data: quality
|
||||
* scores (did the agent answer well, grounded in its tool results) and insight
|
||||
* classification (what the user wanted, whether we have a product/docs/support
|
||||
* gap). Append-only analytics; the higher-level views ("top capability gaps",
|
||||
* "what users struggle with") are aggregations over these rows, not stored here.
|
||||
*
|
||||
* Structured columns are the things we filter, alert, and chart on; the evolving
|
||||
* taxonomy (typed `signals`) and the raw judge output live in JSONB so adding a
|
||||
* signal type is never a migration. Org + user scoped, FK-free (cross-db), with
|
||||
* a composite `(chatId, turn)` key so a re-delivered turn can't double-insert.
|
||||
*/
|
||||
export const chatTurnEvals = dashboardAgentSchema.table(
|
||||
"chat_turn_evals",
|
||||
{
|
||||
chatId: text("chat_id").notNull(), // = chats.id
|
||||
turn: integer("turn").notNull(), // 0-indexed turn within the chat
|
||||
organizationId: text("organization_id").notNull(),
|
||||
userId: text("user_id").notNull(),
|
||||
agentRunId: text("agent_run_id"), // the chat.agent run that produced the turn
|
||||
evalRunId: text("eval_run_id"), // the eval task's own run, for tracing
|
||||
// Per-turn context (the project/env/page the user was looking at).
|
||||
projectRef: text("project_ref"),
|
||||
environment: text("environment"),
|
||||
currentPage: text("current_page"),
|
||||
// Operational + model. `promptVersion` lets a quality drop be attributed to a
|
||||
// dashboard-managed prompt edit that never went through CI.
|
||||
model: text("model"),
|
||||
promptSlug: text("prompt_slug"),
|
||||
promptVersion: integer("prompt_version"),
|
||||
toolsUsed: jsonb("tools_used").$type<string[]>().notNull().default([]),
|
||||
toolError: boolean("tool_error").notNull().default(false),
|
||||
// Quality (LLM judge), scored 1-5.
|
||||
judgeModel: text("judge_model"),
|
||||
scoreGrounded: smallint("score_grounded"),
|
||||
scoreAnswered: smallint("score_answered"),
|
||||
scoreConcise: smallint("score_concise"),
|
||||
passed: boolean("passed"),
|
||||
// Insight classification — the filterable summary of `signals`.
|
||||
intentCategory: text("intent_category"),
|
||||
outcome: text("outcome"), // resolved | partial | unresolved | deflected
|
||||
sentiment: text("sentiment"),
|
||||
capabilityGap: boolean("capability_gap").notNull().default(false),
|
||||
docsGap: boolean("docs_gap").notNull().default(false),
|
||||
supportOpportunity: boolean("support_opportunity").notNull().default(false),
|
||||
featureRequest: boolean("feature_request").notNull().default(false),
|
||||
// Rich / evolving.
|
||||
topics: jsonb("topics").$type<string[]>().notNull().default([]),
|
||||
signals: jsonb("signals").$type<unknown[]>().notNull().default([]),
|
||||
summary: text("summary"),
|
||||
userText: text("user_text"), // the user's question (clustering input)
|
||||
judge: jsonb("judge").$type<Record<string, unknown>>(), // full raw verdict
|
||||
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
primaryKey({ columns: [t.chatId, t.turn] }),
|
||||
// "what happened in this org lately", recent first.
|
||||
index("chat_turn_evals_org_created_idx").on(t.organizationId, t.createdAt.desc()),
|
||||
// The opportunities feed: gaps, struggles, support, feature asks.
|
||||
index("chat_turn_evals_org_opps_idx")
|
||||
.on(t.organizationId, t.createdAt.desc())
|
||||
.where(
|
||||
sql`${t.capabilityGap} or ${t.docsGap} or ${t.supportOpportunity} or ${t.featureRequest}`
|
||||
),
|
||||
]
|
||||
);
|
||||
|
||||
export type Chat = typeof chats.$inferSelect;
|
||||
export type NewChat = typeof chats.$inferInsert;
|
||||
export type ChatSession = typeof chatSessions.$inferSelect;
|
||||
export type NewChatSession = typeof chatSessions.$inferInsert;
|
||||
export type ChatTurnEval = typeof chatTurnEvals.$inferSelect;
|
||||
export type NewChatTurnEval = typeof chatTurnEvals.$inferInsert;
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"strict": true
|
||||
},
|
||||
"exclude": ["node_modules", "drizzle"]
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
node_modules
|
||||
.trigger
|
||||
.env
|
||||
@@ -0,0 +1,42 @@
|
||||
# @internal/dashboard-agent
|
||||
|
||||
The in-dashboard agent, built on `chat.agent` and deployed as its own Trigger
|
||||
project. This is the launch-week dogfood: we run our own product on the
|
||||
primitive we ship.
|
||||
|
||||
## Why a separate package (not inside apps/webapp)
|
||||
|
||||
The agent has **no access to the main database, ClickHouse, or webapp
|
||||
internals** — it reads everything via the API. Living in a standalone package
|
||||
that doesn't depend on the webapp makes that firewall **structural**: the
|
||||
package physically cannot import webapp server code. It also keeps the webapp a
|
||||
pure Remix app instead of a dual Remix-app-and-Trigger-project, and gives the
|
||||
agent a small, fast, independently deployable + testable build context.
|
||||
|
||||
It writes conversation state to its own datastore via `@internal/dashboard-agent-db`
|
||||
(the same package the webapp reads from for the History tab). It never touches
|
||||
Prisma.
|
||||
|
||||
## Deploy / dev
|
||||
|
||||
This is a Trigger project with its own `trigger.config.ts`. The project ref is
|
||||
read from `TRIGGER_DASHBOARD_AGENT_PROJECT_REF` (never hardcoded — public repo).
|
||||
|
||||
```bash
|
||||
cd internal-packages/dashboard-agent
|
||||
TRIGGER_DASHBOARD_AGENT_PROJECT_REF=<your-project> pnpm run dev # trigger dev
|
||||
TRIGGER_DASHBOARD_AGENT_PROJECT_REF=<your-project> pnpm run deploy # trigger deploy
|
||||
```
|
||||
|
||||
Runtime env the deployed task needs: `DASHBOARD_AGENT_DATABASE_URL` (the agent
|
||||
datastore) and `OBJECT_STORE_*` (chat.agent's built-in conversation snapshot).
|
||||
|
||||
## Consumed by the webapp
|
||||
|
||||
The webapp imports only the task **type** for transport type-safety:
|
||||
|
||||
```ts
|
||||
import type { dashboardAgent } from "@internal/dashboard-agent";
|
||||
```
|
||||
|
||||
Never a value import (see `src/index.ts`).
|
||||
@@ -0,0 +1,33 @@
|
||||
// Load the monorepo root .env (the package's .env is a symlink to it) so the
|
||||
// real-model evals pick up ANTHROPIC_API_KEY without the caller exporting it.
|
||||
// Runs as a vitest setupFile before the eval modules are imported, so the
|
||||
// `ANTHROPIC_API_KEY` gate in dashboard-agent.eval.ts sees the loaded value.
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const candidates = [
|
||||
resolve(here, ".env"), // package symlink to the root .env
|
||||
resolve(here, "../../.env"), // monorepo root
|
||||
resolve(process.cwd(), ".env"),
|
||||
];
|
||||
|
||||
for (const path of candidates) {
|
||||
if (!existsSync(path)) continue;
|
||||
for (const line of readFileSync(path, "utf8").split("\n")) {
|
||||
const match = /^\s*([A-Z0-9_]+)\s*=\s*(.*)$/.exec(line);
|
||||
if (!match) continue;
|
||||
const key = match[1]!;
|
||||
if (process.env[key] !== undefined) continue;
|
||||
let value = match[2]!.trim();
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
process.env[key] = value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"name": "@internal/dashboard-agent",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./tool-schemas": "./src/tool-schemas.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.0",
|
||||
"@internal/dashboard-agent-db": "workspace:*",
|
||||
"@trigger.dev/sdk": "workspace:*",
|
||||
"ai": "^6.0.116",
|
||||
"zod": "3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@ai-sdk/provider": "3.0.8",
|
||||
"@trigger.dev/build": "workspace:*",
|
||||
"trigger.dev": "workspace:*",
|
||||
"vitest": "4.1.7"
|
||||
},
|
||||
"scripts": {
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:evals": "vitest run --config vitest.eval.config.ts",
|
||||
"dev": "trigger dev",
|
||||
"deploy": "trigger deploy"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
// Evals for the dashboard agent: they run the REAL model through the agent and
|
||||
// score behavior, which unit tests (mock model) can't. Two layers, following
|
||||
// common practice (DeepEval "tool correctness", Vercel AI SDK + vitest evals,
|
||||
// LLM-as-judge with an analytic rubric):
|
||||
//
|
||||
// 1. Tool selection — does the model pick the right read tool for a question?
|
||||
// Scored as an aggregate pass rate with a threshold, since a real model is
|
||||
// nondeterministic; a single miss shouldn't red the suite, a trend should.
|
||||
// 2. Answer quality — an LLM judge scores the final answer against the tool
|
||||
// data it was given (reason-before-score, structured output, grounded on
|
||||
// facts to blunt verbosity/self-enhancement bias).
|
||||
//
|
||||
// These hit the real Anthropic API (cost + nondeterminism), so they live in
|
||||
// `*.eval.ts` (not run by `pnpm test`) and skip unless ANTHROPIC_API_KEY is set.
|
||||
// Run with `pnpm --filter @internal/dashboard-agent run test:evals`.
|
||||
//
|
||||
// `@trigger.dev/sdk/ai/test` first so the resource catalog installs before the
|
||||
// agent module registers.
|
||||
import { mockChatAgent } from "@trigger.dev/sdk/ai/test";
|
||||
|
||||
import { anthropic } from "@ai-sdk/anthropic";
|
||||
import { generateObject, tool, type ToolSet, type UIMessage, type UIMessageChunk } from "ai";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
dashboardAgent,
|
||||
dashboardAgentModelKey,
|
||||
dashboardAgentStoreKey,
|
||||
dashboardAgentToolsKey,
|
||||
type DashboardAgentStore,
|
||||
} from "./dashboard-agent";
|
||||
import { dashboardAgentToolSchemas } from "./tool-schemas";
|
||||
|
||||
const HAS_KEY = Boolean(process.env.ANTHROPIC_API_KEY);
|
||||
// The agent's real model; a capable judge (a stronger/different judge would
|
||||
// further reduce self-enhancement bias).
|
||||
const AGENT_MODEL = "claude-sonnet-4-6";
|
||||
const JUDGE_MODEL = "claude-sonnet-4-6";
|
||||
|
||||
const CLIENT_DATA = { userId: "user_eval", organizationId: "org_eval" };
|
||||
const NOOP_STORE: DashboardAgentStore = {
|
||||
ensureChat: async () => {},
|
||||
persistMessages: async () => {},
|
||||
persistTurn: async () => {},
|
||||
setChatTitleIfDefault: async () => {},
|
||||
};
|
||||
|
||||
// Realistic, fixed tool results so the model has something concrete to act on
|
||||
// and the judge has a ground truth to check the answer against.
|
||||
const FIXTURES: Record<string, unknown> = {
|
||||
list_projects: {
|
||||
projects: [{ ref: "proj_eval1", name: "Checkout", slug: "checkout", organization: "Acme" }],
|
||||
},
|
||||
list_environments: {
|
||||
environments: [
|
||||
{ slug: "dev", type: "DEVELOPMENT", paused: false },
|
||||
{ slug: "prod", type: "PRODUCTION", paused: false },
|
||||
],
|
||||
},
|
||||
list_tasks: {
|
||||
tasks: [
|
||||
{ slug: "send-receipt", filePath: "src/trigger/receipt.ts", triggerSource: "STANDARD" },
|
||||
{ slug: "nightly-rollup", filePath: "src/trigger/rollup.ts", triggerSource: "SCHEDULED" },
|
||||
],
|
||||
},
|
||||
list_runs: {
|
||||
runs: [
|
||||
{ id: "run_a1", status: "FAILED", taskIdentifier: "send-receipt", durationMs: 0 },
|
||||
{ id: "run_a2", status: "COMPLETED", taskIdentifier: "send-receipt", durationMs: 1200 },
|
||||
],
|
||||
nextCursor: undefined,
|
||||
},
|
||||
get_run: {
|
||||
id: "run_a1",
|
||||
status: "FAILED",
|
||||
taskIdentifier: "send-receipt",
|
||||
durationMs: 0,
|
||||
error: { name: "TimeoutError", message: "Stripe API timed out after 30s" },
|
||||
},
|
||||
get_run_trace: {
|
||||
traceId: "trace_a1",
|
||||
spans: [
|
||||
{ depth: 0, task: "send-receipt", durationMs: 30010, isError: true, message: "run" },
|
||||
{ depth: 1, durationMs: 30000, isError: true, message: "POST api.stripe.com/charges" },
|
||||
],
|
||||
truncated: false,
|
||||
},
|
||||
list_errors: {
|
||||
errors: [
|
||||
{
|
||||
id: "error_stripe",
|
||||
taskIdentifier: "send-receipt",
|
||||
errorType: "TimeoutError",
|
||||
errorMessage: "Stripe API timed out after 30s",
|
||||
status: "unresolved",
|
||||
count: 37,
|
||||
},
|
||||
{
|
||||
id: "error_oom",
|
||||
taskIdentifier: "nightly-rollup",
|
||||
errorType: "OutOfMemoryError",
|
||||
errorMessage: "JS heap out of memory",
|
||||
status: "ignored",
|
||||
count: 4,
|
||||
},
|
||||
],
|
||||
nextCursor: undefined,
|
||||
},
|
||||
get_error: {
|
||||
id: "error_stripe",
|
||||
taskIdentifier: "send-receipt",
|
||||
errorType: "TimeoutError",
|
||||
errorMessage: "Stripe API timed out after 30s",
|
||||
status: "unresolved",
|
||||
count: 37,
|
||||
affectedVersions: ["20260101.1", "20260102.1"],
|
||||
resolvedAt: null,
|
||||
},
|
||||
};
|
||||
|
||||
// Real schemas (so the model sees the real tool descriptions) + stubbed executes
|
||||
// that record each call (a spy) and return the fixture. This is the seam that
|
||||
// lets us observe tool selection and judge answers with no live API.
|
||||
function makeFixtureTools(calls: Array<{ tool: string; input: unknown }>): ToolSet {
|
||||
const entries = Object.entries(dashboardAgentToolSchemas).map(([name, schema]) => {
|
||||
const s = schema as { description?: string; inputSchema: z.ZodTypeAny };
|
||||
const withExecute = tool({
|
||||
description: s.description,
|
||||
inputSchema: s.inputSchema,
|
||||
execute: async (input: unknown) => {
|
||||
calls.push({ tool: name, input });
|
||||
// render_view is a presentation tool: it echoes the spec, like the real one.
|
||||
if (name === "render_view") return input;
|
||||
return FIXTURES[name] ?? {};
|
||||
},
|
||||
});
|
||||
return [name, withExecute] as const;
|
||||
});
|
||||
return Object.fromEntries(entries) as ToolSet;
|
||||
}
|
||||
|
||||
function userMessage(text: string, id = "u1"): UIMessage {
|
||||
return { id, role: "user", parts: [{ type: "text", text }] };
|
||||
}
|
||||
|
||||
function collectText(chunks: UIMessageChunk[]): string {
|
||||
return chunks
|
||||
.filter((c): c is Extract<UIMessageChunk, { type: "text-delta" }> => c.type === "text-delta")
|
||||
.map((c) => c.delta)
|
||||
.join("");
|
||||
}
|
||||
|
||||
let caseCounter = 0;
|
||||
|
||||
async function runCase(question: string): Promise<{
|
||||
calls: Array<{ tool: string; input: unknown }>;
|
||||
answer: string;
|
||||
}> {
|
||||
const calls: Array<{ tool: string; input: unknown }> = [];
|
||||
const harness = mockChatAgent(dashboardAgent, {
|
||||
chatId: `eval_${caseCounter++}`,
|
||||
clientData: CLIENT_DATA,
|
||||
setupLocals: ({ set }) => {
|
||||
set(dashboardAgentStoreKey, NOOP_STORE);
|
||||
set(dashboardAgentModelKey, anthropic(AGENT_MODEL));
|
||||
set(dashboardAgentToolsKey, makeFixtureTools(calls));
|
||||
},
|
||||
});
|
||||
try {
|
||||
const turn = await harness.sendMessage(userMessage(question));
|
||||
return { calls, answer: collectText(turn.chunks) };
|
||||
} finally {
|
||||
await harness.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LLM-as-judge: analytic rubric, reason-before-score, structured output.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Verdict = z.object({
|
||||
reasoning: z.string().describe("One or two sentences of reasoning, written BEFORE the scores."),
|
||||
grounded: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(5)
|
||||
.describe(
|
||||
"Is every fact in the answer present in the tool data? Penalize any run id, error name, count, status, version, or metric not in the data. 5 = fully grounded, 1 = fabricated."
|
||||
),
|
||||
answersQuestion: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(5)
|
||||
.describe("Does the answer directly address the user's question? 5 = fully, 1 = not at all."),
|
||||
concise: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(5)
|
||||
.describe("Direct and free of padding. Do not reward length."),
|
||||
});
|
||||
|
||||
const JUDGE_SYSTEM = [
|
||||
"You are a strict evaluator of a Trigger.dev dashboard assistant.",
|
||||
"You are given the user's question, the data the assistant retrieved through its tools (treat this as the only ground truth), and the assistant's answer.",
|
||||
"Reason briefly first, then score each criterion from 1 to 5.",
|
||||
"Judge only on factual grounding and whether the question is answered. Do NOT reward verbosity, confidence, or style. Penalize any value (run id, error name, count, status, version, metric) that does not appear in the tool data.",
|
||||
].join(" ");
|
||||
|
||||
async function judge(args: {
|
||||
question: string;
|
||||
toolData: unknown;
|
||||
answer: string;
|
||||
}): Promise<z.infer<typeof Verdict>> {
|
||||
const { object } = await generateObject({
|
||||
model: anthropic(JUDGE_MODEL),
|
||||
schema: Verdict,
|
||||
system: JUDGE_SYSTEM,
|
||||
prompt: [
|
||||
`User question:\n${args.question}`,
|
||||
`Tool data (ground truth):\n${JSON.stringify(args.toolData, null, 2)}`,
|
||||
`Assistant answer:\n${args.answer}`,
|
||||
"Score the answer.",
|
||||
].join("\n\n"),
|
||||
});
|
||||
return object;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tool-selection cases
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const TOOL_CASES: Array<{ question: string; expect: string }> = [
|
||||
{ question: "What errors are happening in this environment?", expect: "list_errors" },
|
||||
{ question: "What's broken right now?", expect: "list_errors" },
|
||||
{ question: "Are there any unresolved errors?", expect: "list_errors" },
|
||||
{ question: "Give me the full detail for error_stripe.", expect: "get_error" },
|
||||
{ question: "Show me the runs behind the error error_stripe.", expect: "list_runs" },
|
||||
{ question: "Show me the failed runs in this environment.", expect: "list_runs" },
|
||||
{ question: "List the most recent runs of the send-receipt task.", expect: "list_runs" },
|
||||
{ question: "What's the status of run run_a1?", expect: "get_run" },
|
||||
{ question: "Why did run run_a1 fail? Walk me through what happened.", expect: "get_run_trace" },
|
||||
{ question: "What tasks are deployed in this environment?", expect: "list_tasks" },
|
||||
{ question: "Which projects can I access?", expect: "list_projects" },
|
||||
{ question: "What environments does this project have?", expect: "list_environments" },
|
||||
];
|
||||
|
||||
const TOOL_SELECTION_THRESHOLD = 0.83; // tolerate ~2/12 misses; a trend reds the suite
|
||||
|
||||
describe.skipIf(!HAS_KEY)("dashboardAgent evals (real model)", () => {
|
||||
it("tool selection: picks the right tool for the question", async () => {
|
||||
const results: Array<{ question: string; expected: string; got: string; ok: boolean }> = [];
|
||||
for (const c of TOOL_CASES) {
|
||||
const { calls } = await runCase(c.question);
|
||||
const got = calls[0]?.tool ?? "(none)";
|
||||
results.push({ question: c.question, expected: c.expect, got, ok: got === c.expect });
|
||||
}
|
||||
|
||||
const passed = results.filter((r) => r.ok).length;
|
||||
const rate = passed / results.length;
|
||||
// Surface the full table so a failing case is diagnosable, not just a number.
|
||||
// process.stdout.write (not console.log) so it survives vitest's console intercept.
|
||||
process.stdout.write(
|
||||
`\ntool selection: ${passed}/${results.length} (${(rate * 100).toFixed(0)}%)\n` +
|
||||
results
|
||||
.map(
|
||||
(r) =>
|
||||
` ${r.ok ? "PASS" : "FAIL"} ${r.got.padEnd(18)} (want ${r.expected}) ${r.question}`
|
||||
)
|
||||
.join("\n") +
|
||||
"\n"
|
||||
);
|
||||
|
||||
expect(rate).toBeGreaterThanOrEqual(TOOL_SELECTION_THRESHOLD);
|
||||
}, 180_000);
|
||||
|
||||
it("answer quality: grounded and on-question (LLM judge)", async () => {
|
||||
const question = "What errors are happening in this environment? Summarize the top ones.";
|
||||
const { calls, answer } = await runCase(question);
|
||||
|
||||
expect(calls[0]?.tool).toBe("list_errors");
|
||||
expect(answer.length).toBeGreaterThan(0);
|
||||
|
||||
const verdict = await judge({ question, toolData: FIXTURES.list_errors, answer });
|
||||
process.stdout.write(`\nanswer:\n${answer}\n\njudge: ${JSON.stringify(verdict)}\n`);
|
||||
|
||||
expect(verdict.grounded).toBeGreaterThanOrEqual(4);
|
||||
expect(verdict.answersQuestion).toBeGreaterThanOrEqual(4);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,315 @@
|
||||
// `@trigger.dev/sdk/ai/test` MUST be imported before the agent module so the
|
||||
// resource catalog is installed before `chat.agent({ id })` / `prompts.define`
|
||||
// register at module load.
|
||||
import { mockChatAgent, type MockChatAgentHarness } from "@trigger.dev/sdk/ai/test";
|
||||
|
||||
import type {
|
||||
LanguageModelV3FinishReason,
|
||||
LanguageModelV3StreamPart,
|
||||
LanguageModelV3Usage,
|
||||
} from "@ai-sdk/provider";
|
||||
import { simulateReadableStream, type UIMessage, type UIMessageChunk } from "ai";
|
||||
import { MockLanguageModelV3 } from "ai/test";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
dashboardAgent,
|
||||
dashboardAgentModelKey,
|
||||
dashboardAgentStoreKey,
|
||||
type DashboardAgentStore,
|
||||
} from "./dashboard-agent";
|
||||
import { buildDashboardAgentTools } from "./tools";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mock model helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const USAGE: LanguageModelV3Usage = {
|
||||
inputTokens: { total: 10, noCache: 10, cacheRead: undefined, cacheWrite: undefined },
|
||||
outputTokens: { total: 5, text: 5, reasoning: undefined },
|
||||
};
|
||||
|
||||
function finish(unified: LanguageModelV3FinishReason["unified"]): LanguageModelV3StreamPart {
|
||||
return { type: "finish", finishReason: { unified, raw: unified }, usage: USAGE };
|
||||
}
|
||||
|
||||
function textStep(text: string, id = "t1"): LanguageModelV3StreamPart[] {
|
||||
return [
|
||||
{ type: "text-start", id },
|
||||
{ type: "text-delta", id, delta: text },
|
||||
{ type: "text-end", id },
|
||||
finish("stop"),
|
||||
];
|
||||
}
|
||||
|
||||
function toolCallStep(
|
||||
toolName: string,
|
||||
input: Record<string, unknown> = {},
|
||||
toolCallId = "tc1"
|
||||
): LanguageModelV3StreamPart[] {
|
||||
return [
|
||||
{ type: "tool-call", toolCallId, toolName, input: JSON.stringify(input) },
|
||||
finish("tool-calls"),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* A MockLanguageModelV3 that plays one stream per `streamText` step (call), plus
|
||||
* a `doGenerate` for the background title generation (`generateText`). Each
|
||||
* `doStream` call returns a fresh stream for the next entry in `steps` (the last
|
||||
* entry repeats if the model is called more times than there are steps).
|
||||
*/
|
||||
function mockModel(steps: LanguageModelV3StreamPart[][], titleText = "Test Chat Title") {
|
||||
let call = 0;
|
||||
return new MockLanguageModelV3({
|
||||
doStream: async () => {
|
||||
const chunks = steps[Math.min(call, steps.length - 1)] ?? [];
|
||||
call++;
|
||||
return { stream: simulateReadableStream({ chunks }) };
|
||||
},
|
||||
doGenerate: async () => ({
|
||||
content: [{ type: "text", text: titleText }],
|
||||
finishReason: { unified: "stop", raw: "stop" },
|
||||
usage: USAGE,
|
||||
warnings: [],
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fake store — records the persistence the agent performs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type StoreCalls = {
|
||||
ensureChat: unknown[];
|
||||
persistMessages: unknown[];
|
||||
persistTurn: unknown[];
|
||||
setChatTitleIfDefault: unknown[];
|
||||
};
|
||||
|
||||
function fakeStore(): { store: DashboardAgentStore; calls: StoreCalls } {
|
||||
const calls: StoreCalls = {
|
||||
ensureChat: [],
|
||||
persistMessages: [],
|
||||
persistTurn: [],
|
||||
setChatTitleIfDefault: [],
|
||||
};
|
||||
const store: DashboardAgentStore = {
|
||||
ensureChat: async (args) => void calls.ensureChat.push(args),
|
||||
persistMessages: async (args) => void calls.persistMessages.push(args),
|
||||
persistTurn: async (args) => void calls.persistTurn.push(args),
|
||||
setChatTitleIfDefault: async (args) => void calls.setChatTitleIfDefault.push(args),
|
||||
};
|
||||
return { store, calls };
|
||||
}
|
||||
|
||||
const CLIENT_DATA = { userId: "user_1", organizationId: "org_1" };
|
||||
|
||||
function userMessage(text: string, id = "u1"): UIMessage {
|
||||
return { id, role: "user", parts: [{ type: "text", text }] };
|
||||
}
|
||||
|
||||
function collectText(chunks: UIMessageChunk[]): string {
|
||||
return chunks
|
||||
.filter((c): c is Extract<UIMessageChunk, { type: "text-delta" }> => c.type === "text-delta")
|
||||
.map((c) => c.delta)
|
||||
.join("");
|
||||
}
|
||||
|
||||
// A tool executed when the agent emits a `tool-output-available` chunk (carries
|
||||
// the result, keyed by toolCallId). On a head-start handover the tool-call is
|
||||
// supplied by the handover partial rather than streamed by the model, so the
|
||||
// output chunk is the only reliable signal that the call actually ran.
|
||||
function executedTool(chunks: UIMessageChunk[]): boolean {
|
||||
return chunks.some((c) => (c as { type?: string }).type === "tool-output-available");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Harness tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("dashboardAgent (mock harness)", () => {
|
||||
let harness: MockChatAgentHarness | undefined;
|
||||
|
||||
afterEach(async () => {
|
||||
await harness?.close();
|
||||
harness = undefined;
|
||||
});
|
||||
|
||||
it("streams the model's response and persists the turn", async () => {
|
||||
const { store, calls } = fakeStore();
|
||||
harness = mockChatAgent(dashboardAgent, {
|
||||
chatId: "chat_text",
|
||||
clientData: CLIENT_DATA,
|
||||
setupLocals: ({ set }) => {
|
||||
set(dashboardAgentStoreKey, store);
|
||||
set(dashboardAgentModelKey, mockModel([textStep("hello from the agent")]));
|
||||
},
|
||||
});
|
||||
|
||||
const turn = await harness.sendMessage(userMessage("hi"));
|
||||
|
||||
expect(collectText(turn.chunks)).toBe("hello from the agent");
|
||||
|
||||
// Persistence ran through the injected store, not a real database.
|
||||
expect(calls.ensureChat).toHaveLength(1);
|
||||
expect(calls.persistMessages).toHaveLength(1);
|
||||
// onTurnComplete persists after the turn-complete chunk; give it a tick.
|
||||
await new Promise((r) => setTimeout(r, 30));
|
||||
expect(calls.persistTurn).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("executes a read tool the model calls, then answers from the result", async () => {
|
||||
const { store } = fakeStore();
|
||||
harness = mockChatAgent(dashboardAgent, {
|
||||
chatId: "chat_tool",
|
||||
clientData: CLIENT_DATA,
|
||||
setupLocals: ({ set }) => {
|
||||
set(dashboardAgentStoreKey, store);
|
||||
// Step 1: the model calls list_errors. Step 2: it answers.
|
||||
set(
|
||||
dashboardAgentModelKey,
|
||||
mockModel([toolCallStep("list_errors"), textStep("you have no errors")])
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
const turn = await harness.sendMessage(userMessage("any errors?"));
|
||||
|
||||
// The tool executed inside the agent (no delegated token in clientData, so it
|
||||
// returns its graceful no-auth result — no network), and the model answered.
|
||||
expect(executedTool(turn.chunks)).toBe(true);
|
||||
expect(collectText(turn.chunks)).toBe("you have no errors");
|
||||
});
|
||||
|
||||
it("rolls an Anthropic cache breakpoint onto the last message", async () => {
|
||||
const { store } = fakeStore();
|
||||
const model = mockModel([textStep("cached")]);
|
||||
harness = mockChatAgent(dashboardAgent, {
|
||||
chatId: "chat_cache",
|
||||
clientData: CLIENT_DATA,
|
||||
setupLocals: ({ set }) => {
|
||||
set(dashboardAgentStoreKey, store);
|
||||
set(dashboardAgentModelKey, model);
|
||||
},
|
||||
});
|
||||
|
||||
await harness.sendMessage(userMessage("hi"));
|
||||
|
||||
// The prepareMessages hook should have placed a cacheControl breakpoint on
|
||||
// the last message of the prompt the model received.
|
||||
const prompt = model.doStreamCalls[0]?.prompt ?? [];
|
||||
const last = prompt[prompt.length - 1] as { providerOptions?: Record<string, unknown> };
|
||||
expect(last?.providerOptions?.anthropic).toMatchObject({
|
||||
cacheControl: { type: "ephemeral" },
|
||||
});
|
||||
});
|
||||
|
||||
it("Head Start handover: executes the handed-over tool call despite the cache hook (regression)", async () => {
|
||||
const { store } = fakeStore();
|
||||
// Only step 2 runs in the agent — the warm route already did step 1 and hands
|
||||
// over the pending tool call.
|
||||
const model = mockModel([textStep("resolved from the tool")]);
|
||||
harness = mockChatAgent(dashboardAgent, {
|
||||
chatId: "chat_headstart",
|
||||
clientData: CLIENT_DATA,
|
||||
mode: "handover-prepare",
|
||||
headStartMessages: [userMessage("what errors are happening?")],
|
||||
setupLocals: ({ set }) => {
|
||||
set(dashboardAgentStoreKey, store);
|
||||
set(dashboardAgentModelKey, model);
|
||||
},
|
||||
});
|
||||
|
||||
// The reshaped partial the SDK's chat.headStart sends on a tool-calls finish:
|
||||
// a tool-approval round whose trailing tool message must survive prepareMessages
|
||||
// for collectToolApprovals to execute the pending call.
|
||||
const toolCallId = "tc_hs";
|
||||
const approvalId = "ap_hs";
|
||||
const turn = await harness.sendHandover({
|
||||
partialAssistantMessage: [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "tool-call", toolCallId, toolName: "list_errors", input: {} },
|
||||
{ type: "tool-approval-request", approvalId, toolCallId },
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "tool",
|
||||
content: [{ type: "tool-approval-response", approvalId, approved: true }],
|
||||
},
|
||||
],
|
||||
isFinal: false,
|
||||
});
|
||||
|
||||
// With the SDK guard (preserveToolApprovalTail) the handed-over tool executes
|
||||
// and the model answers from its result. Without it, the bare tool_use would
|
||||
// never execute (no tool output) — this is the regression guard.
|
||||
expect(executedTool(turn.chunks)).toBe(true);
|
||||
expect(collectText(turn.chunks)).toBe("resolved from the tool");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tool unit tests (no harness) — the data lane fails closed without a token
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("buildDashboardAgentTools", () => {
|
||||
it("exposes the read tools plus render_view, and the data tools fail closed with no token", async () => {
|
||||
const tools = buildDashboardAgentTools({});
|
||||
expect(Object.keys(tools).sort()).toEqual(
|
||||
[
|
||||
"ask_support",
|
||||
"get_error",
|
||||
"get_query_schema",
|
||||
"get_run",
|
||||
"get_run_trace",
|
||||
"list_environments",
|
||||
"list_errors",
|
||||
"list_projects",
|
||||
"list_runs",
|
||||
"list_tasks",
|
||||
"run_query",
|
||||
"render_view",
|
||||
].sort()
|
||||
);
|
||||
|
||||
// No userActorToken / apiOrigin => every data tool returns a graceful
|
||||
// error, never throws and never hits the network. render_view is a
|
||||
// presentation tool and ask_support is gated on its own env config
|
||||
// (not the token), so both are exempt.
|
||||
for (const name of Object.keys(tools)) {
|
||||
if (name === "render_view" || name === "ask_support") continue;
|
||||
const tool = tools[name] as { execute?: (input: unknown, opts: unknown) => Promise<unknown> };
|
||||
const result = (await tool.execute?.({}, {})) as { error?: string };
|
||||
expect(result).toHaveProperty("error");
|
||||
expect(typeof result.error).toBe("string");
|
||||
}
|
||||
});
|
||||
|
||||
it("render_view echoes a validated view spec back as its output", async () => {
|
||||
const tools = buildDashboardAgentTools({});
|
||||
const renderView = tools.render_view as {
|
||||
execute: (input: unknown, opts: unknown) => Promise<unknown>;
|
||||
};
|
||||
const spec = {
|
||||
blocks: [
|
||||
{
|
||||
type: "diagnosis",
|
||||
runId: "run_abc123",
|
||||
summary: "The task threw because the order had no line items.",
|
||||
category: "user_code_error",
|
||||
likelyCause: "processOrder throws when items is empty.",
|
||||
confidence: "high",
|
||||
evidence: [
|
||||
{ type: "error", detail: "Error: order has no items", reference: "run_abc123" },
|
||||
],
|
||||
nextSteps: ["Validate the payload before triggering."],
|
||||
},
|
||||
],
|
||||
};
|
||||
const output = await renderView.execute(spec, {});
|
||||
expect(output).toEqual(spec);
|
||||
});
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user