wehub/triggerdotdev--trigger.dev
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:32:57 +08:00
commit cd420f9332
4811 changed files with 884702 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal-packages/clickhouse/schema/001_create_databases.sql
+6
View File
@@ -0,0 +1,6 @@
-- +goose up
CREATE DATABASE trigger_dev;
-- +goose down
DROP DATABASE trigger_dev;
internal-packages/clickhouse/schema/002_create_smoke_test.sql
+11
View File
@@ -0,0 +1,11 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS trigger_dev.smoke_test (
id UUID DEFAULT generateUUIDv4(),
timestamp DateTime64(3) DEFAULT now64(3),
message String,
number UInt32
) ENGINE = MergeTree()
ORDER BY (timestamp, id);
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.smoke_test;
internal-packages/clickhouse/schema/003_create_task_runs_v1.sql
+100
View File
@@ -0,0 +1,100 @@
-- +goose Up
CREATE TABLE trigger_dev.task_runs_v1
(
/* ─── ids & hierarchy ─────────────────────────────────────── */
environment_id String,
organization_id String,
project_id String,
run_id String,
environment_type LowCardinality(String),
friendly_id String,
attempt UInt8 DEFAULT 1,
/* ─── enums / status ──────────────────────────────────────── */
engine LowCardinality(String),
status LowCardinality(String),
/* ─── queue / concurrency / schedule ─────────────────────── */
task_identifier String,
queue String,
schedule_id String,
batch_id String,
/* ─── related runs ─────────────────────────────────────────────── */
root_run_id String,
parent_run_id String,
depth UInt8 DEFAULT 0,
/* ─── telemetry ─────────────────────────────────────────────── */
span_id String,
trace_id String,
idempotency_key String,
/* ─── timing ─────────────────────────────────────────────── */
created_at DateTime64(3),
updated_at DateTime64(3),
started_at Nullable(DateTime64(3)),
executed_at Nullable(DateTime64(3)),
completed_at Nullable(DateTime64(3)),
delay_until Nullable(DateTime64(3)),
queued_at Nullable(DateTime64(3)),
expired_at Nullable(DateTime64(3)),
expiration_ttl String,
/* ─── cost / usage ───────────────────────────────────────── */
usage_duration_ms UInt32 DEFAULT 0,
cost_in_cents Float64 DEFAULT 0,
base_cost_in_cents Float64 DEFAULT 0,
/* ─── payload & context ──────────────────────────────────── */
output JSON(max_dynamic_paths = 1024),
error JSON(max_dynamic_paths = 64),
/* ─── tagging / versions ─────────────────────────────────── */
tags Array(String) CODEC(ZSTD(1)),
task_version String CODEC(LZ4),
sdk_version String CODEC(LZ4),
cli_version String CODEC(LZ4),
machine_preset LowCardinality(String) CODEC(LZ4),
is_test UInt8 DEFAULT 0,
/* ─── commit lsn ─────────────────────────────────────────────── */
_version UInt64,
_is_deleted UInt8 DEFAULT 0
)
ENGINE = ReplacingMergeTree(_version, _is_deleted)
PARTITION BY toYYYYMM(created_at)
ORDER BY (toDate(created_at), environment_id, task_identifier, created_at, run_id)
SETTINGS enable_json_type = 1;
/* Fast tag filtering */
ALTER TABLE trigger_dev.task_runs_v1
ADD INDEX idx_tags tags TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4;
CREATE TABLE trigger_dev.raw_task_runs_payload_v1
(
run_id String,
created_at DateTime64(3),
payload JSON(max_dynamic_paths = 1024)
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (run_id)
SETTINGS enable_json_type = 1;
CREATE VIEW trigger_dev.tmp_eric_task_runs_full_v1 AS
SELECT
s.*,
p.payload as payload
FROM trigger_dev.task_runs_v1 AS s FINAL
LEFT JOIN trigger_dev.raw_task_runs_payload_v1 AS p ON s.run_id = p.run_id
SETTINGS enable_json_type = 1;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.task_runs_v1;
DROP TABLE IF EXISTS trigger_dev.raw_task_runs_payload_v1;
DROP VIEW IF EXISTS trigger_dev.tmp_eric_task_runs_full_v1;
internal-packages/clickhouse/schema/004_create_task_runs_v2.sql
+94
View File
@@ -0,0 +1,94 @@
-- +goose Up
/*
This is the second version of the task runs table.
The main change is we've added organization_id and project_id to the sort key, and removed the toDate(created_at) and task_identifier columns from the sort key.
We will add a skip index for the task_identifier column in a future migration.
*/
CREATE TABLE trigger_dev.task_runs_v2
(
/* ─── ids & hierarchy ─────────────────────────────────────── */
environment_id String,
organization_id String,
project_id String,
run_id String,
environment_type LowCardinality(String),
friendly_id String,
attempt UInt8 DEFAULT 1,
/* ─── enums / status ──────────────────────────────────────── */
engine LowCardinality(String),
status LowCardinality(String),
/* ─── queue / concurrency / schedule ─────────────────────── */
task_identifier String,
queue String,
schedule_id String,
batch_id String,
/* ─── related runs ─────────────────────────────────────────────── */
root_run_id String,
parent_run_id String,
depth UInt8 DEFAULT 0,
/* ─── telemetry ─────────────────────────────────────────────── */
span_id String,
trace_id String,
idempotency_key String,
/* ─── timing ─────────────────────────────────────────────── */
created_at DateTime64(3),
updated_at DateTime64(3),
started_at Nullable(DateTime64(3)),
executed_at Nullable(DateTime64(3)),
completed_at Nullable(DateTime64(3)),
delay_until Nullable(DateTime64(3)),
queued_at Nullable(DateTime64(3)),
expired_at Nullable(DateTime64(3)),
expiration_ttl String,
/* ─── cost / usage ───────────────────────────────────────── */
usage_duration_ms UInt32 DEFAULT 0,
cost_in_cents Float64 DEFAULT 0,
base_cost_in_cents Float64 DEFAULT 0,
/* ─── payload & context ──────────────────────────────────── */
output JSON(max_dynamic_paths = 1024),
error JSON(max_dynamic_paths = 64),
/* ─── tagging / versions ─────────────────────────────────── */
tags Array(String) CODEC(ZSTD(1)),
task_version String CODEC(LZ4),
sdk_version String CODEC(LZ4),
cli_version String CODEC(LZ4),
machine_preset LowCardinality(String) CODEC(LZ4),
is_test UInt8 DEFAULT 0,
/* ─── commit lsn ─────────────────────────────────────────────── */
_version UInt64,
_is_deleted UInt8 DEFAULT 0
)
ENGINE = ReplacingMergeTree(_version, _is_deleted)
PARTITION BY toYYYYMM(created_at)
ORDER BY (organization_id, project_id, environment_id, created_at, run_id)
SETTINGS enable_json_type = 1;
/* Fast tag filtering */
ALTER TABLE trigger_dev.task_runs_v2
ADD INDEX idx_tags tags TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4;
CREATE VIEW trigger_dev.tmp_eric_task_runs_full_v2 AS
SELECT
s.*,
p.payload as payload
FROM trigger_dev.task_runs_v2 AS s FINAL
LEFT JOIN trigger_dev.raw_task_runs_payload_v1 AS p ON s.run_id = p.run_id
SETTINGS enable_json_type = 1;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.task_runs_v2;
DROP VIEW IF EXISTS trigger_dev.tmp_eric_task_runs_full_v2
internal-packages/clickhouse/schema/005_add_task_runs_v2_concurrency_bulkactions.sql
+12
View File
@@ -0,0 +1,12 @@
-- +goose Up
/*
Add concurrency_key and bulk_action_group_ids columns with defaults.
*/
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN concurrency_key String DEFAULT '',
ADD COLUMN bulk_action_group_ids Array(String) DEFAULT [];
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN concurrency_key,
DROP COLUMN bulk_action_group_ids;
internal-packages/clickhouse/schema/006_add_task_runs_v2_workerqueue.sql
+10
View File
@@ -0,0 +1,10 @@
-- +goose Up
/*
Add worker_queue column.
*/
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN worker_queue String DEFAULT '';
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN worker_queue;
internal-packages/clickhouse/schema/007_add_task_events_v1.sql
+52
View File
@@ -0,0 +1,52 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS trigger_dev.task_events_v1
(
-- This the main "tenant" ID
environment_id String,
-- The organization ID here so we can do MV rollups of usage
organization_id String,
-- The project ID here so we can do MV rollups of usage
project_id String,
-- The task slug (e.g. "my-task")
task_identifier String CODEC(ZSTD(1)),
-- The non-friendly ID for the run
run_id String CODEC(ZSTD(1)),
-- nanoseconds since the epoch
start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
trace_id String CODEC(ZSTD(1)),
span_id String CODEC(ZSTD(1)),
-- will be an empty string for root spans
parent_span_id String CODEC(ZSTD(1)),
-- Log body, event name, or span name
message String CODEC(ZSTD(1)),
-- this is the new level column, can be
-- SPAN, SPAN_EVENT, DEBUG_EVENT, LOG_DEBUG, LOG_LOG, LOG_SUCCESS, LOG_INFO, LOG_WARN, LOG_ERROR, ANCESTOR_OVERRIDE
kind LowCardinality(String) CODEC(ZSTD(1)),
-- isError, isPartial, isCancelled will now be in this status column
-- OK, ERROR, PARTIAL, CANCELLED
status LowCardinality(String) CODEC(ZSTD(1)),
-- span/log/event attributes and resource attributes
-- includes error attributes, gen_ai attributes, and other attributes
attributes JSON CODEC(ZSTD(1)),
attributes_text String MATERIALIZED toJSONString(attributes),
-- This is the metadata column, includes style for styling the event in the UI
-- is a JSON stringified object, e.g. {"style":{"icon":"play","variant":"primary"},"error":{"message":"Error message","attributes":{"error.type":"ErrorType","error.code":"123"}}}
metadata String CODEC(ZSTD(1)),
-- nanoseconds since the start time, only non-zero for spans
duration UInt64 CODEC(ZSTD(1)),
-- The TTL for the event, will be deleted 7 days after the event expires
expires_at DateTime64(3),
INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_duration duration TYPE minmax GRANULARITY 1,
INDEX idx_attributes_text attributes_text TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
)
ENGINE = MergeTree
PARTITION BY toDate(start_time)
ORDER BY (environment_id, toUnixTimestamp(start_time), trace_id)
TTL toDateTime(expires_at) + INTERVAL 7 DAY
SETTINGS ttl_only_drop_parts = 1;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.task_events_v1;
internal-packages/clickhouse/schema/008_add_task_events_v1_mvs.sql
+55
View File
@@ -0,0 +1,55 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS trigger_dev.task_event_usage_by_minute_v1
(
organization_id String,
project_id String,
environment_id String,
bucket_start DateTime,
event_count UInt64
)
ENGINE = SummingMergeTree()
PARTITION BY toYYYYMM(bucket_start)
ORDER BY (organization_id, project_id, environment_id, bucket_start)
TTL bucket_start + INTERVAL 8 DAY;
CREATE TABLE IF NOT EXISTS trigger_dev.task_event_usage_by_hour_v1
(
organization_id String,
project_id String,
environment_id String,
bucket_start DateTime,
event_count UInt64
)
ENGINE = SummingMergeTree()
PARTITION BY toYYYYMM(bucket_start)
ORDER BY (organization_id, project_id, environment_id, bucket_start)
TTL bucket_start + INTERVAL 400 DAY;
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_usage_by_minute_v1
TO trigger_dev.task_event_usage_by_minute_v1 AS
SELECT
organization_id,
project_id,
environment_id,
toStartOfMinute(start_time) AS bucket_start,
count() AS event_count
FROM trigger_dev.task_events_v1
GROUP BY organization_id, project_id, environment_id, bucket_start;
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_usage_by_hour_v1
TO trigger_dev.task_event_usage_by_hour_v1 AS
SELECT
organization_id,
project_id,
environment_id,
toStartOfHour(bucket_start) AS bucket_start,
sum(event_count) AS event_count
FROM trigger_dev.task_event_usage_by_minute_v1
GROUP BY organization_id, project_id, environment_id, bucket_start;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.mv_task_event_usage_by_minute_v1;
DROP TABLE IF EXISTS trigger_dev.mv_task_event_usage_by_hour_v1;
DROP TABLE IF EXISTS trigger_dev.task_event_usage_by_hour_v1;
DROP TABLE IF EXISTS trigger_dev.task_event_usage_by_minute_v1;
internal-packages/clickhouse/schema/009_add_task_events_v2_mvs.sql
+29
View File
@@ -0,0 +1,29 @@
-- +goose Up
DROP TABLE IF EXISTS trigger_dev.mv_task_event_usage_by_minute_v1;
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_usage_by_minute_v2
TO trigger_dev.task_event_usage_by_minute_v1 AS
SELECT
organization_id,
project_id,
environment_id,
toStartOfMinute(start_time) AS bucket_start,
count() AS event_count
FROM trigger_dev.task_events_v1
WHERE kind != 'DEBUG_EVENT' AND kind != 'ANCESTOR_OVERRIDE' AND status != 'PARTIAL'
GROUP BY organization_id, project_id, environment_id, bucket_start;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.mv_task_event_usage_by_minute_v2;
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_usage_by_minute_v1
TO trigger_dev.task_event_usage_by_minute_v1 AS
SELECT
organization_id,
project_id,
environment_id,
toStartOfMinute(start_time) AS bucket_start,
count() AS event_count
FROM trigger_dev.task_events_v1
GROUP BY organization_id, project_id, environment_id, bucket_start;
internal-packages/clickhouse/schema/010_add_task_events_v2.sql
+55
View File
@@ -0,0 +1,55 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS trigger_dev.task_events_v2
(
-- This the main "tenant" ID
environment_id String,
-- The organization ID here so we can do MV rollups of usage
organization_id String,
-- The project ID here so we can do MV rollups of usage
project_id String,
-- The task slug (e.g. "my-task")
task_identifier String CODEC(ZSTD(1)),
-- The non-friendly ID for the run
run_id String CODEC(ZSTD(1)),
-- nanoseconds since the epoch
start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
trace_id String CODEC(ZSTD(1)),
span_id String CODEC(ZSTD(1)),
-- will be an empty string for root spans
parent_span_id String CODEC(ZSTD(1)),
-- Log body, event name, or span name
message String CODEC(ZSTD(1)),
-- this is the new level column, can be
-- SPAN, SPAN_EVENT, DEBUG_EVENT, LOG_DEBUG, LOG_LOG, LOG_SUCCESS, LOG_INFO, LOG_WARN, LOG_ERROR, ANCESTOR_OVERRIDE
kind LowCardinality(String) CODEC(ZSTD(1)),
-- isError, isPartial, isCancelled will now be in this status column
-- OK, ERROR, PARTIAL, CANCELLED
status LowCardinality(String) CODEC(ZSTD(1)),
-- span/log/event attributes and resource attributes
-- includes error attributes, gen_ai attributes, and other attributes
attributes JSON CODEC(ZSTD(1)),
attributes_text String MATERIALIZED toJSONString(attributes),
-- This is the metadata column, includes style for styling the event in the UI
-- is a JSON stringified object, e.g. {"style":{"icon":"play","variant":"primary"},"error":{"message":"Error message","attributes":{"error.type":"ErrorType","error.code":"123"}}}
metadata String CODEC(ZSTD(1)),
-- nanoseconds since the start time, only non-zero for spans
duration UInt64 CODEC(ZSTD(1)),
-- The TTL for the event, will be deleted 7 days after the event expires
expires_at DateTime64(3),
-- NEW: Insert timestamp for partitioning (avoids "too many parts" errors from late-arriving events)
inserted_at DateTime64(3) DEFAULT now64(3),
INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_duration duration TYPE minmax GRANULARITY 1,
INDEX idx_attributes_text attributes_text TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
)
ENGINE = MergeTree
PARTITION BY toDate(inserted_at)
ORDER BY (environment_id, toUnixTimestamp(start_time), trace_id)
TTL toDateTime(expires_at) + INTERVAL 7 DAY
SETTINGS ttl_only_drop_parts = 1;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.task_events_v2;
internal-packages/clickhouse/schema/011_add_task_events_v2_table_mvs.sql
+19
View File
@@ -0,0 +1,19 @@
-- +goose Up
-- Create materialized views for task_events_v2 table (partitioned by inserted_at)
-- These write to the same target tables as the v1 MVs so usage is aggregated across both tables
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.mv_task_event_v2_usage_by_minute
TO trigger_dev.task_event_usage_by_minute_v1 AS
SELECT
organization_id,
project_id,
environment_id,
toStartOfMinute(start_time) AS bucket_start,
count() AS event_count
FROM trigger_dev.task_events_v2
WHERE kind != 'DEBUG_EVENT' AND kind != 'ANCESTOR_OVERRIDE' AND status != 'PARTIAL'
GROUP BY organization_id, project_id, environment_id, bucket_start;
-- +goose Down
DROP VIEW IF EXISTS trigger_dev.mv_task_event_v2_usage_by_minute;
internal-packages/clickhouse/schema/012_add_task_runs_v2_max_duration.sql
+10
View File
@@ -0,0 +1,10 @@
-- +goose Up
/*
Add max_duration_in_seconds column.
*/
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN max_duration_in_seconds Nullable (UInt32) DEFAULT NULL;
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN max_duration_in_seconds;
internal-packages/clickhouse/schema/013_add_task_runs_v2_idempotency_key_options.sql
+16
View File
@@ -0,0 +1,16 @@
-- +goose Up
-- Add columns for storing user-provided idempotency key and scope for searching
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN idempotency_key_user String DEFAULT '';
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN idempotency_key_scope String DEFAULT '';
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN idempotency_key_user;
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN idempotency_key_scope;
internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql
+45
View File
@@ -0,0 +1,45 @@
-- +goose Up
-- Update the materialized columns to extract the 'data' field if it exists
-- This avoids the {"data": ...} wrapper in the text representation
-- Note: Direct JSON path access (output.data) returns null for nested objects,
-- so we use JSONExtractRaw on the stringified JSON instead
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN output_text String MATERIALIZED if (
toJSONString (output) = '{}',
'',
if (
length (JSONExtractRaw (toJSONString (output), 'data')) > 0,
JSONExtractRaw (toJSONString (output), 'data'),
toJSONString (output)
)
);
-- For error: extract error.data if it exists
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN error_text String MATERIALIZED if (
toJSONString (error) = '{}',
'',
if (
length (JSONExtractRaw (toJSONString (error), 'data')) > 0,
JSONExtractRaw (toJSONString (error), 'data'),
toJSONString (error)
)
);
-- Add the indexes
ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_output_text output_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4;
ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_error_text error_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4;
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP INDEX IF EXISTS idx_output_text;
ALTER TABLE trigger_dev.task_runs_v2
DROP INDEX IF EXISTS idx_error_text;
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN IF EXISTS output_text;
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN IF EXISTS error_text;
internal-packages/clickhouse/schema/015_add_task_runs_v2_search_indexes.sql
+20
View File
@@ -0,0 +1,20 @@
-- +goose Up
-- Add indexes for text search on task task_events_v2 tables for message and attributes fields
ALTER TABLE trigger_dev.task_events_v2
ADD INDEX IF NOT EXISTS idx_attributes_text_search lower(attributes_text)
TYPE ngrambf_v1(3, 32768, 2, 0)
GRANULARITY 1;
ALTER TABLE trigger_dev.task_events_v2
ADD INDEX IF NOT EXISTS idx_message_text_search lower(message)
TYPE ngrambf_v1(3, 32768, 2, 0)
GRANULARITY 1;
-- +goose Down
ALTER TABLE trigger_dev.task_events_v2
DROP INDEX idx_attributes_text_search;
ALTER TABLE trigger_dev.task_events_v2
DROP INDEX idx_message_text_search;
internal-packages/clickhouse/schema/016_add_task_events_search_v1.sql
+63
View File
@@ -0,0 +1,63 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS trigger_dev.task_events_search_v1
(
environment_id String,
organization_id String,
project_id String,
triggered_timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
trace_id String CODEC(ZSTD(1)),
span_id String CODEC(ZSTD(1)),
run_id String CODEC(ZSTD(1)),
task_identifier String CODEC(ZSTD(1)),
start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
inserted_at DateTime64(3),
message String CODEC(ZSTD(1)),
kind LowCardinality(String) CODEC(ZSTD(1)),
status LowCardinality(String) CODEC(ZSTD(1)),
duration UInt64 CODEC(ZSTD(1)),
parent_span_id String CODEC(ZSTD(1)),
attributes_text String CODEC(ZSTD(1)),
INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_message_text_search lower(message) TYPE ngrambf_v1(3, 32768, 2, 0) GRANULARITY 1,
INDEX idx_attributes_text_search lower(attributes_text) TYPE ngrambf_v1(3, 32768, 2, 0) GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toDate(triggered_timestamp)
ORDER BY (organization_id, environment_id, triggered_timestamp, trace_id)
--Right now we have maximum retention of up to 30 days based on plan.
--We put a logical limit for now, the 90 DAY TTL is just a backup
--This might need to be updated for longer retention periods
TTL toDateTime(triggered_timestamp) + INTERVAL 90 DAY
SETTINGS ttl_only_drop_parts = 1;
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.task_events_search_mv_v1
TO trigger_dev.task_events_search_v1 AS
SELECT
environment_id,
organization_id,
project_id,
trace_id,
span_id,
run_id,
task_identifier,
start_time,
inserted_at,
message,
kind,
status,
duration,
parent_span_id,
attributes_text,
fromUnixTimestamp64Nano(toUnixTimestamp64Nano(start_time) + toInt64(duration)) AS triggered_timestamp
FROM trigger_dev.task_events_v2
WHERE
kind != 'DEBUG_EVENT'
AND status != 'PARTIAL'
AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}')
AND kind != 'ANCESTOR_OVERRIDE'
AND message != 'trigger.dev/start';
-- +goose Down
DROP VIEW IF EXISTS trigger_dev.task_events_search_mv_v1;
DROP TABLE IF EXISTS trigger_dev.task_events_search_v1;
internal-packages/clickhouse/schema/017_update_materialized_conditions_to_task_events_search_v1.sql
+62
View File
@@ -0,0 +1,62 @@
-- +goose Up
-- We drop the existing MV and recreate it with the new filter condition
DROP VIEW IF EXISTS trigger_dev.task_events_search_mv_v1;
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.task_events_search_mv_v1
TO trigger_dev.task_events_search_v1 AS
SELECT
environment_id,
organization_id,
project_id,
trace_id,
span_id,
run_id,
task_identifier,
start_time,
inserted_at,
message,
kind,
status,
duration,
parent_span_id,
attributes_text,
fromUnixTimestamp64Nano(toUnixTimestamp64Nano(start_time) + toInt64(duration)) AS triggered_timestamp
FROM trigger_dev.task_events_v2
WHERE
trace_id != '' -- New condition added here
AND kind != 'DEBUG_EVENT'
AND status != 'PARTIAL'
AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}')
AND kind != 'ANCESTOR_OVERRIDE'
AND message != 'trigger.dev/start';
-- +goose Down
-- In the down migration, we revert to the previous filter set
DROP VIEW IF EXISTS trigger_dev.task_events_search_mv_v1;
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.task_events_search_mv_v1
TO trigger_dev.task_events_search_v1 AS
SELECT
environment_id,
organization_id,
project_id,
trace_id,
span_id,
run_id,
task_identifier,
start_time,
inserted_at,
message,
kind,
status,
duration,
parent_span_id,
attributes_text,
fromUnixTimestamp64Nano(toUnixTimestamp64Nano(start_time) + toInt64(duration)) AS triggered_timestamp
FROM trigger_dev.task_events_v2
WHERE
kind != 'DEBUG_EVENT'
AND status != 'PARTIAL'
AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}')
AND kind != 'ANCESTOR_OVERRIDE'
AND message != 'trigger.dev/start';
internal-packages/clickhouse/schema/018_drop_unused_task_events_v2_indexes.sql
+22
View File
@@ -0,0 +1,22 @@
-- +goose Up
-- These indexes are not used in any WHERE clause.
-- idx_duration: duration is only written/read as a column, never filtered on.
-- idx_attributes_text: search queries use the task_events_search_v1 table instead.
ALTER TABLE trigger_dev.task_events_v2
DROP INDEX IF EXISTS idx_duration;
ALTER TABLE trigger_dev.task_events_v2
DROP INDEX IF EXISTS idx_attributes_text;
-- +goose Down
ALTER TABLE trigger_dev.task_events_v2
ADD INDEX IF NOT EXISTS idx_duration duration
TYPE minmax
GRANULARITY 1;
ALTER TABLE trigger_dev.task_events_v2
ADD INDEX IF NOT EXISTS idx_attributes_text attributes_text
TYPE tokenbf_v1(32768, 3, 0)
GRANULARITY 8;
internal-packages/clickhouse/schema/019_create_metrics_v1.sql
+44
View File
@@ -0,0 +1,44 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS trigger_dev.metrics_v1
(
organization_id LowCardinality(String),
project_id LowCardinality(String),
environment_id String CODEC(ZSTD(1)),
metric_name LowCardinality(String),
metric_type LowCardinality(String),
metric_subject String CODEC(ZSTD(1)),
bucket_start DateTime CODEC(Delta(4), ZSTD(1)),
value Float64 DEFAULT 0 CODEC(ZSTD(1)),
attributes JSON(
`trigger.run_id` String,
`trigger.task_slug` String,
`trigger.attempt_number` Int64,
`trigger.environment_type` LowCardinality(String),
`trigger.machine_id` String,
`trigger.machine_name` LowCardinality(String),
`trigger.worker_id` String,
`trigger.worker_version` String,
`system.cpu.logical_number` String,
`system.cpu.state` LowCardinality(String),
`system.memory.state` LowCardinality(String),
`system.device` String,
`system.filesystem.type` LowCardinality(String),
`system.filesystem.mountpoint` String,
`system.filesystem.mode` LowCardinality(String),
`system.filesystem.state` LowCardinality(String),
`disk.io.direction` LowCardinality(String),
`process.cpu.state` LowCardinality(String),
`network.io.direction` LowCardinality(String),
max_dynamic_paths=8
),
INDEX idx_run_id attributes.trigger.run_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_task_slug attributes.trigger.task_slug TYPE bloom_filter(0.001) GRANULARITY 1
)
ENGINE = MergeTree()
PARTITION BY toDate(bucket_start)
ORDER BY (organization_id, project_id, environment_id, metric_name, metric_subject, bucket_start)
TTL bucket_start + INTERVAL 60 DAY
SETTINGS ttl_only_drop_parts = 1;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.metrics_v1;
internal-packages/clickhouse/schema/020_add_machine_id_to_task_events_v2.sql
+7
View File
@@ -0,0 +1,7 @@
-- +goose Up
ALTER TABLE trigger_dev.task_events_v2
ADD COLUMN machine_id String DEFAULT '' CODEC(ZSTD(1));
-- +goose Down
ALTER TABLE trigger_dev.task_events_v2
DROP COLUMN machine_id;
internal-packages/clickhouse/schema/021_add_error_fingerprint_to_task_runs_v2.sql
+11
View File
@@ -0,0 +1,11 @@
-- +goose Up
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN error_fingerprint String DEFAULT '';
-- Bloom filter index for fast error fingerprint lookups
ALTER TABLE trigger_dev.task_runs_v2
ADD INDEX idx_error_fingerprint error_fingerprint TYPE bloom_filter GRANULARITY 4;
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2 DROP INDEX idx_error_fingerprint;
ALTER TABLE trigger_dev.task_runs_v2 DROP COLUMN error_fingerprint;
internal-packages/clickhouse/schema/022_create_errors_v1_table.sql
+78
View File
@@ -0,0 +1,78 @@
-- +goose Up
-- Aggregated error groups table (per task + fingerprint)
CREATE TABLE trigger_dev.errors_v1
(
organization_id String,
project_id String,
environment_id String,
task_identifier String,
error_fingerprint String,
-- Error details (samples from occurrences)
error_type String,
error_message String,
sample_stack_trace String,
-- SimpleAggregateFunction stores raw values and applies the function during merge,
-- avoiding binary state encoding issues with AggregateFunction.
last_seen_date SimpleAggregateFunction(max, DateTime),
first_seen SimpleAggregateFunction(min, DateTime64(3)),
last_seen SimpleAggregateFunction(max, DateTime64(3)),
occurrence_count AggregateFunction(sum, UInt64),
affected_task_versions AggregateFunction(uniq, String),
-- Samples for debugging
sample_run_id AggregateFunction(any, String),
sample_friendly_id AggregateFunction(any, String),
-- Status distribution
status_distribution AggregateFunction(sumMap, Array(String), Array(UInt64))
)
ENGINE = AggregatingMergeTree()
ORDER BY (organization_id, project_id, environment_id, task_identifier, error_fingerprint)
TTL last_seen_date + INTERVAL 90 DAY
SETTINGS index_granularity = 8192;
-- Materialized view to auto-populate from task_runs_v2
CREATE MATERIALIZED VIEW trigger_dev.errors_mv_v1
TO trigger_dev.errors_v1
AS
SELECT
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
any(coalesce(nullIf(toString(error.data.type), ''), nullIf(toString(error.data.name), ''), 'Error')) as error_type,
any(coalesce(nullIf(substring(toString(error.data.message), 1, 500), ''), 'Unknown error')) as error_message,
any(coalesce(substring(toString(error.data.stack), 1, 2000), '')) as sample_stack_trace,
toDateTime(max(created_at)) as last_seen_date,
min(created_at) as first_seen,
max(created_at) as last_seen,
sumState(toUInt64(1)) as occurrence_count,
uniqState(task_version) as affected_task_versions,
anyState(run_id) as sample_run_id,
anyState(friendly_id) as sample_friendly_id,
sumMapState([status], [toUInt64(1)]) as status_distribution
FROM trigger_dev.task_runs_v2
WHERE
error_fingerprint != ''
AND status IN ('SYSTEM_FAILURE', 'CRASHED', 'INTERRUPTED', 'COMPLETED_WITH_ERRORS', 'TIMED_OUT')
AND _is_deleted = 0
GROUP BY
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint;
-- +goose Down
DROP VIEW IF EXISTS trigger_dev.errors_mv_v1;
DROP TABLE IF EXISTS trigger_dev.errors_v1;
internal-packages/clickhouse/schema/023_create_error_occurrences_v1.sql
+89
View File
@@ -0,0 +1,89 @@
-- +goose Up
-- Per-minute error occurrence counts, keyed by fingerprint + task + version.
-- Powers precise time-range filtering and dynamic-granularity occurrence charts.
CREATE TABLE
trigger_dev.error_occurrences_v1 (
organization_id String,
project_id String,
environment_id String,
task_identifier String,
error_fingerprint String,
task_version String,
minute DateTime,
error_type String,
error_message String,
stack_trace String,
count UInt64,
INDEX idx_error_type_search lower(error_type) TYPE ngrambf_v1 (3, 32768, 2, 0) GRANULARITY 1,
INDEX idx_error_message_search lower(error_message) TYPE ngrambf_v1 (3, 32768, 2, 0) GRANULARITY 1
) ENGINE = SummingMergeTree (count)
PARTITION BY
toDate (minute)
ORDER BY
(
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
task_version,
minute
) TTL minute + INTERVAL 90 DAY SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW trigger_dev.error_occurrences_mv_v1 TO trigger_dev.error_occurrences_v1 AS
SELECT
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
task_version,
toStartOfMinute (created_at) as minute,
any (
coalesce(
nullIf(toString (error.data.type), ''),
nullIf(toString (error.data.name), ''),
'Error'
)
) as error_type,
any (
coalesce(
nullIf(
substring(toString (error.data.message), 1, 500),
''
),
'Unknown error'
)
) as error_message,
any (
coalesce(
substring(toString (error.data.stack), 1, 2000),
''
)
) as stack_trace,
count() as count
FROM
trigger_dev.task_runs_v2
WHERE
error_fingerprint != ''
AND status IN (
'SYSTEM_FAILURE',
'CRASHED',
'INTERRUPTED',
'COMPLETED_WITH_ERRORS',
'TIMED_OUT'
)
AND _is_deleted = 0
GROUP BY
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
task_version,
minute;
-- +goose Down
DROP VIEW IF EXISTS trigger_dev.error_occurrences_mv_v1;
DROP TABLE IF EXISTS trigger_dev.error_occurrences_v1;
internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql
+64
View File
@@ -0,0 +1,64 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS trigger_dev.llm_metrics_v1
(
-- Tenant context
organization_id LowCardinality(String),
project_id LowCardinality(String),
environment_id String CODEC(ZSTD(1)),
run_id String CODEC(ZSTD(1)),
task_identifier LowCardinality(String),
trace_id String CODEC(ZSTD(1)),
span_id String CODEC(ZSTD(1)),
-- Model & provider
gen_ai_system LowCardinality(String),
request_model String CODEC(ZSTD(1)),
response_model String CODEC(ZSTD(1)),
matched_model_id String CODEC(ZSTD(1)),
operation_id LowCardinality(String),
finish_reason LowCardinality(String),
cost_source LowCardinality(String),
-- Pricing
pricing_tier_id String CODEC(ZSTD(1)),
pricing_tier_name LowCardinality(String),
-- Token usage
input_tokens UInt64 DEFAULT 0,
output_tokens UInt64 DEFAULT 0,
total_tokens UInt64 DEFAULT 0,
usage_details Map(LowCardinality(String), UInt64),
-- Cost
input_cost Decimal64(12) DEFAULT 0,
output_cost Decimal64(12) DEFAULT 0,
total_cost Decimal64(12) DEFAULT 0,
cost_details Map(LowCardinality(String), Decimal64(12)),
provider_cost Decimal64(12) DEFAULT 0,
-- Performance
ms_to_first_chunk Float64 DEFAULT 0,
tokens_per_second Float64 DEFAULT 0,
-- Attribution
metadata Map(LowCardinality(String), String),
-- Timing
start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
duration UInt64 DEFAULT 0 CODEC(ZSTD(1)),
inserted_at DateTime64(3) DEFAULT now64(3),
-- Indexes
INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_metadata_keys mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toDate(inserted_at)
ORDER BY (organization_id, project_id, environment_id, toDate(inserted_at), run_id)
TTL toDateTime(inserted_at) + INTERVAL 365 DAY
SETTINGS ttl_only_drop_parts = 1;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.llm_metrics_v1;
internal-packages/clickhouse/schema/025_add_prompt_columns_to_llm_metrics_v1.sql
+13
View File
@@ -0,0 +1,13 @@
-- +goose Up
ALTER TABLE trigger_dev.llm_metrics_v1
ADD COLUMN prompt_slug LowCardinality(String) DEFAULT '';
ALTER TABLE trigger_dev.llm_metrics_v1
ADD COLUMN prompt_version UInt32 DEFAULT 0;
-- +goose Down
ALTER TABLE trigger_dev.llm_metrics_v1
DROP COLUMN prompt_slug;
ALTER TABLE trigger_dev.llm_metrics_v1
DROP COLUMN prompt_version;
internal-packages/clickhouse/schema/026_add_base_response_model_to_llm_metrics_v1.sql
+7
View File
@@ -0,0 +1,7 @@
-- +goose Up
ALTER TABLE trigger_dev.llm_metrics_v1
ADD COLUMN base_response_model String DEFAULT '' CODEC(ZSTD(1));
-- +goose Down
ALTER TABLE trigger_dev.llm_metrics_v1
DROP COLUMN base_response_model;
internal-packages/clickhouse/schema/027_create_llm_model_aggregates_v1.sql
+57
View File
@@ -0,0 +1,57 @@
-- +goose Up
-- Pre-aggregated model performance metrics with no tenant information.
-- Used for cross-tenant model comparisons in the Model Registry.
-- Aggregated per minute for high-resolution model performance tracking.
CREATE TABLE IF NOT EXISTS trigger_dev.llm_model_aggregates_v1
(
response_model String,
base_response_model String DEFAULT '',
gen_ai_system LowCardinality(String),
minute DateTime,
-- Counts & totals (SimpleAggregateFunction for sum)
call_count SimpleAggregateFunction(sum, UInt64),
total_input_tokens SimpleAggregateFunction(sum, UInt64),
total_output_tokens SimpleAggregateFunction(sum, UInt64),
total_cost SimpleAggregateFunction(sum, Float64),
-- Performance quantiles (AggregateFunction for merge across parts)
ttfc_quantiles AggregateFunction(quantiles(0.5, 0.9, 0.95, 0.99), Float64),
tps_quantiles AggregateFunction(quantiles(0.5, 0.9, 0.95, 0.99), Float64),
duration_quantiles AggregateFunction(quantiles(0.5, 0.9, 0.95, 0.99), UInt64),
-- Finish reason distribution
finish_reason_counts SimpleAggregateFunction(sumMap, Map(String, UInt64))
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(minute)
ORDER BY (response_model, base_response_model, gen_ai_system, minute)
TTL toDate(minute) + INTERVAL 365 DAY
SETTINGS ttl_only_drop_parts = 1;
-- Materialized view that feeds the aggregate table from llm_metrics_v1.
-- Strips all tenant-specific columns (org, project, env, run, span, trace).
-- base_response_model comes from the source table (populated during event enrichment).
CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.llm_model_aggregates_mv_v1
TO trigger_dev.llm_model_aggregates_v1
AS SELECT
response_model,
base_response_model,
gen_ai_system,
toStartOfMinute(start_time) AS minute,
count() AS call_count,
sum(input_tokens) AS total_input_tokens,
sum(output_tokens) AS total_output_tokens,
sum(total_cost) AS total_cost,
quantilesStateIf(0.5, 0.9, 0.95, 0.99)(ms_to_first_chunk, ms_to_first_chunk > 0) AS ttfc_quantiles,
quantilesStateIf(0.5, 0.9, 0.95, 0.99)(tokens_per_second, tokens_per_second > 0) AS tps_quantiles,
quantilesState(0.5, 0.9, 0.95, 0.99)(duration) AS duration_quantiles,
sumMap(map(finish_reason, toUInt64(1))) AS finish_reason_counts
FROM trigger_dev.llm_metrics_v1
WHERE response_model != ''
GROUP BY response_model, base_response_model, gen_ai_system, minute;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.llm_model_aggregates_mv_v1;
DROP TABLE IF EXISTS trigger_dev.llm_model_aggregates_v1;
internal-packages/clickhouse/schema/028_add_trigger_source_and_warm_start_to_task_runs_v2.sql
+19
View File
@@ -0,0 +1,19 @@
-- +goose Up
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN trigger_source LowCardinality(String) DEFAULT '';
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN root_trigger_source LowCardinality(String) DEFAULT '';
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN is_warm_start Nullable(UInt8) DEFAULT NULL;
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN trigger_source;
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN root_trigger_source;
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN is_warm_start;
internal-packages/clickhouse/schema/030_create_sessions_v1.sql
+42
View File
@@ -0,0 +1,42 @@
-- +goose Up
CREATE TABLE trigger_dev.sessions_v1
(
/* ─── identity ─────────────────────────────────────────────── */
environment_id String,
organization_id String,
project_id String,
session_id String,
environment_type LowCardinality(String),
friendly_id String,
external_id String DEFAULT '',
/* ─── type discriminator ──────────────────────────────────── */
type LowCardinality(String),
task_identifier String DEFAULT '',
/* ─── filtering / free-form ──────────────────────────────── */
tags Array(String) CODEC(ZSTD(1)),
metadata JSON(max_dynamic_paths = 256),
/* ─── terminal markers ────────────────────────────────────── */
closed_at Nullable(DateTime64(3)),
closed_reason String DEFAULT '',
expires_at Nullable(DateTime64(3)),
/* ─── timing ─────────────────────────────────────────────── */
created_at DateTime64(3),
updated_at DateTime64(3),
/* ─── commit lsn ────────────────────────────────────────── */
_version UInt64,
_is_deleted UInt8 DEFAULT 0
)
ENGINE = ReplacingMergeTree(_version, _is_deleted)
PARTITION BY toYYYYMM(created_at)
ORDER BY (organization_id, project_id, environment_id, created_at, session_id)
SETTINGS enable_json_type = 1;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.sessions_v1;
internal-packages/clickhouse/schema/031_add_task_kind_to_task_runs_v2.sql
+12
View File
@@ -0,0 +1,12 @@
-- +goose Up
-- IF NOT EXISTS is required because this migration was previously numbered
-- 029 and may have been applied in environments where goose accepted it
-- before 030_create_sessions_v1 advanced the version counter. Renaming to
-- 031 makes goose treat this as new everywhere, so the DDL must tolerate
-- the column already being present.
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN IF NOT EXISTS task_kind LowCardinality(String) DEFAULT '';
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN IF EXISTS task_kind;
internal-packages/clickhouse/schema/032_add_task_runs_v2_region.sql
+7
View File
@@ -0,0 +1,7 @@
-- +goose Up
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN IF NOT EXISTS region String DEFAULT '';
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN IF EXISTS region;
internal-packages/clickhouse/schema/033_add_task_runs_v2_plan_type.sql
+7
View File
@@ -0,0 +1,7 @@
-- +goose Up
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN IF NOT EXISTS plan_type LowCardinality(String) DEFAULT '';
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN IF EXISTS plan_type;
internal-packages/clickhouse/schema/034_add_is_test_to_sessions_v1.sql
+11
View File
@@ -0,0 +1,11 @@
-- +goose Up
-- Existing rows default to 0 and are intentionally NOT backfilled: the Sessions
-- list reads isTest from Postgres (ClickHouse only supplies session IDs), so the
-- UI is correct without it. A backfill is only needed if a ClickHouse-side
-- isTest filter/aggregate over sessions_v1 is added later.
ALTER TABLE trigger_dev.sessions_v1
ADD COLUMN IF NOT EXISTS is_test UInt8 DEFAULT 0;
-- +goose Down
ALTER TABLE trigger_dev.sessions_v1
DROP COLUMN IF EXISTS is_test;
internal-packages/clickhouse/schema/035_fix_error_display_derivation.sql
+192
View File
@@ -0,0 +1,192 @@
-- +goose Up
-- Fix how the error materialized views derive their display columns from the
-- stored error JSON:
-- * error_type: use the real class name (or internal code), not the generic
-- serialization tag (BUILT_IN_ERROR / STRING_ERROR / ...).
-- * error_message: fall back name -> raw before 'Unknown error' so messageless
-- errors (tagged errors) and non-Error throws still get a meaningful title.
-- * stack trace: read error.data.stackTrace (the stored field), not
-- error.data.stack, which was always empty.
-- Display-only. Only affects rows inserted after this migration.
ALTER TABLE trigger_dev.errors_mv_v1 MODIFY QUERY
SELECT
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
any(coalesce(nullIf(toString(error.data.name), ''), nullIf(toString(error.data.code), ''), 'Error')) as error_type,
any(coalesce(
nullIf(substring(toString(error.data.message), 1, 500), ''),
nullIf(toString(error.data.name), ''),
nullIf(substring(toString(error.data.raw), 1, 500), ''),
'Unknown error'
)) as error_message,
any(coalesce(substring(toString(error.data.stackTrace), 1, 2000), '')) as sample_stack_trace,
toDateTime(max(created_at)) as last_seen_date,
min(created_at) as first_seen,
max(created_at) as last_seen,
sumState(toUInt64(1)) as occurrence_count,
uniqState(task_version) as affected_task_versions,
anyState(run_id) as sample_run_id,
anyState(friendly_id) as sample_friendly_id,
sumMapState([status], [toUInt64(1)]) as status_distribution
FROM trigger_dev.task_runs_v2
WHERE
error_fingerprint != ''
AND status IN ('SYSTEM_FAILURE', 'CRASHED', 'INTERRUPTED', 'COMPLETED_WITH_ERRORS', 'TIMED_OUT')
AND _is_deleted = 0
GROUP BY
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint;
ALTER TABLE trigger_dev.error_occurrences_mv_v1 MODIFY QUERY
SELECT
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
task_version,
toStartOfMinute (created_at) as minute,
any (
coalesce(
nullIf(toString (error.data.name), ''),
nullIf(toString (error.data.code), ''),
'Error'
)
) as error_type,
any (
coalesce(
nullIf(substring(toString (error.data.message), 1, 500), ''),
nullIf(toString (error.data.name), ''),
nullIf(substring(toString (error.data.raw), 1, 500), ''),
'Unknown error'
)
) as error_message,
any (
coalesce(
substring(toString (error.data.stackTrace), 1, 2000),
''
)
) as stack_trace,
count() as count
FROM
trigger_dev.task_runs_v2
WHERE
error_fingerprint != ''
AND status IN (
'SYSTEM_FAILURE',
'CRASHED',
'INTERRUPTED',
'COMPLETED_WITH_ERRORS',
'TIMED_OUT'
)
AND _is_deleted = 0
GROUP BY
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
task_version,
minute;
-- +goose Down
ALTER TABLE trigger_dev.errors_mv_v1 MODIFY QUERY
SELECT
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
any(coalesce(nullIf(toString(error.data.type), ''), nullIf(toString(error.data.name), ''), 'Error')) as error_type,
any(coalesce(nullIf(substring(toString(error.data.message), 1, 500), ''), 'Unknown error')) as error_message,
any(coalesce(substring(toString(error.data.stack), 1, 2000), '')) as sample_stack_trace,
toDateTime(max(created_at)) as last_seen_date,
min(created_at) as first_seen,
max(created_at) as last_seen,
sumState(toUInt64(1)) as occurrence_count,
uniqState(task_version) as affected_task_versions,
anyState(run_id) as sample_run_id,
anyState(friendly_id) as sample_friendly_id,
sumMapState([status], [toUInt64(1)]) as status_distribution
FROM trigger_dev.task_runs_v2
WHERE
error_fingerprint != ''
AND status IN ('SYSTEM_FAILURE', 'CRASHED', 'INTERRUPTED', 'COMPLETED_WITH_ERRORS', 'TIMED_OUT')
AND _is_deleted = 0
GROUP BY
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint;
ALTER TABLE trigger_dev.error_occurrences_mv_v1 MODIFY QUERY
SELECT
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
task_version,
toStartOfMinute (created_at) as minute,
any (
coalesce(
nullIf(toString (error.data.type), ''),
nullIf(toString (error.data.name), ''),
'Error'
)
) as error_type,
any (
coalesce(
nullIf(
substring(toString (error.data.message), 1, 500),
''
),
'Unknown error'
)
) as error_message,
any (
coalesce(
substring(toString (error.data.stack), 1, 2000),
''
)
) as stack_trace,
count() as count
FROM
trigger_dev.task_runs_v2
WHERE
error_fingerprint != ''
AND status IN (
'SYSTEM_FAILURE',
'CRASHED',
'INTERRUPTED',
'COMPLETED_WITH_ERRORS',
'TIMED_OUT'
)
AND _is_deleted = 0
GROUP BY
organization_id,
project_id,
environment_id,
task_identifier,
error_fingerprint,
task_version,
minute;