{ "annotations": { "list": [] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [], "title": "Throughput", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.3.0", "targets": [ { "expr": "sum(rate(triggerdotdev_runs_replication_task_runs_inserted_inserts_total[5m]))", "legendFormat": "Runs/sec", "refId": "A" } ], "title": "Task Runs Inserted Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.3.0", "targets": [ { "expr": "sum(rate(triggerdotdev_runs_replication_payloads_inserted_inserts_total[5m]))", "legendFormat": "Payloads/sec", "refId": "A" } ], "title": "Payloads Inserted Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.3.0", "targets": [ { "expr": "sum(rate(triggerdotdev_runs_replication_batches_flushed_total[5m]))", "legendFormat": "Batches/sec", "refId": "A" } ], "title": "Batches Flushed Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 0.9 }, { "color": "green", "value": 0.99 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 1 }, "id": 5, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.3.0", "targets": [ { "expr": "sum(rate(triggerdotdev_runs_replication_batches_flushed_total{success=\"true\"}[5m])) / sum(rate(triggerdotdev_runs_replication_batches_flushed_total[5m]))", "legendFormat": "Success Rate", "refId": "A" } ], "title": "Flush Success Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "id": 6, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(rate(triggerdotdev_runs_replication_events_processed_total[5m])) by (event_type)", "legendFormat": "{{event_type}}", "refId": "A" } ], "title": "Events Processed by Type", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "id": 7, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(rate(triggerdotdev_runs_replication_task_runs_inserted_inserts_total[5m]))", "legendFormat": "Task Runs", "refId": "A" }, { "expr": "sum(rate(triggerdotdev_runs_replication_payloads_inserted_inserts_total[5m]))", "legendFormat": "Payloads", "refId": "B" } ], "title": "Insert Rate Over Time", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 8, "panels": [], "title": "Latency & Lag", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "id": 9, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(triggerdotdev_runs_replication_replication_lag_ms_milliseconds_bucket[5m])) by (le))", "legendFormat": "p50", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(triggerdotdev_runs_replication_replication_lag_ms_milliseconds_bucket[5m])) by (le))", "legendFormat": "p95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(triggerdotdev_runs_replication_replication_lag_ms_milliseconds_bucket[5m])) by (le))", "legendFormat": "p99", "refId": "C" } ], "title": "Replication Lag (Postgres → ClickHouse)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "id": 10, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(triggerdotdev_runs_replication_flush_duration_ms_milliseconds_bucket[5m])) by (le))", "legendFormat": "p50", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(triggerdotdev_runs_replication_flush_duration_ms_milliseconds_bucket[5m])) by (le))", "legendFormat": "p95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(triggerdotdev_runs_replication_flush_duration_ms_milliseconds_bucket[5m])) by (le))", "legendFormat": "p99", "refId": "C" } ], "title": "Flush Duration", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 11, "panels": [], "title": "Batching & Queue", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "id": 13, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(triggerdotdev_runs_replication_batch_size_items_bucket[5m])) by (le))", "legendFormat": "p50", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(triggerdotdev_runs_replication_batch_size_items_bucket[5m])) by (le))", "legendFormat": "p95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(triggerdotdev_runs_replication_batch_size_items_bucket[5m])) by (le))", "legendFormat": "p99", "refId": "C" } ], "title": "Batch Size Distribution", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 10 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "id": 14, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "11.3.0", "targets": [ { "expr": "sum(rate(triggerdotdev_runs_replication_insert_retries_total[5m]))", "legendFormat": "Retries/sec", "refId": "A" } ], "title": "Insert Retries (should be low)", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, "id": 15, "panels": [], "title": "Retries by Operation", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 34 }, "id": 16, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum(rate(triggerdotdev_runs_replication_insert_retries_total[5m])) by (operation)", "legendFormat": "{{operation}}", "refId": "A" } ], "title": "Insert Retries by Operation", "type": "timeseries" } ], "refresh": "10s", "schemaVersion": 39, "tags": ["trigger.dev", "runs-replication", "clickhouse"], "templating": { "list": [] }, "time": { "from": "now-15m", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Runs Replication Service", "uid": "runs-replication-metrics", "version": 1 }