{
  "title": "Realtime Native Backend",
  "uid": "realtime-native",
  "tags": ["trigger.dev", "realtime"],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "10s",
  "time": {
    "from": "now-30m",
    "to": "now"
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "links": [],
  "annotations": {
    "list": []
  },
  "templating": {
    "list": []
  },
  "panels": [
    {
      "id": 1,
      "type": "timeseries",
      "title": "Delivery lag (write \u2192 emission)",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 0
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "histogram_quantile(0.5, sum(rate(triggerdotdev_realtime_native_delivery_lag_ms_milliseconds_bucket[$__rate_interval])) by (le))",
          "legendFormat": "p50"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "histogram_quantile(0.99, sum(rate(triggerdotdev_realtime_native_delivery_lag_ms_milliseconds_bucket[$__rate_interval])) by (le))",
          "legendFormat": "p99"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ms",
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 8,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "The end-to-end SLI: now minus the newest emitted row's updatedAt. A p99 approaching the ~20s backstop hold means live wakes are being missed."
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "Live polls by path",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 0
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(triggerdotdev_realtime_native_live_polls_total[$__rate_interval])) by (path)",
          "legendFormat": "{{path}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 8,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "fast-hydrate = router wake, no ClickHouse. full-resolve = backstop. cold-resolve = fresh env subscription probed (instance hop / first poll)."
    },
    {
      "id": 3,
      "type": "stat",
      "title": "Backstop DELIVERED (should be ~0)",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 0
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(triggerdotdev_realtime_native_backstops_total{result=\"delivered\"}[5m])) or vector(0)",
          "legendFormat": "delivered/s"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.01
              }
            ]
          }
        },
        "overrides": []
      },
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": ["lastNotNull"]
        }
      },
      "description": "A backstop that finds missed changes means the notify/replay path is leaking. Alert on sustained non-zero."
    },
    {
      "id": 4,
      "type": "timeseries",
      "title": "Wakeups by reason",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(triggerdotdev_realtime_native_wakeups_total[$__rate_interval])) by (reason)",
          "legendFormat": "{{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 8,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "notify = the architecture working. A rising timeout share with active traffic = publishes not routing."
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Gap recovery: replays + evictions",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(triggerdotdev_realtime_native_replays_total[$__rate_interval])) by (result)",
          "legendFormat": "replay {{result}}"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(triggerdotdev_realtime_native_replay_evictions_total[$__rate_interval])) by (reason)",
          "legendFormat": "evict {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 8,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "Replays recover records that landed between a connection's polls. 'evict cap' = an env churns more runs than the buffer window holds \u2014 retune REPLAY_MAX_RUNS / WINDOW_MS."
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "Rows per emission (p99)",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "histogram_quantile(0.99, sum(rate(triggerdotdev_realtime_native_emitted_rows_bucket[$__rate_interval])) by (le))",
          "legendFormat": "p99 rows"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 8,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "Deltas should be small. A fat tail means working-set / offset-floor fallbacks are re-emitting full sets."
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "Held feeds by kind",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 16
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(triggerdotdev_realtime_native_held_feeds) by (kind)",
          "legendFormat": "{{kind}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 8,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "Long-polls currently held \u2014 the capacity unit."
    },
    {
      "id": 8,
      "type": "timeseries",
      "title": "Envs + channel subscriptions",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 16
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(triggerdotdev_realtime_native_active_envs)",
          "legendFormat": "routed envs"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(triggerdotdev_realtime_notifier_active_subscriptions)",
          "legendFormat": "redis channels"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 8,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "Routed envs includes lingering subscriptions (kept alive briefly after the last feed closes)."
    },
    {
      "id": 9,
      "type": "timeseries",
      "title": "Resolve health",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 16
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(triggerdotdev_realtime_native_runset_resolves_total[$__rate_interval])) by (result)",
          "legendFormat": "resolve {{result}}"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(triggerdotdev_realtime_native_resolve_admission_in_use)",
          "legendFormat": "gate in use"
        },
        {
          "refId": "C",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(triggerdotdev_realtime_native_concurrency_rejections_total[$__rate_interval]))",
          "legendFormat": "429/s"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 8,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "hit/coalesced vs miss = the single-flight cache collapsing same-filter herds. Gate in use near the limit = reconnect stampedes queueing."
    }
  ]
}