triggerdotdev--trigger.dev/apps/webapp/test/otlpUtf16Sanitization.integration.test.ts

import { clickhouseTest } from "@internal/testcontainers";
import { describe, expect } from "vitest";
import {
  INVALID_UTF16_SENTINEL,
  isClickHouseJsonParseError,
  parseRowNumberFromError,
  sanitizeRows,
} from "~/v3/eventRepository/sanitizeRowsOnParseError.server";

/**
 * Integration test that proves the reactive sanitize-and-retry flow works
 * against a real ClickHouse instance. Boots a CH container (via testcontainers)
 * and reproduces the prod failure path end-to-end.
 *
 * Three contracts are verified:
 *
 * 1. **Happy retry path** — insert a row with a lone UTF-16 surrogate, observe
 *    the parse error, recover via `parseRowNumberFromError` +
 *    `sanitizeRowsFrom`, retry once, and confirm the row lands with the
 *    sentinel substituted.
 *
 * 2. **Real CH error shape** — confirm `isClickHouseJsonParseError` correctly
 *    recognises the error string we get back from a real CH (not just synthetic
 *    test fixtures) and that `parseRowNumberFromError` extracts the right
 *    integer from the same string.
 *
 * 3. **Non-parse errors don't get swallowed** — push a row past the CH per-row
 *    size cap and confirm the resulting `Size of JSON object ... is extremely
 *    large` error is NOT misclassified as a JSON parse error by our predicate.
 */

const HIGH_SURROGATE = "\uD800";
const LOW_SURROGATE = "\uDC00";

// ClickHouse container boot + image pull on first run can take well past
// vitest's 5 s default. Match what `internal-packages/clickhouse/vitest.config.ts`
// uses for its own clickhouseTest specs.
const INTEGRATION_TIMEOUT_MS = 60_000;

describe("OTel attribute UTF-16 sanitization → ClickHouse insert", () => {
  clickhouseTest(
    "lone surrogate is rejected by CH, then sanitized and retried successfully",
    async ({ clickhouseClient }) => {
      const table = "trigger_dev_test.utf16_repro";

      await clickhouseClient.command({
        query: "CREATE DATABASE IF NOT EXISTS trigger_dev_test",
      });
      await clickhouseClient.command({ query: `DROP TABLE IF EXISTS ${table}` });
      await clickhouseClient.command({
        query: `
          CREATE TABLE ${table} (
            id String,
            attributes JSON
          ) ENGINE = MergeTree() ORDER BY id
          SETTINGS allow_experimental_json_type = 1
        `,
      });

      const rows = [
        {
          id: "row-clean-prefix",
          attributes: { ai: { prompt: { messages: "valid prompt 1" } } },
        },
        {
          id: "row-poisoned",
          attributes: {
            ai: { prompt: { messages: `valid prefix ${HIGH_SURROGATE} broken tail` } },
          },
        },
        {
          id: "row-clean-suffix",
          attributes: { ai: { prompt: { messages: "valid prompt 3" } } },
        },
      ];

      // --- Contract 1: real CH rejects the raw insert with our recognisable error ---
      const firstError = await clickhouseClient
        .insert({
          table,
          values: rows,
          format: "JSONEachRow",
          clickhouse_settings: { async_insert: 0, input_format_parallel_parsing: 1 },
        })
        .then(
          () => null,
          (e: unknown) => e as Error
        );

      expect(firstError, "first insert must be rejected").not.toBeNull();
      expect(
        isClickHouseJsonParseError(firstError),
        "our predicate must recognise the real CH parse error"
      ).toBe(true);
      const rowN = parseRowNumberFromError(firstError!.message);
      expect(rowN, "real CH error must include `at row N`").not.toBeNull();
      expect(rowN! >= 0).toBe(true);

      // --- Recovery: sanitize the whole batch, retry ---
      // We don't slice on `rowN` even though we logged it — `at row N`
      // semantics under parallel parsing aren't stable enough to skip rows.
      const { rowsTouched, fieldsSanitized } = sanitizeRows(rows);
      expect(fieldsSanitized, "exactly one field should have been replaced").toBe(1);
      expect(rowsTouched).toBe(1);

      // Confirm the targeted row was sanitized and the clean ones were left alone.
      expect(rows[1].attributes.ai.prompt.messages).toBe(INVALID_UTF16_SENTINEL);
      expect(rows[0].attributes.ai.prompt.messages).toBe("valid prompt 1");
      expect(rows[2].attributes.ai.prompt.messages).toBe("valid prompt 3");

      // --- Contract 1 (cont'd): retry now lands cleanly ---
      await clickhouseClient.insert({
        table,
        values: rows,
        format: "JSONEachRow",
        clickhouse_settings: { async_insert: 0, input_format_parallel_parsing: 1 },
      });

      const result = await clickhouseClient
        .query({
          query: `
            SELECT id, toJSONString(attributes) AS attributes_text
            FROM ${table}
            ORDER BY id
          `,
          format: "JSONEachRow",
        })
        .then((r) => r.json<{ id: string; attributes_text: string }>());

      expect(result).toHaveLength(3);
      const byId = Object.fromEntries(result.map((r) => [r.id, r]));
      expect(byId["row-clean-prefix"].attributes_text).toContain("valid prompt 1");
      expect(byId["row-clean-suffix"].attributes_text).toContain("valid prompt 3");
      expect(byId["row-poisoned"].attributes_text).toContain(INVALID_UTF16_SENTINEL);

      // --- Contract 2: lone LOW surrogate also recognised + recoverable ---
      const lowSurrogateRow = {
        id: "row-low-surrogate",
        attributes: {
          ai: { prompt: { messages: `valid prefix ${LOW_SURROGATE} broken tail` } },
        },
      };
      const lowSurrogateError = await clickhouseClient
        .insert({
          table,
          values: [lowSurrogateRow],
          format: "JSONEachRow",
          clickhouse_settings: { async_insert: 0, input_format_parallel_parsing: 1 },
        })
        .then(
          () => null,
          (e: unknown) => e as Error
        );
      expect(lowSurrogateError).not.toBeNull();
      expect(isClickHouseJsonParseError(lowSurrogateError)).toBe(true);

      sanitizeRows([lowSurrogateRow]);
      expect(lowSurrogateRow.attributes.ai.prompt.messages).toBe(INVALID_UTF16_SENTINEL);

      await clickhouseClient.insert({
        table,
        values: [lowSurrogateRow],
        format: "JSONEachRow",
        clickhouse_settings: { async_insert: 0, input_format_parallel_parsing: 1 },
      });
    },
    INTEGRATION_TIMEOUT_MS
  );

  clickhouseTest(
    "non-parse-error rejections (e.g. missing table) are NOT misclassified as JSON parse errors",
    async ({ clickhouseClient }) => {
      // Pick an error class that is unambiguously NOT a JSON parse failure —
      // inserting into a table that doesn't exist. CH returns
      // `Table doesn't exist` (UNKNOWN_TABLE). If our predicate ever started
      // matching it we'd wastefully sanitize-and-retry an unrelated failure.
      const error = await clickhouseClient
        .insert({
          table: "trigger_dev_test_nonexistent.utf16_does_not_exist",
          values: [{ id: "1", attributes: { ok: "yes" } }],
          format: "JSONEachRow",
          clickhouse_settings: { async_insert: 0 },
        })
        .then(
          () => null,
          (e: unknown) => e as Error
        );

      expect(error, "missing-table insert should be rejected").not.toBeNull();
      expect(
        isClickHouseJsonParseError(error),
        "non-parse error must not be misclassified as JSON parse error"
      ).toBe(false);
    },
    INTEGRATION_TIMEOUT_MS
  );
});