chore: import upstream snapshot with attribution

tests/unit/compression/previewRouteTokens.test.ts

@@ -0,0 +1,32 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { join } from "node:path";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+
+const TEST_DATA_DIR = mkdtempSync(join(tmpdir(), "preview-tok-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+process.env.API_KEY_SECRET = process.env.API_KEY_SECRET ?? "test-secret-32-chars-min-aaaaaaaa";
+delete process.env.INITIAL_PASSWORD;
+
+const core = await import("../../../src/lib/db/core.ts");
+const route = await import("../../../src/app/api/compression/preview/route.ts");
+const { countTextTokens } = await import("../../../src/shared/utils/tiktokenCounter.ts");
+
+function makeReq(body: unknown) {
+  return new Request("http://localhost/api/compression/preview", {
+    method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body),
+  });
+}
+test.beforeEach(() => core.resetDbInstance());
+test.after(() => { core.resetDbInstance(); rmSync(TEST_DATA_DIR, { recursive: true, force: true }); });
+
+test("originalTokens equals countTextTokens, not the *1.33 estimate", async () => {
+  const text = "the quick brown fox jumps over the lazy dog repeatedly and often";
+  const res = await route.POST(makeReq({ messages: [{ role: "user", content: text }], mode: "off" }));
+  assert.equal(res.status, 200);
+  const body = await res.json();
+  assert.equal(body.originalTokens, countTextTokens(body.original));
+  const wordEstimate = Math.ceil(body.original.split(/\s+/).filter(Boolean).length * 1.33);
+  assert.notEqual(body.originalTokens, wordEstimate);
+});