chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,23 @@
|
||||
// Calculate next build version based on the previous version
|
||||
// Version formats are YYYYMMDD.1, YYYYMMDD.2, etc.
|
||||
// If there is no previous version, start at Todays date and .1
|
||||
export function calculateNextBuildVersion(latestVersion?: string | null): string {
|
||||
const today = new Date();
|
||||
const year = today.getFullYear();
|
||||
const month = today.getMonth() + 1;
|
||||
const day = today.getDate();
|
||||
const todayFormatted = `${year}${month < 10 ? "0" : ""}${month}${day < 10 ? "0" : ""}${day}`;
|
||||
|
||||
if (!latestVersion) {
|
||||
return `${todayFormatted}.1`;
|
||||
}
|
||||
|
||||
const [date, buildNumber] = latestVersion.split(".");
|
||||
|
||||
if (date === todayFormatted) {
|
||||
const nextBuildNumber = parseInt(buildNumber, 10) + 1;
|
||||
return `${date}.${nextBuildNumber}`;
|
||||
}
|
||||
|
||||
return `${todayFormatted}.1`;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { parseExpression } from "cron-parser";
|
||||
|
||||
export function calculateNextScheduledTimestampFromNow(schedule: string, timezone: string | null) {
|
||||
return calculateNextScheduledTimestamp(schedule, timezone, new Date());
|
||||
}
|
||||
|
||||
export function calculateNextScheduledTimestamp(
|
||||
schedule: string,
|
||||
timezone: string | null,
|
||||
currentDate: Date = new Date()
|
||||
) {
|
||||
return calculateNextStep(schedule, timezone, currentDate);
|
||||
}
|
||||
|
||||
function calculateNextStep(schedule: string, timezone: string | null, currentDate: Date) {
|
||||
return parseExpression(schedule, {
|
||||
currentDate,
|
||||
utc: timezone === null,
|
||||
tz: timezone ?? undefined,
|
||||
})
|
||||
.next()
|
||||
.toDate();
|
||||
}
|
||||
|
||||
export function previousScheduledTimestamp(
|
||||
schedule: string,
|
||||
timezone: string | null,
|
||||
fromTimestamp: Date = new Date()
|
||||
) {
|
||||
return parseExpression(schedule, {
|
||||
currentDate: fromTimestamp,
|
||||
utc: timezone === null,
|
||||
tz: timezone ?? undefined,
|
||||
})
|
||||
.prev()
|
||||
.toDate();
|
||||
}
|
||||
|
||||
export function nextScheduledTimestamps(
|
||||
cron: string,
|
||||
timezone: string | null,
|
||||
lastScheduledTimestamp: Date,
|
||||
count: number = 1
|
||||
) {
|
||||
const result: Array<Date> = [];
|
||||
let nextScheduledTimestamp = lastScheduledTimestamp;
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
nextScheduledTimestamp = calculateNextScheduledTimestamp(
|
||||
cron,
|
||||
timezone,
|
||||
nextScheduledTimestamp
|
||||
);
|
||||
|
||||
result.push(nextScheduledTimestamp);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// Compares two versions of a deployment, like 20250208.1 and 20250208.2
|
||||
// Returns -1 if versionA is older than versionB, 0 if they are the same, and 1 if versionA is newer than versionB
|
||||
export function compareDeploymentVersions(versionA: string, versionB: string) {
|
||||
const [dateA, numberA] = versionA.split(".");
|
||||
const [dateB, numberB] = versionB.split(".");
|
||||
|
||||
if (dateA < dateB) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (dateA > dateB) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Convert to numbers before comparing
|
||||
const numA = Number(numberA);
|
||||
const numB = Number(numberB);
|
||||
|
||||
if (numA < numB) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (numA > numB) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { enrichCreatableEvents, setLlmPricingRegistry } from "./enrichCreatableEvents.server";
|
||||
import type { CreateEventInput } from "../eventRepository/eventRepository.types";
|
||||
|
||||
type Registry = Parameters<typeof setLlmPricingRegistry>[0];
|
||||
type RegistryCost = NonNullable<ReturnType<Registry["calculateCost"]>>;
|
||||
|
||||
function registryReturning(cost: RegistryCost | null, isLoaded = true): Registry {
|
||||
return {
|
||||
isLoaded,
|
||||
calculateCost: () => cost,
|
||||
};
|
||||
}
|
||||
|
||||
// A catalog cost result the registry would produce. The numbers here stand in for a catalog
|
||||
// price that disagrees with what the provider actually billed.
|
||||
function catalogCost(overrides: Partial<RegistryCost> = {}): RegistryCost {
|
||||
return {
|
||||
matchedModelId: "llm_model_test",
|
||||
matchedModelName: "test-model",
|
||||
pricingTierId: "tier_standard",
|
||||
pricingTierName: "Standard",
|
||||
inputCost: 0,
|
||||
outputCost: 0,
|
||||
totalCost: 0,
|
||||
costDetails: {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeEvent(properties: Record<string, unknown>): CreateEventInput {
|
||||
return {
|
||||
message: "ai.generateText.doGenerate",
|
||||
kind: "INTERNAL",
|
||||
isPartial: false,
|
||||
properties: properties as CreateEventInput["properties"],
|
||||
} as unknown as CreateEventInput;
|
||||
}
|
||||
|
||||
function enrichOne(event: CreateEventInput): CreateEventInput {
|
||||
const [out] = enrichCreatableEvents([event]);
|
||||
return out;
|
||||
}
|
||||
|
||||
function costPillText(event: CreateEventInput): string | undefined {
|
||||
const accessory = (event.style as any)?.accessory;
|
||||
const items: Array<{ text: string; icon: string }> = accessory?.items ?? [];
|
||||
return items.find((i) => i.icon === "tabler-currency-dollar")?.text;
|
||||
}
|
||||
|
||||
function modelPillText(event: CreateEventInput): string | undefined {
|
||||
const accessory = (event.style as any)?.accessory;
|
||||
const items: Array<{ text: string; icon: string }> = accessory?.items ?? [];
|
||||
return items.find((i) => i.icon === "tabler-cube")?.text;
|
||||
}
|
||||
|
||||
describe("enrichLlmMetrics — provider-reported cost", () => {
|
||||
it("prefers the provider-reported cost over catalog pricing when a cache discount applies", () => {
|
||||
// OpenRouter served mimo with 25,280 of 34,374 prompt tokens as cache reads. Cache counts
|
||||
// only reach us via providerMetadata, so the catalog bills the full prompt at the input
|
||||
// rate while OpenRouter's exact bill reflects the cache discount.
|
||||
setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.01603584 })));
|
||||
|
||||
const out = enrichOne(
|
||||
makeEvent({
|
||||
"gen_ai.system": "openrouter",
|
||||
"gen_ai.request.model": "xiaomi/mimo-v2.5-pro",
|
||||
"gen_ai.response.model": "xiaomi/mimo-v2.5-pro-20260422",
|
||||
"gen_ai.usage.input_tokens": 34374,
|
||||
"gen_ai.usage.output_tokens": 1245,
|
||||
"ai.response.providerMetadata": JSON.stringify({
|
||||
openrouter: {
|
||||
provider: "Xiaomi",
|
||||
usage: { cost: 0.005130048, promptTokensDetails: { cachedTokens: 25280 } },
|
||||
},
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
expect(out.properties["trigger.llm.total_cost"]).toBe(0.005130048);
|
||||
expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter");
|
||||
// The catalog breakdown must not be written — it priced the wrong (full-rate) total.
|
||||
expect(out.properties["trigger.llm.input_cost"]).toBeUndefined();
|
||||
expect(out.properties["trigger.llm.matched_model"]).toBeUndefined();
|
||||
expect(costPillText(out)).toBe("$0.005130");
|
||||
|
||||
expect(out._llmMetrics?.totalCost).toBe(0.005130048);
|
||||
expect(out._llmMetrics?.costSource).toBe("openrouter");
|
||||
expect(out._llmMetrics?.providerCost).toBe(0.005130048);
|
||||
expect(out._llmMetrics?.inputCost).toBe(0);
|
||||
});
|
||||
|
||||
it("prices the served fallback model's provider cost, not the requested model", () => {
|
||||
// Requested mimo, OpenRouter routed to a gemini fallback: gen_ai.response.model carries the
|
||||
// SERVED model, and the provider cost is authoritative.
|
||||
setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.02 })));
|
||||
|
||||
const out = enrichOne(
|
||||
makeEvent({
|
||||
"gen_ai.system": "openrouter",
|
||||
"gen_ai.request.model": "xiaomi/mimo-v2.5-pro",
|
||||
"gen_ai.response.model": "google/gemini-3.5-flash-20260519",
|
||||
"gen_ai.usage.input_tokens": 5000,
|
||||
"gen_ai.usage.output_tokens": 800,
|
||||
"ai.response.providerMetadata": JSON.stringify({
|
||||
openrouter: { provider: "Google", usage: { cost: 0.011058 } },
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
expect(out.properties["trigger.llm.total_cost"]).toBe(0.011058);
|
||||
expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter");
|
||||
// The pill (and stored response model) reflect the served fallback model.
|
||||
expect(modelPillText(out)).toBe("google/gemini-3.5-flash-20260519");
|
||||
expect(out._llmMetrics?.requestModel).toBe("xiaomi/mimo-v2.5-pro");
|
||||
expect(out._llmMetrics?.responseModel).toBe("google/gemini-3.5-flash-20260519");
|
||||
expect(out._llmMetrics?.totalCost).toBe(0.011058);
|
||||
});
|
||||
|
||||
it("prefers the gateway-reported cost over catalog pricing", () => {
|
||||
setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.02 })));
|
||||
|
||||
const out = enrichOne(
|
||||
makeEvent({
|
||||
"gen_ai.system": "gateway",
|
||||
"gen_ai.response.model": "openai/gpt-4o",
|
||||
"gen_ai.usage.input_tokens": 1000,
|
||||
"gen_ai.usage.output_tokens": 500,
|
||||
"ai.response.providerMetadata": JSON.stringify({
|
||||
gateway: { cost: "0.0006615" },
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
expect(out.properties["trigger.llm.total_cost"]).toBe(0.0006615);
|
||||
expect(out.properties["trigger.llm.cost_source"]).toBe("gateway");
|
||||
expect(out._llmMetrics?.costSource).toBe("gateway");
|
||||
});
|
||||
|
||||
it("uses provider cost even when the catalog does not match the model", () => {
|
||||
setLlmPricingRegistry(registryReturning(null));
|
||||
|
||||
const out = enrichOne(
|
||||
makeEvent({
|
||||
"gen_ai.system": "openrouter",
|
||||
"gen_ai.response.model": "some/unlisted-model",
|
||||
"gen_ai.usage.input_tokens": 2000,
|
||||
"gen_ai.usage.output_tokens": 300,
|
||||
"ai.response.providerMetadata": JSON.stringify({
|
||||
openrouter: { provider: "SomeProvider", usage: { cost: 0.00042 } },
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
expect(out.properties["trigger.llm.total_cost"]).toBe(0.00042);
|
||||
expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter");
|
||||
});
|
||||
|
||||
it("falls back to catalog pricing when no provider cost is reported", () => {
|
||||
setLlmPricingRegistry(
|
||||
registryReturning(
|
||||
catalogCost({
|
||||
matchedModelId: "llm_model_gpt4o",
|
||||
matchedModelName: "gpt-4o",
|
||||
inputCost: 0.04,
|
||||
outputCost: 0.01,
|
||||
totalCost: 0.05,
|
||||
costDetails: { input: 0.04, output: 0.01 },
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
const out = enrichOne(
|
||||
makeEvent({
|
||||
"gen_ai.system": "openai",
|
||||
"gen_ai.response.model": "gpt-4o",
|
||||
"gen_ai.usage.input_tokens": 1000,
|
||||
"gen_ai.usage.output_tokens": 500,
|
||||
})
|
||||
);
|
||||
|
||||
expect(out.properties["trigger.llm.total_cost"]).toBe(0.05);
|
||||
expect(out.properties["trigger.llm.input_cost"]).toBe(0.04);
|
||||
expect(out.properties["trigger.llm.output_cost"]).toBe(0.01);
|
||||
expect(out.properties["trigger.llm.matched_model"]).toBe("gpt-4o");
|
||||
// Registry path does not set a cost_source attribute.
|
||||
expect(out.properties["trigger.llm.cost_source"]).toBeUndefined();
|
||||
expect(out._llmMetrics?.costSource).toBe("registry");
|
||||
expect(out._llmMetrics?.matchedModelId).toBe("llm_model_gpt4o");
|
||||
});
|
||||
|
||||
it("falls back to catalog pricing when providerMetadata carries no cost field", () => {
|
||||
// The cheap `"cost"` guard should skip parsing and let the registry price this span.
|
||||
setLlmPricingRegistry(
|
||||
registryReturning(catalogCost({ totalCost: 0.03, matchedModelName: "claude" }))
|
||||
);
|
||||
|
||||
const out = enrichOne(
|
||||
makeEvent({
|
||||
"gen_ai.system": "anthropic",
|
||||
"gen_ai.response.model": "claude-sonnet-4-0",
|
||||
"gen_ai.usage.input_tokens": 1000,
|
||||
"gen_ai.usage.output_tokens": 500,
|
||||
"ai.response.providerMetadata": JSON.stringify({
|
||||
anthropic: { usage: { service_tier: "standard" } },
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
expect(out.properties["trigger.llm.total_cost"]).toBe(0.03);
|
||||
expect(out.properties["trigger.llm.cost_source"]).toBeUndefined();
|
||||
expect(out._llmMetrics?.costSource).toBe("registry");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,506 @@
|
||||
import { modelCatalog } from "@internal/llm-model-catalog";
|
||||
import type { CreateEventInput, LlmMetricsData } from "../eventRepository/eventRepository.types";
|
||||
|
||||
// Registry interface — matches ModelPricingRegistry from @internal/llm-model-catalog
|
||||
type CostRegistry = {
|
||||
isLoaded: boolean;
|
||||
calculateCost(
|
||||
responseModel: string,
|
||||
usageDetails: Record<string, number>
|
||||
): {
|
||||
matchedModelId: string;
|
||||
matchedModelName: string;
|
||||
pricingTierId: string;
|
||||
pricingTierName: string;
|
||||
inputCost: number;
|
||||
outputCost: number;
|
||||
totalCost: number;
|
||||
costDetails: Record<string, number>;
|
||||
} | null;
|
||||
};
|
||||
|
||||
let _registry: CostRegistry | undefined;
|
||||
|
||||
const ENRICHABLE_KINDS = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]);
|
||||
|
||||
export function setLlmPricingRegistry(registry: CostRegistry): void {
|
||||
_registry = registry;
|
||||
}
|
||||
|
||||
export function enrichCreatableEvents(events: CreateEventInput[]) {
|
||||
return events.map((event) => {
|
||||
return enrichCreatableEvent(event);
|
||||
});
|
||||
}
|
||||
|
||||
function enrichCreatableEvent(event: CreateEventInput): CreateEventInput {
|
||||
const message = formatPythonStyle(event.message, event.properties);
|
||||
|
||||
event.message = message;
|
||||
event.style = enrichStyle(event);
|
||||
|
||||
enrichLlmMetrics(event);
|
||||
enrichPromptResolve(event);
|
||||
|
||||
return event;
|
||||
}
|
||||
|
||||
function enrichLlmMetrics(event: CreateEventInput): void {
|
||||
const props = event.properties;
|
||||
if (!props) return;
|
||||
|
||||
// Only enrich span-like events (INTERNAL, SERVER, CLIENT, CONSUMER, PRODUCER — not LOG, UNSPECIFIED)
|
||||
if (!ENRICHABLE_KINDS.has(event.kind as string)) return;
|
||||
|
||||
// Skip partial spans (they don't have final token counts)
|
||||
if (event.isPartial) return;
|
||||
|
||||
// Only use gen_ai.* attributes for model resolution to avoid double-counting.
|
||||
// The Vercel AI SDK emits both a parent span (ai.streamText with ai.usage.*)
|
||||
// and a child span (ai.streamText.doStream with gen_ai.*). We only enrich the
|
||||
// child span that has the canonical gen_ai.response.model attribute.
|
||||
const responseModel =
|
||||
typeof props["gen_ai.response.model"] === "string"
|
||||
? props["gen_ai.response.model"]
|
||||
: typeof props["gen_ai.request.model"] === "string"
|
||||
? props["gen_ai.request.model"]
|
||||
: null;
|
||||
|
||||
if (!responseModel) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract usage details, normalizing attribute names
|
||||
const usageDetails = extractUsageDetails(props);
|
||||
|
||||
// Need at least some token usage
|
||||
const hasTokens = Object.values(usageDetails).some((v) => v > 0);
|
||||
if (!hasTokens) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Add style accessories for model and tokens (even without cost data)
|
||||
const inputTokens = usageDetails["input"] ?? 0;
|
||||
const outputTokens = usageDetails["output"] ?? 0;
|
||||
const totalTokens = usageDetails["total"] ?? inputTokens + outputTokens;
|
||||
|
||||
const pillItems: Array<{ text: string; icon: string }> = [
|
||||
{ text: responseModel, icon: "tabler-cube" },
|
||||
{ text: formatTokenCount(totalTokens), icon: "tabler-hash" },
|
||||
];
|
||||
|
||||
// Provider-reported cost (gateway/openrouter) is the exact per-request bill and already
|
||||
// reflects cache-read discounts and the real per-provider rate, so prefer it and only fall
|
||||
// back to catalog pricing when it is absent. The registry handles prefix stripping (e.g.
|
||||
// "mistral/mistral-large-3" → "mistral-large-3") for gateway/openrouter models in match().
|
||||
const providerCost = extractProviderCost(props);
|
||||
|
||||
let cost: ReturnType<NonNullable<typeof _registry>["calculateCost"]> | null = null;
|
||||
if (!providerCost && _registry?.isLoaded) {
|
||||
cost = _registry.calculateCost(responseModel, usageDetails);
|
||||
}
|
||||
|
||||
if (cost) {
|
||||
// Add trigger.llm.* attributes to the span from our pricing registry
|
||||
event.properties = {
|
||||
...props,
|
||||
"trigger.llm.input_cost": cost.inputCost,
|
||||
"trigger.llm.output_cost": cost.outputCost,
|
||||
"trigger.llm.total_cost": cost.totalCost,
|
||||
"trigger.llm.cached_cost": cost.costDetails["input_cached_tokens"] ?? 0,
|
||||
"trigger.llm.cache_creation_cost": cost.costDetails["cache_creation_input_tokens"] ?? 0,
|
||||
"trigger.llm.matched_model": cost.matchedModelName,
|
||||
"trigger.llm.matched_model_id": cost.matchedModelId,
|
||||
"trigger.llm.pricing_tier": cost.pricingTierName,
|
||||
"trigger.llm.pricing_tier_id": cost.pricingTierId,
|
||||
};
|
||||
|
||||
pillItems.push({ text: formatCost(cost.totalCost), icon: "tabler-currency-dollar" });
|
||||
} else if (providerCost) {
|
||||
// Use provider-reported cost as fallback (no input/output breakdown available)
|
||||
event.properties = {
|
||||
...props,
|
||||
"trigger.llm.total_cost": providerCost.totalCost,
|
||||
"trigger.llm.cost_source": providerCost.source,
|
||||
};
|
||||
|
||||
pillItems.push({ text: formatCost(providerCost.totalCost), icon: "tabler-currency-dollar" });
|
||||
}
|
||||
|
||||
event.style = {
|
||||
...(event.style as Record<string, unknown> | undefined),
|
||||
accessory: {
|
||||
style: "pills",
|
||||
items: pillItems,
|
||||
},
|
||||
} as unknown as typeof event.style;
|
||||
|
||||
// Only write llm_metrics when cost data is available
|
||||
if (!cost && !providerCost) return;
|
||||
|
||||
// Build metadata map from run tags and ai.telemetry.metadata.*
|
||||
const metadata: Record<string, string> = {};
|
||||
|
||||
if (event.runTags) {
|
||||
for (const tag of event.runTags) {
|
||||
const colonIdx = tag.indexOf(":");
|
||||
if (colonIdx > 0) {
|
||||
metadata[tag.substring(0, colonIdx)] = tag.substring(colonIdx + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(props)) {
|
||||
if (key.startsWith("ai.telemetry.metadata.") && typeof value === "string") {
|
||||
metadata[key.slice("ai.telemetry.metadata.".length)] = value;
|
||||
}
|
||||
}
|
||||
|
||||
// Extract new performance/behavioral fields.
|
||||
// v6 emits ai.response.finishReason (plain string); v7 (@ai-sdk/otel) emits
|
||||
// gen_ai.response.finish_reasons as a JSON array string (e.g. `["stop"]`).
|
||||
const finishReason = readFinishReason(props);
|
||||
const operationId =
|
||||
typeof props["ai.operationId"] === "string"
|
||||
? props["ai.operationId"]
|
||||
: typeof props["gen_ai.operation.name"] === "string"
|
||||
? props["gen_ai.operation.name"]
|
||||
: typeof props["operation.name"] === "string"
|
||||
? props["operation.name"]
|
||||
: "";
|
||||
const msToFirstChunk =
|
||||
typeof props["ai.response.msToFirstChunk"] === "number"
|
||||
? props["ai.response.msToFirstChunk"]
|
||||
: 0;
|
||||
const avgTokensPerSec =
|
||||
typeof props["ai.response.avgOutputTokensPerSecond"] === "number"
|
||||
? props["ai.response.avgOutputTokensPerSecond"]
|
||||
: 0;
|
||||
const costSource = cost ? "registry" : providerCost ? providerCost.source : "";
|
||||
const providerCostValue = providerCost?.totalCost ?? 0;
|
||||
|
||||
// Set _llmMetrics side-channel for dual-write to llm_metrics_v1
|
||||
const llmMetrics: LlmMetricsData = {
|
||||
genAiSystem:
|
||||
typeof props["gen_ai.system"] === "string"
|
||||
? props["gen_ai.system"]
|
||||
: typeof props["gen_ai.provider.name"] === "string"
|
||||
? props["gen_ai.provider.name"]
|
||||
: "unknown",
|
||||
requestModel:
|
||||
typeof props["gen_ai.request.model"] === "string"
|
||||
? props["gen_ai.request.model"]
|
||||
: responseModel,
|
||||
responseModel,
|
||||
baseResponseModel: modelCatalog[responseModel]?.baseModelName ?? responseModel,
|
||||
matchedModelId: cost?.matchedModelId ?? "",
|
||||
operationId,
|
||||
finishReason,
|
||||
costSource,
|
||||
pricingTierId: cost?.pricingTierId ?? (providerCost ? `provider:${providerCost.source}` : ""),
|
||||
pricingTierName:
|
||||
cost?.pricingTierName ?? (providerCost ? `${providerCost.source} reported` : ""),
|
||||
inputTokens: usageDetails["input"] ?? 0,
|
||||
outputTokens: usageDetails["output"] ?? 0,
|
||||
totalTokens:
|
||||
usageDetails["total"] ?? (usageDetails["input"] ?? 0) + (usageDetails["output"] ?? 0),
|
||||
usageDetails,
|
||||
inputCost: cost?.inputCost ?? 0,
|
||||
outputCost: cost?.outputCost ?? 0,
|
||||
totalCost: cost?.totalCost ?? providerCost?.totalCost ?? 0,
|
||||
costDetails: cost?.costDetails ?? {},
|
||||
providerCost: providerCostValue,
|
||||
msToFirstChunk,
|
||||
tokensPerSecond: avgTokensPerSec,
|
||||
metadata,
|
||||
promptSlug: metadata["prompt.slug"] ?? "",
|
||||
promptVersion: parseInt(metadata["prompt.version"] ?? "0", 10) || 0,
|
||||
};
|
||||
|
||||
event._llmMetrics = llmMetrics;
|
||||
}
|
||||
|
||||
function extractUsageDetails(props: Record<string, unknown>): Record<string, number> {
|
||||
const details: Record<string, number> = {};
|
||||
|
||||
// Only map gen_ai.usage.* attributes — NOT ai.usage.* from parent spans.
|
||||
// This prevents double-counting when both parent (ai.streamText) and child
|
||||
// (ai.streamText.doStream) spans carry token counts.
|
||||
const mappings: Record<string, string> = {
|
||||
"gen_ai.usage.input_tokens": "input",
|
||||
"gen_ai.usage.output_tokens": "output",
|
||||
"gen_ai.usage.prompt_tokens": "input",
|
||||
"gen_ai.usage.completion_tokens": "output",
|
||||
"gen_ai.usage.total_tokens": "total",
|
||||
"gen_ai.usage.cache_read_input_tokens": "input_cached_tokens",
|
||||
"gen_ai.usage.input_tokens_cache_read": "input_cached_tokens",
|
||||
// AI SDK 7 (@ai-sdk/otel) nests cache token counts: gen_ai.usage.cache_read.input_tokens
|
||||
"gen_ai.usage.cache_read.input_tokens": "input_cached_tokens",
|
||||
"gen_ai.usage.cache_creation_input_tokens": "cache_creation_input_tokens",
|
||||
"gen_ai.usage.input_tokens_cache_write": "cache_creation_input_tokens",
|
||||
"gen_ai.usage.cache_creation.input_tokens": "cache_creation_input_tokens",
|
||||
"gen_ai.usage.reasoning_tokens": "reasoning_tokens",
|
||||
};
|
||||
|
||||
for (const [attrKey, usageKey] of Object.entries(mappings)) {
|
||||
const value = props[attrKey];
|
||||
if (typeof value === "number" && value > 0) {
|
||||
// Don't overwrite if already set (first mapping wins)
|
||||
if (details[usageKey] === undefined) {
|
||||
details[usageKey] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return details;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the finish reason across AI SDK majors. v6 emits
|
||||
* `ai.response.finishReason` as a plain string; v7 (@ai-sdk/otel) emits
|
||||
* `gen_ai.response.finish_reasons` as a JSON array string (e.g. `["stop"]`).
|
||||
*/
|
||||
function readFinishReason(props: Record<string, unknown>): string {
|
||||
const v6 = props["ai.response.finishReason"];
|
||||
if (typeof v6 === "string" && v6) return v6;
|
||||
|
||||
const v7 = props["gen_ai.response.finish_reasons"];
|
||||
if (typeof v7 === "string" && v7) {
|
||||
const trimmed = v7.trim();
|
||||
if (trimmed.startsWith("[")) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
if (Array.isArray(parsed)) {
|
||||
const first = parsed.find((r) => typeof r === "string");
|
||||
if (typeof first === "string") return first;
|
||||
}
|
||||
} catch {
|
||||
// fall through to the raw value
|
||||
}
|
||||
}
|
||||
return v7;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function enrichStyle(event: CreateEventInput) {
|
||||
const baseStyle = event.style ?? {};
|
||||
const props = event.properties;
|
||||
|
||||
if (!props) {
|
||||
return baseStyle;
|
||||
}
|
||||
|
||||
const system = props["gen_ai.system"] ?? props["gen_ai.provider.name"];
|
||||
const modelId = props["gen_ai.request.model"] ?? props["ai.model.id"];
|
||||
|
||||
const provider = resolveAiProvider(
|
||||
typeof system === "string" ? system : undefined,
|
||||
typeof modelId === "string" ? modelId : undefined
|
||||
);
|
||||
|
||||
if (provider) {
|
||||
return { ...baseStyle, icon: `ai-provider-${provider}` };
|
||||
}
|
||||
|
||||
// Agent workflow check
|
||||
const name = props["name"];
|
||||
if (typeof name === "string" && name.includes("Agent workflow")) {
|
||||
return { ...baseStyle, icon: "tabler-brain" };
|
||||
}
|
||||
|
||||
const message = event.message;
|
||||
|
||||
if (typeof message === "string" && message === "ai.toolCall") {
|
||||
return { ...baseStyle, icon: "hero-wrench" };
|
||||
}
|
||||
|
||||
if (typeof message === "string" && message.startsWith("ai.")) {
|
||||
return { ...baseStyle, icon: "hero-sparkles" };
|
||||
}
|
||||
|
||||
return baseStyle;
|
||||
}
|
||||
|
||||
function formatTokenCount(tokens: number): string {
|
||||
if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(1)}M`;
|
||||
if (tokens >= 1_000) return `${(tokens / 1_000).toFixed(1)}k`;
|
||||
return tokens.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract provider-reported cost from ai.response.providerMetadata.
|
||||
* Gateway and OpenRouter include per-request cost in their metadata.
|
||||
*/
|
||||
function extractProviderCost(
|
||||
props: Record<string, unknown>
|
||||
): { totalCost: number; source: string } | null {
|
||||
const rawMeta = props["ai.response.providerMetadata"];
|
||||
if (typeof rawMeta !== "string") return null;
|
||||
|
||||
// Cheap guard: providerMetadata can be large for reasoning models (it carries the full
|
||||
// reasoning_details text), and this now runs on every AI span. Skip the JSON parse when
|
||||
// there is no cost field to find.
|
||||
if (!rawMeta.includes('"cost"')) return null;
|
||||
|
||||
let meta: Record<string, unknown>;
|
||||
try {
|
||||
meta = JSON.parse(rawMeta) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!meta || typeof meta !== "object") return null;
|
||||
|
||||
// Gateway: { gateway: { cost: "0.0006615" } }
|
||||
const gateway = meta.gateway;
|
||||
if (gateway && typeof gateway === "object") {
|
||||
const gw = gateway as Record<string, unknown>;
|
||||
const cost = parseFloat(String(gw.cost ?? "0"));
|
||||
if (cost > 0) return { totalCost: cost, source: "gateway" };
|
||||
}
|
||||
|
||||
// OpenRouter: { openrouter: { usage: { cost: 0.000135 } } }
|
||||
const openrouter = meta.openrouter;
|
||||
if (openrouter && typeof openrouter === "object") {
|
||||
const or = openrouter as Record<string, unknown>;
|
||||
const usage = or.usage;
|
||||
if (usage && typeof usage === "object") {
|
||||
const cost = Number((usage as Record<string, unknown>).cost ?? 0);
|
||||
if (cost > 0) return { totalCost: cost, source: "openrouter" };
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function formatCost(cost: number): string {
|
||||
if (cost >= 1) return `$${cost.toFixed(2)}`;
|
||||
if (cost >= 0.01) return `$${cost.toFixed(4)}`;
|
||||
return `$${cost.toFixed(6)}`;
|
||||
}
|
||||
|
||||
function repr(value: any): string {
|
||||
if (typeof value === "string") {
|
||||
return `'${value}'`;
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function formatPythonStyle(template: string, values: Record<string, any>): string {
|
||||
// Early return if template is too long
|
||||
if (template.length >= 256) {
|
||||
return template;
|
||||
}
|
||||
|
||||
// Early return if no template variables present
|
||||
if (!template.includes("{")) {
|
||||
return template;
|
||||
}
|
||||
|
||||
return template.replace(/\{([^}]+?)(?:!r)?\}/g, (match, key) => {
|
||||
const hasRepr = match.endsWith("!r}");
|
||||
const actualKey = hasRepr ? key : key;
|
||||
const value = values?.[actualKey];
|
||||
|
||||
if (value === undefined) {
|
||||
return match;
|
||||
}
|
||||
|
||||
return hasRepr ? repr(value) : String(value);
|
||||
});
|
||||
}
|
||||
|
||||
type AiProvider =
|
||||
| "anthropic"
|
||||
| "openai"
|
||||
| "gemini"
|
||||
| "llama"
|
||||
| "deepseek"
|
||||
| "xai"
|
||||
| "perplexity"
|
||||
| "cerebras"
|
||||
| "azure"
|
||||
| "mistral";
|
||||
|
||||
const systemToProvider: Record<string, AiProvider> = {
|
||||
anthropic: "anthropic",
|
||||
openai: "openai",
|
||||
azure: "azure",
|
||||
"google.generative-ai": "gemini",
|
||||
google: "gemini",
|
||||
xai: "xai",
|
||||
deepseek: "deepseek",
|
||||
cerebras: "cerebras",
|
||||
perplexity: "perplexity",
|
||||
"meta-llama": "llama",
|
||||
mistral: "mistral",
|
||||
};
|
||||
|
||||
const modelPatterns: [RegExp, AiProvider][] = [
|
||||
[/\banthropic\b|claude/i, "anthropic"],
|
||||
[/\bopenai\b|gpt-|o[134]-|chatgpt/i, "openai"],
|
||||
[/gemini/i, "gemini"],
|
||||
[/llama/i, "llama"],
|
||||
[/deepseek/i, "deepseek"],
|
||||
[/grok/i, "xai"],
|
||||
[/sonar/i, "perplexity"],
|
||||
[/cerebras/i, "cerebras"],
|
||||
[/mistral|mixtral|codestral|pixtral/i, "mistral"],
|
||||
];
|
||||
|
||||
function resolveAiProvider(
|
||||
system: string | undefined,
|
||||
modelId: string | undefined
|
||||
): AiProvider | undefined {
|
||||
if (modelId) {
|
||||
if (modelId.includes("/")) {
|
||||
const prefix = modelId.split("/")[0].toLowerCase();
|
||||
const fromPrefix = systemToProvider[prefix];
|
||||
if (fromPrefix) return fromPrefix;
|
||||
}
|
||||
|
||||
for (const [pattern, provider] of modelPatterns) {
|
||||
if (pattern.test(modelId)) return provider;
|
||||
}
|
||||
}
|
||||
|
||||
if (system) {
|
||||
const normalized = system.toLowerCase().split(".")[0];
|
||||
return systemToProvider[system] ?? systemToProvider[normalized];
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function enrichPromptResolve(event: CreateEventInput): void {
|
||||
const props = event.properties;
|
||||
if (!props) return;
|
||||
|
||||
const slug = props["prompt.slug"];
|
||||
const version = props["prompt.version"];
|
||||
|
||||
if (typeof slug !== "string") return;
|
||||
|
||||
const style = (event.style ?? {}) as Record<string, unknown>;
|
||||
const accessory = style.accessory as Record<string, unknown> | undefined;
|
||||
const existingItems =
|
||||
accessory && "items" in accessory
|
||||
? (accessory.items as Array<{ text: string; icon?: string; variant?: string }>)
|
||||
: [];
|
||||
|
||||
const items = [
|
||||
...existingItems,
|
||||
{
|
||||
text: `${slug}${typeof version === "number" ? ` v${version}` : ""}`,
|
||||
icon: "tabler-file-text-ai",
|
||||
},
|
||||
];
|
||||
|
||||
event.style = {
|
||||
...style,
|
||||
icon: style.icon ?? "tabler-file-text-ai",
|
||||
accessory: { style: "pills" as const, items },
|
||||
} as unknown as typeof event.style;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
const MINIMUM_MAX_DURATION = 5;
|
||||
const MAXIMUM_MAX_DURATION = 2_147_483_647; // largest 32-bit signed integer
|
||||
|
||||
export function clampMaxDuration(maxDuration: number): number {
|
||||
return Math.min(Math.max(maxDuration, MINIMUM_MAX_DURATION), MAXIMUM_MAX_DURATION);
|
||||
}
|
||||
|
||||
export function getMaxDuration(
|
||||
maxDuration?: number | null,
|
||||
defaultMaxDuration?: number | null
|
||||
): number | undefined {
|
||||
if (!maxDuration) {
|
||||
return defaultMaxDuration ?? undefined;
|
||||
}
|
||||
|
||||
// Setting the maxDuration to MAXIMUM_MAX_DURATION means we don't want to use the default maxDuration
|
||||
if (maxDuration === MAXIMUM_MAX_DURATION) {
|
||||
return;
|
||||
}
|
||||
|
||||
return maxDuration;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import type { RuntimeEnvironmentType } from "@trigger.dev/database";
|
||||
import { env } from "~/env.server";
|
||||
|
||||
/**
|
||||
* Organization fields needed for queue limit calculation.
|
||||
*/
|
||||
export type QueueLimitOrganization = {
|
||||
maximumDevQueueSize: number | null;
|
||||
maximumDeployedQueueSize: number | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Calculates the queue size limit for an environment based on its type and organization settings.
|
||||
*
|
||||
* Resolution order:
|
||||
* 1. Organization-level override (set by billing sync or admin)
|
||||
* 2. Environment variable fallback
|
||||
* 3. null if neither is set
|
||||
*
|
||||
* @param environmentType - The type of the runtime environment
|
||||
* @param organization - Organization with queue limit fields
|
||||
* @returns The queue size limit, or null if unlimited
|
||||
*/
|
||||
export function getQueueSizeLimit(
|
||||
environmentType: RuntimeEnvironmentType,
|
||||
organization: QueueLimitOrganization
|
||||
): number | null {
|
||||
if (environmentType === "DEVELOPMENT") {
|
||||
return organization.maximumDevQueueSize ?? env.MAXIMUM_DEV_QUEUE_SIZE ?? null;
|
||||
}
|
||||
|
||||
return organization.maximumDeployedQueueSize ?? env.MAXIMUM_DEPLOYED_QUEUE_SIZE ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines the source of the queue size limit for display purposes.
|
||||
*
|
||||
* @param environmentType - The type of the runtime environment
|
||||
* @param organization - Organization with queue limit fields
|
||||
* @returns "plan" if org has a value (typically set by billing), "default" if using env var fallback
|
||||
*/
|
||||
export function getQueueSizeLimitSource(
|
||||
environmentType: RuntimeEnvironmentType,
|
||||
organization: QueueLimitOrganization
|
||||
): "plan" | "default" {
|
||||
if (environmentType === "DEVELOPMENT") {
|
||||
return organization.maximumDevQueueSize !== null ? "plan" : "default";
|
||||
}
|
||||
|
||||
return organization.maximumDeployedQueueSize !== null ? "plan" : "default";
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import type { Logger } from "@trigger.dev/core/logger";
|
||||
import type { ZodMessageCatalogSchema } from "@trigger.dev/core/v3/zodMessageHandler";
|
||||
import { ZodMessageHandler } from "@trigger.dev/core/v3/zodMessageHandler";
|
||||
import { Evt } from "evt";
|
||||
import type { z } from "zod";
|
||||
import type { RedisClient, RedisWithClusterOptions } from "~/redis.server";
|
||||
import { createRedisClient } from "~/redis.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { safeJsonParse } from "~/utils/json";
|
||||
|
||||
export type ZodPubSubOptions<TMessageCatalog extends ZodMessageCatalogSchema> = {
|
||||
redis: RedisWithClusterOptions;
|
||||
schema: TMessageCatalog;
|
||||
};
|
||||
|
||||
export interface ZodSubscriber<TMessageCatalog extends ZodMessageCatalogSchema> {
|
||||
on<K extends keyof TMessageCatalog>(
|
||||
eventName: K,
|
||||
listener: (payload: z.infer<TMessageCatalog[K]>) => Promise<void>
|
||||
): void;
|
||||
|
||||
stopListening(): Promise<void>;
|
||||
}
|
||||
|
||||
class RedisZodSubscriber<
|
||||
TMessageCatalog extends ZodMessageCatalogSchema,
|
||||
> implements ZodSubscriber<TMessageCatalog> {
|
||||
private _subscriber: RedisClient;
|
||||
private _listeners: Map<string, (payload: unknown) => Promise<void>> = new Map();
|
||||
private _messageHandler: ZodMessageHandler<TMessageCatalog>;
|
||||
|
||||
public onUnsubscribed: Evt<{
|
||||
pattern: string;
|
||||
}> = new Evt();
|
||||
|
||||
constructor(
|
||||
private readonly _pattern: string,
|
||||
private readonly _options: ZodPubSubOptions<TMessageCatalog>,
|
||||
private readonly _logger: Logger
|
||||
) {
|
||||
this._subscriber = createRedisClient("trigger:zodSubscriber", _options.redis);
|
||||
this._messageHandler = new ZodMessageHandler({
|
||||
schema: _options.schema,
|
||||
logger: this._logger,
|
||||
});
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
await this._subscriber.psubscribe(this._pattern);
|
||||
this._subscriber.on("pmessage", this.#onMessage.bind(this));
|
||||
}
|
||||
|
||||
public on<K extends keyof TMessageCatalog>(
|
||||
eventName: K,
|
||||
listener: (payload: z.infer<TMessageCatalog[K]>) => Promise<void>
|
||||
): void {
|
||||
this._listeners.set(eventName as string, listener);
|
||||
}
|
||||
|
||||
public async stopListening(): Promise<void> {
|
||||
this._listeners.clear();
|
||||
await this._subscriber.punsubscribe();
|
||||
|
||||
this.onUnsubscribed.post({ pattern: this._pattern });
|
||||
|
||||
this._subscriber.quit();
|
||||
}
|
||||
|
||||
async #onMessage(pattern: string, channel: string, serializedMessage: string) {
|
||||
if (pattern !== this._pattern) {
|
||||
return;
|
||||
}
|
||||
|
||||
const parsedMessage = safeJsonParse(serializedMessage);
|
||||
|
||||
if (!parsedMessage) {
|
||||
return;
|
||||
}
|
||||
|
||||
const message = this._messageHandler.parseMessage(parsedMessage);
|
||||
|
||||
if (!message.success) {
|
||||
this._logger.error(`Failed to parse message: ${message.error}`, { parsedMessage });
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof message.data.type !== "string") {
|
||||
this._logger.error(`Failed to parse message: invalid type`, { parsedMessage });
|
||||
return;
|
||||
}
|
||||
|
||||
const listener = this._listeners.get(message.data.type);
|
||||
|
||||
if (!listener) {
|
||||
this._logger.debug(`No listener for message type: ${message.data.type}`, { parsedMessage });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await listener(message.data.payload);
|
||||
} catch (error) {
|
||||
this._logger.error("Error handling message", { error, message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class ZodPubSub<TMessageCatalog extends ZodMessageCatalogSchema> {
|
||||
private _publisher: RedisClient;
|
||||
private _logger = logger.child({ module: "ZodPubSub" });
|
||||
private _subscriberCount = 0;
|
||||
|
||||
get subscriberCount() {
|
||||
return this._subscriberCount;
|
||||
}
|
||||
|
||||
constructor(private _options: ZodPubSubOptions<TMessageCatalog>) {
|
||||
this._publisher = createRedisClient("trigger:zodSubscriber", _options.redis);
|
||||
}
|
||||
|
||||
get redisOptions() {
|
||||
return this._options.redis;
|
||||
}
|
||||
|
||||
public async publish<K extends keyof TMessageCatalog>(
|
||||
channel: string,
|
||||
type: K,
|
||||
payload: z.input<TMessageCatalog[K]>
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this._publisher.publish(channel, JSON.stringify({ type, payload, version: "v1" }));
|
||||
} catch (e) {
|
||||
logger.error("Failed to publish message", { channel, type, payload, error: e });
|
||||
}
|
||||
}
|
||||
|
||||
public async subscribe(channel: string): Promise<ZodSubscriber<TMessageCatalog>> {
|
||||
const subscriber = new RedisZodSubscriber(channel, this._options, this._logger);
|
||||
|
||||
await subscriber.initialize();
|
||||
|
||||
this._subscriberCount++;
|
||||
|
||||
subscriber.onUnsubscribed.attachOnce(({ pattern }) => {
|
||||
logger.debug("Subscriber unsubscribed", { pattern });
|
||||
|
||||
this._subscriberCount--;
|
||||
});
|
||||
|
||||
return subscriber;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user