chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
# MLflow Typescript SDK - Anthropic
|
||||
|
||||
Seamlessly integrate [MLflow Tracing](https://github.com/mlflow/mlflow/tree/main/libs/typescript) with Anthropic to automatically trace your Claude API calls.
|
||||
|
||||
| Package | NPM | Description |
|
||||
| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- |
|
||||
| [@mlflow/anthropic](./) | [](https://www.npmjs.com/package/@mlflow/anthropic) | Auto-instrumentation integration for Anthropic. |
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @mlflow/anthropic
|
||||
```
|
||||
|
||||
The package includes the [`@mlflow/core`](https://github.com/mlflow/mlflow/tree/main/libs/typescript) package and `@anthropic-ai/sdk` package as peer dependencies. Depending on your package manager, you may need to install these two packages separately.
|
||||
|
||||
## Quickstart
|
||||
|
||||
Start MLflow Tracking Server if you don't have one already:
|
||||
|
||||
```bash
|
||||
pip install mlflow
|
||||
mlflow server --backend-store-uri sqlite:///mlruns.db --port 5000
|
||||
```
|
||||
|
||||
Self-hosting MLflow server requires Python 3.10 or higher. If you don't have one, you can also use [managed MLflow service](https://mlflow.org/#get-started) for free to get started quickly.
|
||||
|
||||
Instantiate MLflow SDK in your application:
|
||||
|
||||
```typescript
|
||||
import * as mlflow from '@mlflow/core';
|
||||
|
||||
mlflow.init({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '<experiment-id>',
|
||||
});
|
||||
```
|
||||
|
||||
Create a trace for Anthropic Claude:
|
||||
|
||||
```typescript
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { tracedAnthropic } from '@mlflow/anthropic';
|
||||
|
||||
const anthropic = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
|
||||
const client = tracedAnthropic(anthropic);
|
||||
|
||||
const response = await client.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello Claude' }],
|
||||
});
|
||||
```
|
||||
|
||||
View traces in MLflow UI:
|
||||
|
||||

|
||||
|
||||
## Documentation 📘
|
||||
|
||||
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart).
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
|
||||
@@ -0,0 +1,16 @@
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
roots: ['<rootDir>/tests', '<rootDir>/src'],
|
||||
testMatch: ['**/*.test.ts'],
|
||||
moduleFileExtensions: ['ts', 'js', 'json', 'node'],
|
||||
transform: {
|
||||
'^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
|
||||
},
|
||||
globalSetup: '<rootDir>/../../jest.global-server-setup.ts',
|
||||
globalTeardown: '<rootDir>/../../jest.global-server-teardown.ts',
|
||||
testTimeout: 30000,
|
||||
forceExit: true,
|
||||
detectOpenHandles: true,
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"name": "@mlflow/anthropic",
|
||||
"version": "0.3.0",
|
||||
"description": "Anthropic integration package for MLflow Tracing",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mlflow/mlflow.git"
|
||||
},
|
||||
"homepage": "https://mlflow.org/",
|
||||
"author": {
|
||||
"name": "MLflow",
|
||||
"url": "https://mlflow.org/"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/mlflow/mlflow/issues"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"keywords": [
|
||||
"mlflow",
|
||||
"tracing",
|
||||
"observability",
|
||||
"opentelemetry",
|
||||
"llm",
|
||||
"anthropic",
|
||||
"claude",
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"prepublishOnly": "npm run build",
|
||||
"test": "jest",
|
||||
"lint": "eslint . --ext .ts --max-warnings 0",
|
||||
"lint:fix": "eslint . --ext .ts --fix",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@mlflow/core": "^0.3.0",
|
||||
"@anthropic-ai/sdk": "^0.71.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"jest": "^29.6.2",
|
||||
"typescript": "^5.8.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"files": [
|
||||
"dist/"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
/**
|
||||
* Main tracedAnthropic wrapper function for MLflow tracing integration
|
||||
*/
|
||||
|
||||
import {
|
||||
startSpan,
|
||||
getCurrentActiveSpan,
|
||||
SpanAttributeKey,
|
||||
SpanType,
|
||||
SpanStatusCode,
|
||||
TokenUsage,
|
||||
LiveSpan,
|
||||
withSpan,
|
||||
} from '@mlflow/core';
|
||||
|
||||
const SUPPORTED_MODULES = ['Messages'];
|
||||
const SUPPORTED_METHODS = ['create', 'stream'];
|
||||
|
||||
/**
|
||||
* Create a traced version of Anthropic client with MLflow tracing
|
||||
* @param anthropicClient - The Anthropic client instance to trace
|
||||
* @returns Traced Anthropic client with tracing capabilities
|
||||
*/
|
||||
export function tracedAnthropic<T = any>(anthropicClient: T): T {
|
||||
const tracedClient = new Proxy(anthropicClient as any, {
|
||||
get(target, prop, receiver) {
|
||||
const original = Reflect.get(target, prop, receiver);
|
||||
const moduleName = (target as object).constructor?.name;
|
||||
|
||||
if (typeof original === 'function') {
|
||||
if (shouldTraceMethod(moduleName, String(prop))) {
|
||||
const methodName = String(prop);
|
||||
if (methodName === 'stream') {
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return wrapStreamWithTracing(original as Function, moduleName) as T;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return wrapWithTracing(original as Function, moduleName) as T;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return (original as Function).bind(target) as T;
|
||||
}
|
||||
|
||||
if (
|
||||
original &&
|
||||
!Array.isArray(original) &&
|
||||
!(original instanceof Date) &&
|
||||
typeof original === 'object'
|
||||
) {
|
||||
return tracedAnthropic(original) as T;
|
||||
}
|
||||
|
||||
return original as T;
|
||||
},
|
||||
});
|
||||
|
||||
return tracedClient as T;
|
||||
}
|
||||
|
||||
function shouldTraceMethod(moduleName: string | undefined, methodName: string): boolean {
|
||||
if (!moduleName) {
|
||||
return false;
|
||||
}
|
||||
return SUPPORTED_MODULES.includes(moduleName) && SUPPORTED_METHODS.includes(methodName);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
function wrapWithTracing(fn: Function, moduleName: string): Function {
|
||||
const spanType = getSpanType(moduleName);
|
||||
const name = moduleName;
|
||||
|
||||
return function (this: any, ...args: any[]) {
|
||||
if (!spanType) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return fn.apply(this, args);
|
||||
}
|
||||
|
||||
// Skip tracing for create() calls with stream: true
|
||||
// The Anthropic SDK's stream() method internally calls create() with stream: true,
|
||||
// and we don't want to create a duplicate trace - the streaming wrapper handles it
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (args[0]?.stream === true) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return fn.apply(this, args);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return withSpan(
|
||||
async (span: LiveSpan) => {
|
||||
span.setInputs(args[0]);
|
||||
|
||||
const result = await fn.apply(this, args);
|
||||
|
||||
span.setOutputs(result);
|
||||
|
||||
try {
|
||||
const usage = extractTokenUsage(result);
|
||||
if (usage) {
|
||||
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
|
||||
}
|
||||
} catch (error) {
|
||||
console.debug('Error extracting token usage', error);
|
||||
}
|
||||
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return result;
|
||||
},
|
||||
{ name, spanType },
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
function getSpanType(moduleName: string): SpanType | undefined {
|
||||
switch (moduleName) {
|
||||
case 'Messages':
|
||||
return SpanType.LLM;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
function wrapStreamWithTracing(fn: Function, moduleName: string): Function {
|
||||
const spanType = getSpanType(moduleName);
|
||||
const name = moduleName;
|
||||
|
||||
return function (this: any, ...args: any[]) {
|
||||
if (!spanType) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return fn.apply(this, args);
|
||||
}
|
||||
|
||||
// Create the stream (synchronous call)
|
||||
const stream = fn.apply(this, args);
|
||||
|
||||
// Return a proxy that wraps the stream with tracing
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return wrapMessageStream(stream, args[0], name, spanType);
|
||||
};
|
||||
}
|
||||
|
||||
function wrapMessageStream(stream: any, inputs: any, name: string, spanType: SpanType): any {
|
||||
// Use a flag that is set synchronously on first access to prevent duplicate spans.
|
||||
// It is claimed either when the async iterator getter is invoked or when the wrapped
|
||||
// finalMessage function is called, before any asynchronous work begins, so only one
|
||||
// of these access paths will perform tracing for a given stream instance.
|
||||
let tracingClaimed = false;
|
||||
|
||||
return new Proxy(stream, {
|
||||
get(target, prop, receiver) {
|
||||
const original = Reflect.get(target, prop, receiver);
|
||||
|
||||
// Wrap finalMessage() to add tracing
|
||||
if (prop === 'finalMessage') {
|
||||
return async function () {
|
||||
if (tracingClaimed) {
|
||||
// Already traced via async iteration, just return the message
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
|
||||
return await target.finalMessage();
|
||||
}
|
||||
tracingClaimed = true;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return withSpan(
|
||||
async (span: LiveSpan) => {
|
||||
span.setInputs(inputs);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
|
||||
const message = await target.finalMessage();
|
||||
|
||||
span.setOutputs(message);
|
||||
|
||||
try {
|
||||
const usage = extractTokenUsage(message);
|
||||
if (usage) {
|
||||
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
|
||||
}
|
||||
} catch (error) {
|
||||
console.debug('Error extracting token usage', error);
|
||||
}
|
||||
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return message;
|
||||
},
|
||||
{ name, spanType },
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
// Wrap async iterator for `for await (const event of stream)` pattern
|
||||
if (prop === Symbol.asyncIterator) {
|
||||
return function () {
|
||||
if (tracingClaimed) {
|
||||
// In practice, MessageStreams are typically consumed once and iterating again would
|
||||
// yield no events, so this may not be a real issue but for completeness we return the
|
||||
// unwrapped iterator in this case.
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-call
|
||||
return target[Symbol.asyncIterator]();
|
||||
}
|
||||
tracingClaimed = true;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument
|
||||
return wrapAsyncIterator(target[Symbol.asyncIterator](), target, inputs, name, spanType);
|
||||
};
|
||||
}
|
||||
|
||||
if (typeof original === 'function') {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
|
||||
return original.bind(target);
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return original;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function* wrapAsyncIterator(
|
||||
iterator: AsyncIterator<any>,
|
||||
stream: any,
|
||||
inputs: any,
|
||||
name: string,
|
||||
spanType: SpanType,
|
||||
): AsyncGenerator<any> {
|
||||
// Use startSpan for manual lifecycle management since withSpan doesn't support async generators
|
||||
const parentSpan = getCurrentActiveSpan();
|
||||
const span = startSpan({ name, spanType, parent: parentSpan ?? undefined });
|
||||
span.setInputs(inputs);
|
||||
|
||||
let iterationError: Error | undefined;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { value, done } = await iterator.next();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
yield value;
|
||||
}
|
||||
} catch (error) {
|
||||
iterationError = error as Error;
|
||||
throw error;
|
||||
} finally {
|
||||
if (iterationError) {
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
|
||||
span.setStatus(SpanStatusCode.ERROR, iterationError.message);
|
||||
span.end();
|
||||
} else {
|
||||
// After iteration completes (or early termination via break/return),
|
||||
// get the final message for outputs and token usage
|
||||
try {
|
||||
// Prefer a proxy-aware finalMessage if available on the iterator, and fall back to the stream.
|
||||
let finalMessage: unknown;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
|
||||
const iteratorAny = iterator as any;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (iteratorAny && typeof iteratorAny.finalMessage === 'function') {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
|
||||
finalMessage = await iteratorAny.finalMessage();
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
} else if (stream && typeof stream.finalMessage === 'function') {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
|
||||
finalMessage = await stream.finalMessage();
|
||||
}
|
||||
|
||||
if (finalMessage !== undefined) {
|
||||
span.setOutputs(finalMessage);
|
||||
|
||||
const usage = extractTokenUsage(finalMessage);
|
||||
if (usage) {
|
||||
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Stream may have completed without finalMessage available
|
||||
console.debug('Could not get final message from stream', e);
|
||||
span.setAttribute('mlflow.tracing.token_usage_capture_failed', true);
|
||||
span.setAttribute(
|
||||
'mlflow.tracing.token_usage_capture_error',
|
||||
e instanceof Error ? e.message : String(e),
|
||||
);
|
||||
}
|
||||
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'anthropic');
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function extractTokenUsage(response: any): TokenUsage | undefined {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const usage = response?.usage;
|
||||
if (!usage) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const inputTokens = usage.input_tokens;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const outputTokens = usage.output_tokens;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const totalTokens = usage.total_tokens ?? (inputTokens ?? 0) + (outputTokens ?? 0);
|
||||
|
||||
const result: TokenUsage = {
|
||||
input_tokens: inputTokens ?? 0,
|
||||
output_tokens: outputTokens ?? 0,
|
||||
total_tokens: totalTokens ?? 0,
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (usage.cache_read_input_tokens != null) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
result.cache_read_input_tokens = usage.cache_read_input_tokens;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (usage.cache_creation_input_tokens != null) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
result.cache_creation_input_tokens = usage.cache_creation_input_tokens;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,429 @@
|
||||
/**
|
||||
* Tests for MLflow Anthropic integration with MSW mock server
|
||||
*/
|
||||
|
||||
import * as mlflow from '@mlflow/core';
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { tracedAnthropic } from '../src';
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { anthropicMockHandlers, createStreamingErrorHandler } from './mockAnthropicServer';
|
||||
import { createAuthProvider } from '@mlflow/core/src/auth';
|
||||
|
||||
const TEST_TRACKING_URI = 'http://localhost:5000';
|
||||
|
||||
describe('tracedAnthropic', () => {
|
||||
let experimentId: string;
|
||||
let client: mlflow.MlflowClient;
|
||||
let server: ReturnType<typeof setupServer>;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = setupServer(...anthropicMockHandlers);
|
||||
server.listen();
|
||||
|
||||
const authProvider = createAuthProvider({ trackingUri: TEST_TRACKING_URI });
|
||||
client = new mlflow.MlflowClient({ trackingUri: TEST_TRACKING_URI, authProvider });
|
||||
|
||||
const experimentName = `anthropic-test-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
||||
experimentId = await client.createExperiment(experimentName);
|
||||
|
||||
mlflow.init({
|
||||
trackingUri: TEST_TRACKING_URI,
|
||||
experimentId,
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
server.close();
|
||||
await client.deleteExperiment(experimentId);
|
||||
});
|
||||
|
||||
const getLastActiveTrace = async (): Promise<mlflow.Trace> => {
|
||||
await mlflow.flushTraces();
|
||||
const traceId = mlflow.getLastActiveTraceId();
|
||||
const trace = await client.getTrace(traceId!);
|
||||
return trace;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
server.resetHandlers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should trace messages.create()', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const result = await wrappedAnthropic.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello Claude!' }],
|
||||
});
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
const tokenUsage = trace.info.tokenUsage;
|
||||
expect(tokenUsage).toBeDefined();
|
||||
expect(typeof tokenUsage?.input_tokens).toBe('number');
|
||||
expect(typeof tokenUsage?.output_tokens).toBe('number');
|
||||
expect(typeof tokenUsage?.total_tokens).toBe('number');
|
||||
expect(tokenUsage?.cache_read_input_tokens).toBe(64);
|
||||
expect(tokenUsage?.cache_creation_input_tokens).toBe(32);
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.name).toBe('Messages');
|
||||
expect(span.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(span.logLevel).toBe(mlflow.SpanLogLevel.INFO);
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello Claude!' }],
|
||||
});
|
||||
expect(span.outputs).toEqual(result);
|
||||
|
||||
const spanTokenUsage = span.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
|
||||
expect(spanTokenUsage).toBeDefined();
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.TOTAL_TOKENS]).toBe('number');
|
||||
|
||||
const messageFormat = span.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
|
||||
it('should handle Anthropic errors properly', async () => {
|
||||
server.use(
|
||||
http.post('https://api.anthropic.com/v1/messages', () =>
|
||||
HttpResponse.json(
|
||||
{
|
||||
error: {
|
||||
type: 'rate_limit',
|
||||
message: 'Rate limit exceeded',
|
||||
},
|
||||
},
|
||||
{ status: 429 },
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// Disable retries to prevent the SDK from retrying on errors.
|
||||
// SDK 0.50+ calls CancelReadableStream on error responses before retrying,
|
||||
// which hangs indefinitely with MSW mock responses.
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key', maxRetries: 0 });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
await expect(
|
||||
wrappedAnthropic.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'This request should fail.' }],
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('ERROR');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'This request should fail.' }],
|
||||
});
|
||||
expect(span.outputs).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should trace Anthropic request wrapped in a parent span', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const result = await mlflow.withSpan(
|
||||
async (_span) => {
|
||||
const response = await wrappedAnthropic.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'Hello from parent span.' }],
|
||||
});
|
||||
return response.content[0];
|
||||
},
|
||||
{
|
||||
name: 'predict',
|
||||
spanType: mlflow.SpanType.CHAIN,
|
||||
inputs: 'Hello from parent span.',
|
||||
},
|
||||
);
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
expect(trace.data.spans.length).toBe(2);
|
||||
|
||||
const parentSpan = trace.data.spans[0];
|
||||
expect(parentSpan.name).toBe('predict');
|
||||
expect(parentSpan.spanType).toBe(mlflow.SpanType.CHAIN);
|
||||
expect(parentSpan.outputs).toEqual(result);
|
||||
|
||||
const childSpan = trace.data.spans[1];
|
||||
expect(childSpan.name).toBe('Messages');
|
||||
expect(childSpan.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(childSpan.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'Hello from parent span.' }],
|
||||
});
|
||||
expect(childSpan.outputs).toBeDefined();
|
||||
|
||||
const messageFormat = childSpan.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
|
||||
it('should trace messages.stream() with async iteration', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const events: any[] = [];
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello streaming!' }],
|
||||
});
|
||||
|
||||
for await (const event of stream) {
|
||||
events.push(event);
|
||||
}
|
||||
|
||||
expect(events.length).toBeGreaterThan(0);
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.name).toBe('Messages');
|
||||
expect(span.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(span.inputs).toMatchObject({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello streaming!' }],
|
||||
});
|
||||
|
||||
// Verify outputs contain actual message content, not a stream object
|
||||
expect(span.outputs).toBeDefined();
|
||||
expect(span.outputs).toHaveProperty('id');
|
||||
expect(span.outputs).toHaveProperty('type', 'message');
|
||||
expect(span.outputs).toHaveProperty('content');
|
||||
expect(span.outputs.content[0]).toHaveProperty('type', 'text');
|
||||
|
||||
// Verify token usage is captured for streaming
|
||||
const spanTokenUsage = span.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
|
||||
expect(spanTokenUsage).toBeDefined();
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe('number');
|
||||
|
||||
const messageFormat = span.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
|
||||
it('should trace messages.stream() with finalMessage()', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello finalMessage!' }],
|
||||
});
|
||||
|
||||
const finalMessage = await stream.finalMessage();
|
||||
|
||||
expect(finalMessage).toBeDefined();
|
||||
expect(finalMessage.content).toBeDefined();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.name).toBe('Messages');
|
||||
expect(span.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello finalMessage!' }],
|
||||
});
|
||||
expect(span.outputs).toEqual(finalMessage);
|
||||
|
||||
const messageFormat = span.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
|
||||
it('should handle errors during streaming async iteration', async () => {
|
||||
server.use(createStreamingErrorHandler());
|
||||
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key', maxRetries: 0 });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello streaming error!' }],
|
||||
});
|
||||
|
||||
await expect(async () => {
|
||||
for await (const _event of stream) {
|
||||
// consume events until error
|
||||
}
|
||||
}).rejects.toThrow();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('ERROR');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
|
||||
expect(span.inputs).toMatchObject({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello streaming error!' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle errors during streaming finalMessage()', async () => {
|
||||
server.use(createStreamingErrorHandler());
|
||||
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key', maxRetries: 0 });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello finalMessage error!' }],
|
||||
});
|
||||
|
||||
await expect(stream.finalMessage()).rejects.toThrow();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('ERROR');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello finalMessage error!' }],
|
||||
});
|
||||
});
|
||||
|
||||
it('should create only one span when async iteration is followed by finalMessage()', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello dedup!' }],
|
||||
});
|
||||
|
||||
// First consume via async iteration (claims tracing)
|
||||
for await (const _event of stream) {
|
||||
// consume all events
|
||||
}
|
||||
|
||||
// Then call finalMessage() on the same stream — should NOT create a second span
|
||||
const finalMessage = await stream.finalMessage();
|
||||
expect(finalMessage).toBeDefined();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
// Only one LLM span should exist (no duplicate from finalMessage)
|
||||
const llmSpans = trace.data.spans.filter((s) => s.spanType === mlflow.SpanType.LLM);
|
||||
expect(llmSpans.length).toBe(1);
|
||||
});
|
||||
|
||||
it('should not trace messages.create() when stream: true is passed directly', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
// Calling create() with stream: true is skipped by the tracing wrapper
|
||||
// because the SDK's stream() method internally calls create({ stream: true }),
|
||||
// and tracing is handled by the stream wrapper instead.
|
||||
// Users should use messages.stream() for traced streaming.
|
||||
const response = await wrappedAnthropic.messages.create({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 256,
|
||||
messages: [{ role: 'user', content: 'Hello direct stream!' }],
|
||||
stream: true,
|
||||
});
|
||||
|
||||
// The SDK returns a Stream object when stream: true
|
||||
expect(response).toBeDefined();
|
||||
|
||||
// Consume the stream to completion
|
||||
for await (const _event of response as any) {
|
||||
// consume
|
||||
}
|
||||
|
||||
// No traced span should have been created by the create() wrapper
|
||||
await mlflow.flushTraces();
|
||||
const traceId = mlflow.getLastActiveTraceId();
|
||||
// The last active trace should be from a previous test, not this call,
|
||||
// OR if it is from this call, it should not have a Messages LLM span
|
||||
// created by wrapWithTracing (since create with stream:true is skipped).
|
||||
if (traceId) {
|
||||
const trace = await client.getTrace(traceId);
|
||||
const matchingSpans = trace.data.spans.filter(
|
||||
(s) =>
|
||||
s.spanType === mlflow.SpanType.LLM &&
|
||||
s.inputs?.messages?.[0]?.content === 'Hello direct stream!',
|
||||
);
|
||||
expect(matchingSpans.length).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('should trace streaming request wrapped in a parent span', async () => {
|
||||
const anthropic = new Anthropic({ apiKey: 'test-key' });
|
||||
const wrappedAnthropic = tracedAnthropic(anthropic);
|
||||
|
||||
const result = await mlflow.withSpan(
|
||||
async (_span) => {
|
||||
const stream = wrappedAnthropic.messages.stream({
|
||||
model: 'claude-3-7-sonnet-20250219',
|
||||
max_tokens: 128,
|
||||
messages: [{ role: 'user', content: 'Hello streaming parent!' }],
|
||||
});
|
||||
|
||||
const message = await stream.finalMessage();
|
||||
return message.content[0];
|
||||
},
|
||||
{
|
||||
name: 'streaming-predict',
|
||||
spanType: mlflow.SpanType.CHAIN,
|
||||
inputs: 'Hello streaming parent!',
|
||||
},
|
||||
);
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
// At least 2 spans: parent chain span and child LLM span
|
||||
expect(trace.data.spans.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const parentSpan = trace.data.spans[0];
|
||||
expect(parentSpan.name).toBe('streaming-predict');
|
||||
expect(parentSpan.spanType).toBe(mlflow.SpanType.CHAIN);
|
||||
expect(parentSpan.outputs).toEqual(result);
|
||||
|
||||
// Find child spans with LLM type
|
||||
const llmSpans = trace.data.spans.filter((s) => s.spanType === mlflow.SpanType.LLM);
|
||||
expect(llmSpans.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const llmSpan = llmSpans[0];
|
||||
expect(llmSpan.outputs).toBeDefined();
|
||||
|
||||
const messageFormat = llmSpan.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('anthropic');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* MSW-based Anthropic API mock server for testing
|
||||
*/
|
||||
|
||||
import { http, HttpResponse } from 'msw';
|
||||
|
||||
interface MessagesCreateRequest {
|
||||
model: string;
|
||||
messages: Array<{
|
||||
role: string;
|
||||
content: string | Array<{ type: string; text: string }>;
|
||||
}>;
|
||||
max_tokens?: number;
|
||||
system?: string | Array<{ type: string; text: string }>;
|
||||
stream?: boolean;
|
||||
}
|
||||
|
||||
function createMessageResponse(request: MessagesCreateRequest) {
|
||||
return {
|
||||
id: `msg_${Math.random().toString(36).slice(2)}`,
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: request.model,
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
usage: {
|
||||
input_tokens: 128,
|
||||
output_tokens: 256,
|
||||
total_tokens: 384,
|
||||
cache_read_input_tokens: 64,
|
||||
cache_creation_input_tokens: 32,
|
||||
},
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'This is a mocked Anthropic response.',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function createStreamingEvents(request: MessagesCreateRequest): string[] {
|
||||
const messageId = `msg_${Math.random().toString(36).slice(2)}`;
|
||||
const responseText = 'This is a mocked streaming response.';
|
||||
|
||||
return [
|
||||
`event: message_start\ndata: ${JSON.stringify({
|
||||
type: 'message_start',
|
||||
message: {
|
||||
id: messageId,
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: request.model,
|
||||
content: [],
|
||||
stop_reason: null,
|
||||
stop_sequence: null,
|
||||
usage: {
|
||||
input_tokens: 128,
|
||||
output_tokens: 0,
|
||||
},
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: content_block_start\ndata: ${JSON.stringify({
|
||||
type: 'content_block_start',
|
||||
index: 0,
|
||||
content_block: {
|
||||
type: 'text',
|
||||
text: '',
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: content_block_delta\ndata: ${JSON.stringify({
|
||||
type: 'content_block_delta',
|
||||
index: 0,
|
||||
delta: {
|
||||
type: 'text_delta',
|
||||
text: responseText,
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: content_block_stop\ndata: ${JSON.stringify({
|
||||
type: 'content_block_stop',
|
||||
index: 0,
|
||||
})}\n\n`,
|
||||
`event: message_delta\ndata: ${JSON.stringify({
|
||||
type: 'message_delta',
|
||||
delta: {
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
},
|
||||
usage: {
|
||||
output_tokens: 256,
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: message_stop\ndata: ${JSON.stringify({
|
||||
type: 'message_stop',
|
||||
})}\n\n`,
|
||||
];
|
||||
}
|
||||
|
||||
export function createStreamingErrorHandler() {
|
||||
return http.post('https://api.anthropic.com/v1/messages', async ({ request }) => {
|
||||
const body = (await request.json()) as MessagesCreateRequest;
|
||||
|
||||
if (body.stream) {
|
||||
// Send a partial stream that ends with an error event
|
||||
const messageId = `msg_${Math.random().toString(36).slice(2)}`;
|
||||
const events = [
|
||||
`event: message_start\ndata: ${JSON.stringify({
|
||||
type: 'message_start',
|
||||
message: {
|
||||
id: messageId,
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: body.model,
|
||||
content: [],
|
||||
stop_reason: null,
|
||||
stop_sequence: null,
|
||||
usage: { input_tokens: 128, output_tokens: 0 },
|
||||
},
|
||||
})}\n\n`,
|
||||
`event: error\ndata: ${JSON.stringify({
|
||||
type: 'error',
|
||||
error: {
|
||||
type: 'overloaded_error',
|
||||
message: 'Overloaded',
|
||||
},
|
||||
})}\n\n`,
|
||||
];
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
for (const event of events) {
|
||||
controller.enqueue(encoder.encode(event));
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
return new HttpResponse(stream, {
|
||||
headers: {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return HttpResponse.json(
|
||||
{ error: { type: 'overloaded_error', message: 'Overloaded' } },
|
||||
{ status: 529 },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export const anthropicMockHandlers = [
|
||||
http.post('https://api.anthropic.com/v1/messages', async ({ request }) => {
|
||||
const body = (await request.json()) as MessagesCreateRequest;
|
||||
|
||||
// Check if streaming is requested
|
||||
if (body.stream) {
|
||||
const events = createStreamingEvents(body);
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
for (const event of events) {
|
||||
controller.enqueue(encoder.encode(event));
|
||||
// Small delay to simulate streaming
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
return new HttpResponse(stream, {
|
||||
headers: {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return HttpResponse.json(createMessageResponse(body));
|
||||
}),
|
||||
];
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"include": ["src/**/*"],
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "mlflow-tracing",
|
||||
"version": "0.2.0",
|
||||
"description": "Observability plugin for Claude Code with MLflow tracing",
|
||||
"author": {
|
||||
"name": "MLflow",
|
||||
"url": "https://mlflow.org/"
|
||||
},
|
||||
"homepage": "https://mlflow.org/",
|
||||
"license": "Apache-2.0",
|
||||
"keywords": ["mlflow", "tracing", "observability", "llm"],
|
||||
"skills": "./skills/"
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
# MLflow Typescript SDK - Claude Code
|
||||
|
||||
Trace [Claude Code](https://docs.claude.com/en/docs/claude-code/overview) sessions and [Claude Agent SDK](https://docs.claude.com/en/api/agent-sdk/typescript) runs with [MLflow Tracing](https://github.com/mlflow/mlflow/tree/main/libs/typescript).
|
||||
|
||||
| Package | NPM | Description |
|
||||
| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ |
|
||||
| [@mlflow/claude-code](./) | [](https://www.npmjs.com/package/@mlflow/claude-code) | Trace Claude Code CLI sessions (Stop-hook) and Claude Agent SDK queries. |
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @mlflow/claude-code
|
||||
```
|
||||
|
||||
For Claude Agent SDK tracing, also install the SDK:
|
||||
|
||||
```bash
|
||||
npm install @anthropic-ai/claude-agent-sdk
|
||||
```
|
||||
|
||||
## Quickstart
|
||||
|
||||
Start an MLflow Tracking Server if you don't have one already, then point your environment at it:
|
||||
|
||||
```bash
|
||||
export MLFLOW_TRACKING_URI=http://localhost:5000
|
||||
export MLFLOW_EXPERIMENT_ID=<experiment-id>
|
||||
```
|
||||
|
||||
### Tracing the Claude Code CLI
|
||||
|
||||
Install the bundled Stop-hook plugin once; it will trace every CLI session automatically. See the [docs](https://mlflow.org/docs/latest/genai/tracing/integrations/claude-code) for plugin setup.
|
||||
|
||||
### Tracing the Claude Agent SDK
|
||||
|
||||
Wrap the SDK's `query` function with `createTracedQuery`:
|
||||
|
||||
```typescript
|
||||
import { query } from '@anthropic-ai/claude-agent-sdk';
|
||||
import { createTracedQuery } from '@mlflow/claude-code';
|
||||
import * as mlflow from '@mlflow/core';
|
||||
|
||||
mlflow.init({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '<experiment-id>',
|
||||
});
|
||||
|
||||
const tracedQuery = createTracedQuery(query);
|
||||
|
||||
const result = tracedQuery({
|
||||
prompt: 'List the files in this directory',
|
||||
options: {
|
||||
permissionMode: 'bypassPermissions',
|
||||
},
|
||||
});
|
||||
|
||||
for await (const message of result) {
|
||||
if (message.type === 'result') {
|
||||
console.log('Result:', message.result);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The wrapper produces the same span tree as the Claude Code CLI integration: an `AGENT` root span with `LLM` and `TOOL` children, plus nested `AGENT` spans for sub-agents invoked via the Task tool. Token usage (including `cache_read_input_tokens` and `cache_creation_input_tokens`) is recorded on every LLM span and aggregated on the root.
|
||||
|
||||
## Documentation
|
||||
|
||||
Official documentation for the MLflow TypeScript SDK is at https://mlflow.org/docs/latest/genai/tracing/quickstart.
|
||||
@@ -0,0 +1,31 @@
|
||||
import { build } from 'esbuild';
|
||||
import { chmodSync } from 'node:fs';
|
||||
|
||||
const banner = { js: '#!/usr/bin/env node' };
|
||||
|
||||
// @mlflow/core's WAL supervisor resolves the daemon bundle at runtime
|
||||
// via `require.resolve('@mlflow/core/package.json')` (string-concatenated
|
||||
// to defeat static analysis) and then `path.join(..., 'bundle',
|
||||
// 'daemon.cjs')`. Marking both paths as `external` makes the intent
|
||||
// explicit and guarantees esbuild leaves any future references to them
|
||||
// alone — the daemon ships as its own binary inside @mlflow/core and
|
||||
// must be loaded from the installed package on disk, not inlined into
|
||||
// the hook bundle.
|
||||
const external = ['node:*', '@mlflow/core/package.json', '@mlflow/core/bundle/daemon.cjs'];
|
||||
|
||||
for (const [entryPoint, outfile] of [
|
||||
['dist/hooks/stop.js', 'bundle/stop.cjs'],
|
||||
['dist/cli.js', 'bundle/cli.cjs'],
|
||||
]) {
|
||||
await build({
|
||||
entryPoints: [entryPoint],
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'cjs',
|
||||
outfile,
|
||||
external,
|
||||
banner,
|
||||
});
|
||||
|
||||
chmodSync(outfile, 0o755);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"hooks": {
|
||||
"Stop": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/bundle/stop.cjs\"",
|
||||
"timeout": 120
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
const path = require('path');
|
||||
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
roots: ['<rootDir>/tests'],
|
||||
testMatch: ['**/*.test.ts'],
|
||||
moduleFileExtensions: ['ts', 'js', 'json', 'node'],
|
||||
modulePaths: [path.resolve(__dirname, '../../node_modules')],
|
||||
// Integration tests require a real MLflow server at MLFLOW_TRACKING_URI
|
||||
// (default http://localhost:5000). In CI, the typescript-sdk job starts one
|
||||
// via the shared global setup; locally you can point at any running server.
|
||||
globalSetup: path.resolve(__dirname, '../../jest.global-server-setup.ts'),
|
||||
globalTeardown: path.resolve(__dirname, '../../jest.global-server-teardown.ts'),
|
||||
transform: {
|
||||
'^.+\\.tsx?$': [
|
||||
'ts-jest',
|
||||
{
|
||||
tsconfig: {
|
||||
target: 'ES2022',
|
||||
module: 'CommonJS',
|
||||
moduleResolution: 'Node',
|
||||
esModuleInterop: true,
|
||||
strict: true,
|
||||
skipLibCheck: true,
|
||||
types: ['jest', 'node'],
|
||||
baseUrl: '.',
|
||||
paths: {
|
||||
'@mlflow/core': ['../../core/src/index.ts'],
|
||||
'@mlflow/core/*': ['../../core/src/*'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
moduleNameMapper: {
|
||||
'^@mlflow/core$': '<rootDir>/../../core/src',
|
||||
'^@mlflow/core/(.*)$': '<rootDir>/../../core/src/$1',
|
||||
// Strip .js extensions for ESM → CJS test resolution
|
||||
'^(\\.{1,2}/.*)\\.js$': '$1',
|
||||
},
|
||||
testTimeout: 30000,
|
||||
forceExit: true,
|
||||
detectOpenHandles: true,
|
||||
};
|
||||
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"name": "@mlflow/claude-code",
|
||||
"version": "0.3.0",
|
||||
"description": "Claude Code and Claude Agent SDK integration package for MLflow Tracing",
|
||||
"type": "module",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mlflow/mlflow.git"
|
||||
},
|
||||
"homepage": "https://mlflow.org/",
|
||||
"author": {
|
||||
"name": "MLflow",
|
||||
"url": "https://mlflow.org/"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/mlflow/mlflow/issues"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"keywords": [
|
||||
"mlflow",
|
||||
"tracing",
|
||||
"observability",
|
||||
"claude-code",
|
||||
"claude",
|
||||
"claude-agent-sdk",
|
||||
"anthropic",
|
||||
"llm",
|
||||
"agent",
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"bin": {
|
||||
"mlflow-claude-code": "./bundle/cli.cjs"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc && npm run build:bundle",
|
||||
"build:bundle": "node esbuild.config.mjs",
|
||||
"test": "jest --config jest.config.cjs",
|
||||
"lint": "eslint src --ext .ts",
|
||||
"lint:fix": "eslint src --ext .ts --fix",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlflow/core": "^0.3.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.2.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@anthropic-ai/claude-agent-sdk": {
|
||||
"optional": true
|
||||
}
|
||||
},
|
||||
"devDependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.2.0",
|
||||
"@types/jest": "^29.5.3",
|
||||
"esbuild": "^0.25.0",
|
||||
"jest": "^29.6.2",
|
||||
"ts-jest": "^29.1.1",
|
||||
"typescript": "^5.8.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"files": [
|
||||
"dist/",
|
||||
"bundle/",
|
||||
"hooks/",
|
||||
"skills/",
|
||||
".claude-plugin/"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
name: setup
|
||||
description: Configure MLflow tracing for Claude Code.
|
||||
disable-model-invocation: true
|
||||
---
|
||||
|
||||
# MLflow Tracing Setup
|
||||
|
||||
Run this skill ONLY when the user explicitly asks to configure MLflow tracing.
|
||||
|
||||
1. Run `mlflow-claude-code setup --help` and read the available options.
|
||||
2. Ask the user for each required value. Do not pick defaults silently. All values come from the user.
|
||||
3. Run `mlflow-claude-code setup` with the collected options.
|
||||
4. Echo the CLI output. Briefly state the settings file path, tracking URI, experiment, and that tracing is enabled for the next Claude conversation.
|
||||
@@ -0,0 +1,15 @@
|
||||
---
|
||||
name: status
|
||||
description: Show the current MLflow tracing configuration for Claude Code.
|
||||
disable-model-invocation: true
|
||||
---
|
||||
|
||||
# MLflow Tracing Status
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
mlflow-claude-code status
|
||||
```
|
||||
|
||||
Summarize the effective configuration for the user, including whether tracing is enabled, where the config came from, and which tracking URI / experiment settings are active.
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* Internal helpers shared between the CLI transcript path (tracing.ts) and the
|
||||
* live SDK-stream path (liveTracing.ts). Not part of the package's public API:
|
||||
* the leading underscore signals "do not import from `@mlflow/claude-code`".
|
||||
*/
|
||||
|
||||
import { TokenUsageKey } from '@mlflow/core';
|
||||
|
||||
import type { ContentBlock, TokenUsage } from './types.js';
|
||||
|
||||
// ============================================================================
|
||||
// Shared constants
|
||||
// ============================================================================
|
||||
|
||||
export const MAX_PREVIEW_LENGTH = 1000;
|
||||
|
||||
export const METADATA_KEY_CLAUDE_CODE_VERSION = 'mlflow.claude_code_version';
|
||||
export const METADATA_KEY_WORKING_DIRECTORY = 'mlflow.trace.working_directory';
|
||||
export const METADATA_KEY_PERMISSION_MODE = 'mlflow.trace.permission_mode';
|
||||
|
||||
// ============================================================================
|
||||
// Content / token helpers
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Separate text content from tool_use blocks in an assistant response.
|
||||
*/
|
||||
export function extractContentAndTools(
|
||||
content: string | ContentBlock[],
|
||||
): [string, Array<{ type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }>] {
|
||||
let textContent = '';
|
||||
const toolUses: Array<{
|
||||
type: 'tool_use';
|
||||
id: string;
|
||||
name: string;
|
||||
input: Record<string, unknown>;
|
||||
}> = [];
|
||||
|
||||
if (!Array.isArray(content)) {
|
||||
return [typeof content === 'string' ? content : '', toolUses];
|
||||
}
|
||||
|
||||
for (const part of content) {
|
||||
if (typeof part !== 'object' || part == null || !('type' in part)) {
|
||||
continue;
|
||||
}
|
||||
if (part.type === 'text' && 'text' in part) {
|
||||
textContent += (part as { type: 'text'; text: string }).text;
|
||||
} else if (part.type === 'tool_use') {
|
||||
toolUses.push(
|
||||
part as { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return [textContent, toolUses];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a Claude Code usage payload into the TOKEN_USAGE schema.
|
||||
* Stores fields as the Anthropic API reports them, matching
|
||||
* `mlflow.anthropic.autolog`: `input_tokens` is the non-cached input,
|
||||
* cache tokens are exposed as separate optional keys so consumers can
|
||||
* compute cache hit rate, and `total_tokens` follows the
|
||||
* `mlflow.anthropic` convention of `input_tokens + output_tokens`
|
||||
* (cache tokens excluded).
|
||||
*/
|
||||
export function buildUsageDict(usage: TokenUsage): Record<string, number> {
|
||||
const inputTokens = usage.input_tokens ?? 0;
|
||||
const outputTokens = usage.output_tokens ?? 0;
|
||||
|
||||
const usageDict: Record<string, number> = {
|
||||
[TokenUsageKey.INPUT_TOKENS]: inputTokens,
|
||||
[TokenUsageKey.OUTPUT_TOKENS]: outputTokens,
|
||||
[TokenUsageKey.TOTAL_TOKENS]: inputTokens + outputTokens,
|
||||
};
|
||||
if (usage.cache_read_input_tokens !== undefined) {
|
||||
usageDict[TokenUsageKey.CACHE_READ_INPUT_TOKENS] = usage.cache_read_input_tokens;
|
||||
}
|
||||
if (usage.cache_creation_input_tokens !== undefined) {
|
||||
usageDict[TokenUsageKey.CACHE_CREATION_INPUT_TOKENS] = usage.cache_creation_input_tokens;
|
||||
}
|
||||
return usageDict;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// JSON-safe sanitization
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Deep-clone a value, replacing functions with a placeholder. Used to record
|
||||
* the caller's `options` on the root span without serializing callbacks like
|
||||
* `canUseTool`, `mcpServers[].command`, or `hooks[].hooks[]`. Returning the
|
||||
* original `options` object would leak function references into the span
|
||||
* inputs (which then fail to JSON-serialize or grow unbounded).
|
||||
*/
|
||||
export function sanitizeForSpan(value: unknown, seen: WeakSet<object> = new WeakSet()): unknown {
|
||||
if (typeof value === 'function') {
|
||||
return '[function]';
|
||||
}
|
||||
if (value == null || typeof value !== 'object') {
|
||||
return value;
|
||||
}
|
||||
if (seen.has(value)) {
|
||||
return '[circular]';
|
||||
}
|
||||
seen.add(value);
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((v) => sanitizeForSpan(v, seen));
|
||||
}
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const [k, v] of Object.entries(value)) {
|
||||
result[k] = sanitizeForSpan(v, seen);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { runSetup, runStatus } from './commands/setup.js';
|
||||
|
||||
function printUsage(): void {
|
||||
console.error('Usage: mlflow-claude-code <command> [options]');
|
||||
console.error('');
|
||||
console.error('Commands:');
|
||||
console.error(' setup Configure MLflow tracing for Claude Code.');
|
||||
console.error(' Options:');
|
||||
console.error(
|
||||
' -p, --project Write settings to ./.claude/settings.json',
|
||||
);
|
||||
console.error(' (this repo only).');
|
||||
console.error(
|
||||
' -u, --user Write settings to ~/.claude/settings.json',
|
||||
);
|
||||
console.error(' (all repos).');
|
||||
console.error(' --tracking-uri <uri> MLflow tracking URI. One of');
|
||||
console.error(" 'databricks',");
|
||||
console.error(" 'databricks://<profile>',");
|
||||
console.error(' or an absolute http(s) URL.');
|
||||
console.error(' --experiment-id <id> Use an existing MLflow experiment by ID.');
|
||||
console.error(
|
||||
' --experiment-name <n> Create or reuse an MLflow experiment by name.',
|
||||
);
|
||||
console.error(' --trace-location <loc> Optional Databricks Unity Catalog trace');
|
||||
console.error(
|
||||
" location as 'catalog.schema.table_prefix'.",
|
||||
);
|
||||
console.error('');
|
||||
console.error(' Required values (all must come from the user; do not pick');
|
||||
console.error(' defaults silently):');
|
||||
console.error(' - Scope: pass exactly one of --project or --user.');
|
||||
console.error(' - Tracking URI: pass --tracking-uri.');
|
||||
console.error(' - Experiment: pass exactly one of --experiment-id or');
|
||||
console.error(' --experiment-name.');
|
||||
console.error('');
|
||||
console.error(' Examples:');
|
||||
console.error(
|
||||
' $ mlflow-claude-code setup --project --tracking-uri http://localhost:5000 \\',
|
||||
);
|
||||
console.error(' --experiment-name my-exp');
|
||||
console.error(
|
||||
' $ mlflow-claude-code setup --user --tracking-uri databricks --experiment-id 12345',
|
||||
);
|
||||
console.error(' $ mlflow-claude-code setup --user --tracking-uri databricks \\');
|
||||
console.error(
|
||||
' --experiment-id 12345 --trace-location my_catalog.my_schema.my_prefix',
|
||||
);
|
||||
console.error('');
|
||||
console.error(' status Show the current MLflow tracing configuration.');
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const [, , command, ...rest] = process.argv;
|
||||
const wantsHelp = rest.includes('--help') || rest.includes('-h');
|
||||
|
||||
if (command === undefined || command === '--help' || command === '-h' || command === 'help') {
|
||||
printUsage();
|
||||
if (command === undefined) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === 'setup') {
|
||||
if (wantsHelp) {
|
||||
printUsage();
|
||||
return;
|
||||
}
|
||||
await runSetup(rest);
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === 'status') {
|
||||
if (wantsHelp) {
|
||||
printUsage();
|
||||
return;
|
||||
}
|
||||
runStatus();
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(`Unknown command: ${command}\n`);
|
||||
printUsage();
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -0,0 +1,171 @@
|
||||
import { existsSync } from 'node:fs';
|
||||
|
||||
import {
|
||||
getEffectiveTracingConfig,
|
||||
isValidTrackingUri,
|
||||
parseTraceLocation,
|
||||
resolveExperiment,
|
||||
resolveSettingsPath,
|
||||
writeTracingSettings,
|
||||
type ConfigPathOptions,
|
||||
} from '../config.js';
|
||||
|
||||
export interface SetupOptions extends ConfigPathOptions {
|
||||
trackingUri?: string;
|
||||
experimentId?: string;
|
||||
experimentName?: string;
|
||||
traceLocation?: string;
|
||||
}
|
||||
|
||||
export interface ParsedSetupArgs extends SetupOptions {
|
||||
projectLocal: boolean | undefined;
|
||||
}
|
||||
|
||||
export function parseSetupArgs(args: string[]): ParsedSetupArgs {
|
||||
const parsed: ParsedSetupArgs = { projectLocal: undefined };
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
if (arg === '--project' || arg === '-p') {
|
||||
parsed.projectLocal = true;
|
||||
} else if (arg === '--user' || arg === '-u') {
|
||||
parsed.projectLocal = false;
|
||||
} else if (arg === '--tracking-uri') {
|
||||
parsed.trackingUri = args[++i];
|
||||
} else if (arg === '--experiment-id') {
|
||||
parsed.experimentId = args[++i];
|
||||
} else if (arg === '--experiment-name') {
|
||||
parsed.experimentName = args[++i];
|
||||
} else if (arg === '--trace-location') {
|
||||
// Treat a missing value as empty (not undefined) so validation fails
|
||||
// loudly instead of silently ignoring the flag.
|
||||
parsed.traceLocation = args[++i] ?? '';
|
||||
}
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function printSummary(
|
||||
settingsPath: string,
|
||||
config: {
|
||||
trackingUri: string;
|
||||
experimentId: string;
|
||||
experimentName?: string;
|
||||
traceLocation?: string;
|
||||
},
|
||||
): void {
|
||||
console.error('\nCurrent configuration');
|
||||
console.error(' Tracing enabled: true');
|
||||
console.error(` Settings file: ${settingsPath}`);
|
||||
console.error(` Tracking URI: ${config.trackingUri}`);
|
||||
console.error(` Experiment ID: ${config.experimentId}`);
|
||||
if (config.experimentName) {
|
||||
console.error(` Experiment name: ${config.experimentName}`);
|
||||
}
|
||||
if (config.traceLocation) {
|
||||
console.error(` Trace location: ${config.traceLocation}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function runSetup(args: string[], options: SetupOptions = {}): Promise<void> {
|
||||
const parsed = parseSetupArgs(args);
|
||||
const merged: ParsedSetupArgs = { ...parsed, ...options, projectLocal: parsed.projectLocal };
|
||||
|
||||
if (merged.projectLocal === undefined) {
|
||||
console.error('Error: must pass --project or --user.');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!merged.trackingUri) {
|
||||
console.error('Error: --tracking-uri is required.');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isValidTrackingUri(merged.trackingUri)) {
|
||||
console.error(
|
||||
`Error: invalid --tracking-uri: ${merged.trackingUri}. ` +
|
||||
"Must be 'databricks', 'databricks://<profile>', or an absolute http(s) URL.",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (merged.experimentId && merged.experimentName) {
|
||||
console.error('Error: pass only one of --experiment-id or --experiment-name.');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!merged.experimentId && !merged.experimentName) {
|
||||
console.error('Error: must pass --experiment-id or --experiment-name.');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (merged.traceLocation !== undefined && !parseTraceLocation(merged.traceLocation)) {
|
||||
console.error(
|
||||
`Error: invalid --trace-location: ${merged.traceLocation}. ` +
|
||||
"Must be in 'catalog.schema.table_prefix' format.",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const settingsPath = resolveSettingsPath(merged.projectLocal, merged);
|
||||
|
||||
try {
|
||||
const settingsFileExisted = existsSync(settingsPath);
|
||||
const resolved = await resolveExperiment(
|
||||
merged.trackingUri,
|
||||
merged.experimentId,
|
||||
merged.experimentName,
|
||||
);
|
||||
writeTracingSettings(settingsPath, {
|
||||
trackingUri: merged.trackingUri,
|
||||
experimentId: resolved.experimentId,
|
||||
experimentName: resolved.experimentName,
|
||||
traceLocation: merged.traceLocation,
|
||||
});
|
||||
|
||||
console.error(`\n${settingsFileExisted ? 'Updated' : 'Created'} ${settingsPath}`);
|
||||
if (resolved.created && resolved.experimentName) {
|
||||
console.error(
|
||||
`Created MLflow experiment ${resolved.experimentName} (${resolved.experimentId})`,
|
||||
);
|
||||
} else {
|
||||
console.error(`Resolved MLflow experiment ID ${resolved.experimentId}`);
|
||||
}
|
||||
|
||||
printSummary(settingsPath, {
|
||||
trackingUri: merged.trackingUri,
|
||||
experimentId: resolved.experimentId,
|
||||
experimentName: resolved.experimentName,
|
||||
traceLocation: merged.traceLocation,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
export function runStatus(options: ConfigPathOptions = {}): void {
|
||||
const config = getEffectiveTracingConfig(options);
|
||||
|
||||
console.error('\nMLflow Tracing Status\n');
|
||||
console.error(` Enabled: ${config.enabled ? 'true' : 'false'}`);
|
||||
console.error(` Source: ${config.source}`);
|
||||
if (config.settingsPath) {
|
||||
console.error(` Settings file: ${config.settingsPath}`);
|
||||
}
|
||||
console.error(` Tracking URI: ${config.trackingUri ?? 'not set'}`);
|
||||
console.error(` Experiment ID: ${config.experimentId ?? 'not set'}`);
|
||||
console.error(` Experiment name: ${config.experimentName ?? 'not set'}`);
|
||||
if (config.traceLocation) {
|
||||
console.error(` Trace location: ${config.traceLocation}`);
|
||||
}
|
||||
|
||||
if (!config.enabled) {
|
||||
console.error('\nTracing is disabled. Run `mlflow-claude-code setup` to configure it.');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
|
||||
import { createAuthProvider, init, MlflowClient } from '@mlflow/core';
|
||||
|
||||
export const MLFLOW_CLAUDE_TRACING_ENABLED = 'MLFLOW_CLAUDE_TRACING_ENABLED';
|
||||
export const MLFLOW_TRACKING_URI = 'MLFLOW_TRACKING_URI';
|
||||
export const MLFLOW_EXPERIMENT_ID = 'MLFLOW_EXPERIMENT_ID';
|
||||
export const MLFLOW_EXPERIMENT_NAME = 'MLFLOW_EXPERIMENT_NAME';
|
||||
/**
|
||||
* Optional Databricks Unity Catalog trace location, in
|
||||
* `catalog.schema.table_prefix` form. When set, Claude Code traces are routed
|
||||
* to the UC table-prefix destination (V4 trace IDs) instead of the V3
|
||||
* experiment-backed path. The UC location must already be provisioned in the
|
||||
* workspace; the SDK does not create it.
|
||||
*/
|
||||
export const MLFLOW_TRACE_LOCATION = 'MLFLOW_TRACE_LOCATION';
|
||||
|
||||
type ConfigSource = 'environment' | 'project' | 'user' | 'none';
|
||||
|
||||
export interface ClaudeSettings {
|
||||
env?: Record<string, string>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface TracingConfig {
|
||||
enabled: boolean;
|
||||
trackingUri?: string;
|
||||
experimentId?: string;
|
||||
experimentName?: string;
|
||||
/** Raw `catalog.schema.table_prefix` UC trace location, if configured. */
|
||||
traceLocation?: string;
|
||||
source: ConfigSource;
|
||||
settingsPath?: string;
|
||||
}
|
||||
|
||||
export interface UnityCatalogTraceLocation {
|
||||
catalogName: string;
|
||||
schemaName: string;
|
||||
tablePrefix: string;
|
||||
}
|
||||
|
||||
export interface ConfigPathOptions {
|
||||
home?: string;
|
||||
cwd?: string;
|
||||
}
|
||||
|
||||
let initializedKey: string | null = null;
|
||||
|
||||
function isTruthy(value: string | undefined): boolean {
|
||||
const normalized = (value ?? '').trim().toLowerCase();
|
||||
return normalized === 'true' || normalized === '1' || normalized === 'yes';
|
||||
}
|
||||
|
||||
function hasConfigValue(value: string | undefined): value is string {
|
||||
return Boolean(value && value.trim().length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a `catalog.schema.table_prefix` string into a UC trace location.
|
||||
* Returns null when the value is empty or not exactly three non-empty,
|
||||
* dot-separated parts. All three parts are required because the SDK does not
|
||||
* upsert UC trace locations.
|
||||
*/
|
||||
export function parseTraceLocation(value: string | undefined): UnityCatalogTraceLocation | null {
|
||||
if (!hasConfigValue(value)) {
|
||||
return null;
|
||||
}
|
||||
const parts = value.trim().split('.');
|
||||
if (parts.length !== 3 || parts.some((part) => part.trim().length === 0)) {
|
||||
return null;
|
||||
}
|
||||
const [catalogName, schemaName, tablePrefix] = parts.map((part) => part.trim());
|
||||
return { catalogName, schemaName, tablePrefix };
|
||||
}
|
||||
|
||||
function hasAnyTracingKey(env: Record<string, string | undefined>): boolean {
|
||||
return [
|
||||
MLFLOW_CLAUDE_TRACING_ENABLED,
|
||||
MLFLOW_TRACKING_URI,
|
||||
MLFLOW_EXPERIMENT_ID,
|
||||
MLFLOW_EXPERIMENT_NAME,
|
||||
MLFLOW_TRACE_LOCATION,
|
||||
].some((key) => env[key] !== undefined);
|
||||
}
|
||||
|
||||
function hasTracingConfig(config: TracingConfig): boolean {
|
||||
return Boolean(
|
||||
config.trackingUri ||
|
||||
config.experimentId ||
|
||||
config.experimentName ||
|
||||
config.traceLocation ||
|
||||
config.enabled,
|
||||
);
|
||||
}
|
||||
|
||||
function parseTracingConfig(
|
||||
env: Record<string, string | undefined>,
|
||||
source: ConfigSource,
|
||||
settingsPath?: string,
|
||||
): TracingConfig {
|
||||
return {
|
||||
enabled: isTruthy(env[MLFLOW_CLAUDE_TRACING_ENABLED]),
|
||||
trackingUri: env[MLFLOW_TRACKING_URI],
|
||||
experimentId: env[MLFLOW_EXPERIMENT_ID],
|
||||
experimentName: env[MLFLOW_EXPERIMENT_NAME],
|
||||
traceLocation: env[MLFLOW_TRACE_LOCATION],
|
||||
source,
|
||||
settingsPath,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveSettingsPath(
|
||||
projectLocal: boolean,
|
||||
options: ConfigPathOptions = {},
|
||||
): string {
|
||||
return projectLocal
|
||||
? resolve(options.cwd ?? process.cwd(), '.claude', 'settings.json')
|
||||
: resolve(options.home ?? homedir(), '.claude', 'settings.json');
|
||||
}
|
||||
|
||||
export function loadSettings(path: string): ClaudeSettings {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = readFileSync(path, 'utf-8');
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
return JSON.parse(raw) as ClaudeSettings;
|
||||
}
|
||||
|
||||
export function saveSettings(path: string, settings: ClaudeSettings): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, JSON.stringify(settings, null, 2) + '\n', 'utf-8');
|
||||
}
|
||||
|
||||
export function getScopeTracingConfig(
|
||||
projectLocal: boolean,
|
||||
options: ConfigPathOptions = {},
|
||||
): TracingConfig {
|
||||
const settingsPath = resolveSettingsPath(projectLocal, options);
|
||||
const settings = loadSettings(settingsPath);
|
||||
return parseTracingConfig(
|
||||
Object.fromEntries(
|
||||
Object.entries(settings.env ?? {}).map(([key, value]) => [key, String(value)]),
|
||||
),
|
||||
projectLocal ? 'project' : 'user',
|
||||
settingsPath,
|
||||
);
|
||||
}
|
||||
|
||||
export function getEffectiveTracingConfig(options: ConfigPathOptions = {}): TracingConfig {
|
||||
const userConfig = getScopeTracingConfig(false, options);
|
||||
const projectConfig = getScopeTracingConfig(true, options);
|
||||
|
||||
const merged = {
|
||||
enabled: userConfig.enabled,
|
||||
trackingUri: userConfig.trackingUri,
|
||||
experimentId: userConfig.experimentId,
|
||||
experimentName: userConfig.experimentName,
|
||||
traceLocation: userConfig.traceLocation,
|
||||
...(hasTracingConfig(projectConfig)
|
||||
? {
|
||||
enabled: projectConfig.enabled,
|
||||
trackingUri: projectConfig.trackingUri,
|
||||
experimentId: projectConfig.experimentId,
|
||||
experimentName: projectConfig.experimentName,
|
||||
traceLocation: projectConfig.traceLocation,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
|
||||
const envConfig = parseTracingConfig(process.env, 'environment');
|
||||
const effective: TracingConfig = {
|
||||
enabled:
|
||||
process.env[MLFLOW_CLAUDE_TRACING_ENABLED] !== undefined ? envConfig.enabled : merged.enabled,
|
||||
trackingUri: process.env[MLFLOW_TRACKING_URI] ?? merged.trackingUri,
|
||||
experimentId: process.env[MLFLOW_EXPERIMENT_ID] ?? merged.experimentId,
|
||||
experimentName: process.env[MLFLOW_EXPERIMENT_NAME] ?? merged.experimentName,
|
||||
traceLocation: process.env[MLFLOW_TRACE_LOCATION] ?? merged.traceLocation,
|
||||
source: 'none',
|
||||
};
|
||||
|
||||
if (hasAnyTracingKey(process.env)) {
|
||||
effective.source = 'environment';
|
||||
} else if (hasTracingConfig(projectConfig)) {
|
||||
effective.source = 'project';
|
||||
effective.settingsPath = projectConfig.settingsPath;
|
||||
} else if (hasTracingConfig(userConfig)) {
|
||||
effective.source = 'user';
|
||||
effective.settingsPath = userConfig.settingsPath;
|
||||
}
|
||||
|
||||
return effective;
|
||||
}
|
||||
|
||||
export function isTracingEnabled(): boolean {
|
||||
return getEffectiveTracingConfig().enabled;
|
||||
}
|
||||
|
||||
export function isValidTrackingUri(raw: string): boolean {
|
||||
if (raw === 'databricks' || raw.startsWith('databricks://')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return parsed.protocol === 'http:' || parsed.protocol === 'https:';
|
||||
}
|
||||
|
||||
export async function resolveExperiment(
|
||||
trackingUri: string,
|
||||
experimentId?: string,
|
||||
experimentName?: string,
|
||||
): Promise<{ experimentId: string; experimentName?: string; created: boolean }> {
|
||||
if (hasConfigValue(experimentId)) {
|
||||
return { experimentId, experimentName, created: false };
|
||||
}
|
||||
|
||||
if (!hasConfigValue(experimentName)) {
|
||||
throw new Error(
|
||||
'Either MLFLOW_EXPERIMENT_ID or MLFLOW_EXPERIMENT_NAME must be configured for Claude tracing.',
|
||||
);
|
||||
}
|
||||
|
||||
const authProvider = createAuthProvider({ trackingUri });
|
||||
const client = new MlflowClient({ trackingUri, authProvider });
|
||||
const existing = await client.getExperimentByName(experimentName);
|
||||
if (existing) {
|
||||
return {
|
||||
experimentId: existing.experimentId,
|
||||
experimentName: existing.name,
|
||||
created: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
experimentId: await client.createExperiment(experimentName),
|
||||
experimentName,
|
||||
created: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function writeTracingSettings(
|
||||
settingsPath: string,
|
||||
config: {
|
||||
trackingUri: string;
|
||||
experimentId: string;
|
||||
experimentName?: string;
|
||||
traceLocation?: string;
|
||||
enabled?: boolean;
|
||||
},
|
||||
): void {
|
||||
const settings = loadSettings(settingsPath);
|
||||
const env = { ...(settings.env ?? {}) };
|
||||
|
||||
env[MLFLOW_CLAUDE_TRACING_ENABLED] = config.enabled === false ? 'false' : 'true';
|
||||
env[MLFLOW_TRACKING_URI] = config.trackingUri;
|
||||
|
||||
if (hasConfigValue(config.experimentId)) {
|
||||
env[MLFLOW_EXPERIMENT_ID] = config.experimentId;
|
||||
} else {
|
||||
delete env[MLFLOW_EXPERIMENT_ID];
|
||||
}
|
||||
|
||||
if (hasConfigValue(config.experimentName)) {
|
||||
env[MLFLOW_EXPERIMENT_NAME] = config.experimentName;
|
||||
} else {
|
||||
delete env[MLFLOW_EXPERIMENT_NAME];
|
||||
}
|
||||
|
||||
if (hasConfigValue(config.traceLocation)) {
|
||||
// Trim on write so stored settings match what parsing/init() will use.
|
||||
env[MLFLOW_TRACE_LOCATION] = config.traceLocation.trim();
|
||||
} else {
|
||||
delete env[MLFLOW_TRACE_LOCATION];
|
||||
}
|
||||
|
||||
settings.env = env;
|
||||
saveSettings(settingsPath, settings);
|
||||
}
|
||||
|
||||
export async function ensureInitialized(): Promise<boolean> {
|
||||
const config = getEffectiveTracingConfig();
|
||||
if (!config.enabled) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!hasConfigValue(config.trackingUri)) {
|
||||
console.error('[mlflow] MLFLOW_TRACKING_URI is not set');
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!hasConfigValue(config.experimentId) && !hasConfigValue(config.experimentName)) {
|
||||
console.error('[mlflow] MLFLOW_EXPERIMENT_ID or MLFLOW_EXPERIMENT_NAME is not set');
|
||||
return false;
|
||||
}
|
||||
|
||||
let traceLocation: UnityCatalogTraceLocation | null = null;
|
||||
if (hasConfigValue(config.traceLocation)) {
|
||||
traceLocation = parseTraceLocation(config.traceLocation);
|
||||
if (!traceLocation) {
|
||||
console.error(
|
||||
`[mlflow] MLFLOW_TRACE_LOCATION must be in 'catalog.schema.table_prefix' format, ` +
|
||||
`got '${config.traceLocation}'`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Fast path: skip network call when experimentId is already known.
|
||||
if (hasConfigValue(config.experimentId)) {
|
||||
const quickKey = JSON.stringify({
|
||||
trackingUri: config.trackingUri,
|
||||
experimentId: config.experimentId,
|
||||
traceLocation,
|
||||
});
|
||||
if (initializedKey === quickKey) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const resolvedExperiment = await resolveExperiment(
|
||||
config.trackingUri,
|
||||
config.experimentId,
|
||||
config.experimentName,
|
||||
);
|
||||
const initKey = JSON.stringify({
|
||||
trackingUri: config.trackingUri,
|
||||
experimentId: resolvedExperiment.experimentId,
|
||||
traceLocation,
|
||||
});
|
||||
if (initializedKey === initKey) {
|
||||
return true;
|
||||
}
|
||||
|
||||
init({
|
||||
trackingUri: config.trackingUri,
|
||||
experimentId: resolvedExperiment.experimentId,
|
||||
...(traceLocation ? { traceLocation } : {}),
|
||||
});
|
||||
initializedKey = initKey;
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error('[mlflow] Failed to initialize:', err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { readStdin } from '../utils/stdin.js';
|
||||
import { isTracingEnabled, ensureInitialized } from '../config.js';
|
||||
import { processTranscript } from '../tracing.js';
|
||||
import type { StopHookInput } from '../types.js';
|
||||
|
||||
async function main(): Promise<void> {
|
||||
try {
|
||||
const input = await readStdin<StopHookInput>();
|
||||
if (!isTracingEnabled()) {
|
||||
return;
|
||||
}
|
||||
if (!(await ensureInitialized())) {
|
||||
return;
|
||||
}
|
||||
await processTranscript(input.transcript_path, input.session_id);
|
||||
} catch (err) {
|
||||
console.error('[mlflow]', err);
|
||||
}
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -0,0 +1,19 @@
|
||||
export { processTranscript } from './tracing.js';
|
||||
export {
|
||||
isTracingEnabled,
|
||||
ensureInitialized,
|
||||
getEffectiveTracingConfig,
|
||||
resolveSettingsPath,
|
||||
} from './config.js';
|
||||
export { createTracedQuery } from './tracedClaudeAgent.js';
|
||||
export type {
|
||||
TranscriptEntry,
|
||||
MessageContent,
|
||||
ContentBlock,
|
||||
TextBlock,
|
||||
ToolUseBlock,
|
||||
ToolResultBlock,
|
||||
ThinkingBlock,
|
||||
TokenUsage,
|
||||
StopHookInput,
|
||||
} from './types.js';
|
||||
@@ -0,0 +1,421 @@
|
||||
/**
|
||||
* Live trace construction for the Claude Agent SDK message stream.
|
||||
*
|
||||
* Mirrors the span shape produced by processTranscript (CLI Stop-hook path)
|
||||
* but emits spans incrementally as SDKMessages arrive so that long-running
|
||||
* agent sessions show progress and partial traces survive crashes.
|
||||
*
|
||||
* Span tree (matches CLI integration):
|
||||
* claude_code_conversation (AGENT)
|
||||
* ├── llm (LLM) per assistant turn with text/thinking
|
||||
* ├── tool_<name> (TOOL) per tool_use block at root
|
||||
* │ └── subagent_<type> (AGENT) lazily created for Task/Agent tool calls
|
||||
* │ ├── llm
|
||||
* │ └── tool_<name>
|
||||
* └── ...
|
||||
*
|
||||
* Sub-agent attribution is driven by SDKMessage.parent_tool_use_id (set by
|
||||
* the SDK when forwardSubagentText: true is passed on query() options).
|
||||
*/
|
||||
|
||||
import {
|
||||
startSpan,
|
||||
flushTraces,
|
||||
SpanType,
|
||||
SpanAttributeKey,
|
||||
SpanStatusCode,
|
||||
TraceMetadataKey,
|
||||
InMemoryTraceManager,
|
||||
type LiveSpan,
|
||||
} from '@mlflow/core';
|
||||
|
||||
import type { ContentBlock, MessageContent, TokenUsage } from './types.js';
|
||||
import {
|
||||
MAX_PREVIEW_LENGTH,
|
||||
METADATA_KEY_CLAUDE_CODE_VERSION,
|
||||
METADATA_KEY_PERMISSION_MODE,
|
||||
METADATA_KEY_WORKING_DIRECTORY,
|
||||
buildUsageDict,
|
||||
extractContentAndTools,
|
||||
sanitizeForSpan,
|
||||
} from './_internal.js';
|
||||
|
||||
// Tool names that invoke a sub-agent. `'Agent'` is what the Claude Agent SDK
|
||||
// emits as of 0.2.x (confirmed via live stream inspection); `'Task'` is the
|
||||
// legacy name still used by the Claude Code CLI transcript. Both surfaces
|
||||
// land in this code path now that SDK and CLI traces share their builders.
|
||||
const SUBAGENT_TOOL_NAMES = new Set(['Task', 'Agent']);
|
||||
|
||||
type ConversationKey = string | null;
|
||||
|
||||
interface InputMessage {
|
||||
role: 'user' | 'assistant';
|
||||
content: string | ContentBlock[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal structural types for the SDKMessage subset we consume. The wrapper
|
||||
* narrows real SDKMessage values to these shapes before dispatching, so the
|
||||
* SDK package itself is only a peer dependency (and only required for types).
|
||||
*/
|
||||
export interface SDKSystemInit {
|
||||
type: 'system';
|
||||
subtype: 'init';
|
||||
session_id?: string;
|
||||
model?: string;
|
||||
cwd?: string;
|
||||
permissionMode?: string;
|
||||
claude_code_version?: string;
|
||||
}
|
||||
|
||||
export interface SDKAssistantLike {
|
||||
type: 'assistant';
|
||||
message: MessageContent & { usage?: TokenUsage };
|
||||
parent_tool_use_id?: string | null;
|
||||
}
|
||||
|
||||
export interface SDKUserLike {
|
||||
type: 'user';
|
||||
message: { role: 'user'; content: string | ContentBlock[] };
|
||||
parent_tool_use_id?: string | null;
|
||||
}
|
||||
|
||||
export interface SDKResultLike {
|
||||
type: 'result';
|
||||
/** 'success' on normal completion; any other value is an SDK-defined error subtype. */
|
||||
subtype: string;
|
||||
duration_ms?: number;
|
||||
total_cost_usd?: number;
|
||||
usage?: TokenUsage;
|
||||
result?: string;
|
||||
}
|
||||
|
||||
export class LiveTracingContext {
|
||||
private rootSpan: LiveSpan;
|
||||
private openToolSpans = new Map<string, LiveSpan>();
|
||||
private subagentSpans = new Map<string, LiveSpan>();
|
||||
private conversations = new Map<ConversationKey, InputMessage[]>();
|
||||
private lastAssistantText = new Map<ConversationKey, string>();
|
||||
private sessionId: string | undefined;
|
||||
private model: string | undefined;
|
||||
private permissionMode: string | undefined;
|
||||
private claudeCodeVersion: string | undefined;
|
||||
private ended = false;
|
||||
private readonly spanOptions: unknown;
|
||||
|
||||
constructor(prompt: string, options?: Record<string, unknown>) {
|
||||
this.spanOptions = sanitizeOptions(options);
|
||||
this.rootSpan = startSpan({
|
||||
name: 'claude_code_conversation',
|
||||
spanType: SpanType.AGENT,
|
||||
inputs: { prompt, options: this.spanOptions },
|
||||
});
|
||||
this.conversations.set(null, [{ role: 'user', content: prompt }]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Append captured prompt text (used when the caller passes an AsyncIterable
|
||||
* prompt to query() — content arrives over time, not as a single string).
|
||||
* Each call refreshes the root span's `inputs.prompt` so users see the most
|
||||
* complete prompt available in the trace UI as more chunks arrive.
|
||||
*/
|
||||
appendPromptText(text: string): void {
|
||||
const conversation = this.getConversation(null);
|
||||
const first = conversation[0];
|
||||
if (first?.role === 'user' && typeof first.content === 'string') {
|
||||
first.content = first.content ? `${first.content}\n${text}` : text;
|
||||
} else {
|
||||
conversation.unshift({ role: 'user', content: text });
|
||||
}
|
||||
this.rootSpan.setInputs({ prompt: conversation[0].content, options: this.spanOptions });
|
||||
}
|
||||
|
||||
/** Update root span metadata with session-level info from the init message. */
|
||||
onSystemInit(msg: SDKSystemInit): void {
|
||||
this.sessionId = msg.session_id ?? this.sessionId;
|
||||
this.model = msg.model ?? this.model;
|
||||
this.permissionMode = msg.permissionMode ?? this.permissionMode;
|
||||
this.claudeCodeVersion = msg.claude_code_version ?? this.claudeCodeVersion;
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit an LLM span (if the message has text/thinking) and open TOOL spans
|
||||
* for each tool_use block. Sub-agent routing is keyed on parent_tool_use_id.
|
||||
*/
|
||||
onAssistantMessage(msg: SDKAssistantLike): void {
|
||||
const parentKey: ConversationKey = msg.parent_tool_use_id ?? null;
|
||||
const parentSpan = this.resolveParent(parentKey);
|
||||
const content = msg.message.content;
|
||||
const [textContent, toolUses] = extractContentAndTools(content);
|
||||
|
||||
const conversation = this.getConversation(parentKey);
|
||||
const priorMessages = conversation.slice();
|
||||
conversation.push({ role: 'assistant', content });
|
||||
|
||||
const model = msg.message.model ?? this.model ?? 'unknown';
|
||||
if (textContent.trim() || hasThinking(content)) {
|
||||
const llmSpan = startSpan({
|
||||
name: 'llm',
|
||||
spanType: SpanType.LLM,
|
||||
parent: parentSpan,
|
||||
inputs: { model, messages: priorMessages },
|
||||
attributes: {
|
||||
model,
|
||||
'mlflow.llm.model': model,
|
||||
[SpanAttributeKey.MESSAGE_FORMAT]: 'anthropic',
|
||||
},
|
||||
});
|
||||
if (msg.message.usage) {
|
||||
llmSpan.setAttribute(SpanAttributeKey.TOKEN_USAGE, buildUsageDict(msg.message.usage));
|
||||
}
|
||||
llmSpan.setOutputs({ type: 'message', role: 'assistant', content });
|
||||
llmSpan.end();
|
||||
|
||||
if (textContent.trim()) {
|
||||
this.lastAssistantText.set(parentKey, textContent);
|
||||
}
|
||||
}
|
||||
|
||||
for (const toolUse of toolUses) {
|
||||
const toolSpan = startSpan({
|
||||
name: `tool_${toolUse.name}`,
|
||||
spanType: SpanType.TOOL,
|
||||
parent: parentSpan,
|
||||
inputs: toolUse.input ?? {},
|
||||
attributes: { tool_name: toolUse.name, tool_id: toolUse.id },
|
||||
});
|
||||
this.openToolSpans.set(toolUse.id, toolSpan);
|
||||
|
||||
// Lazily create the sub-agent wrapper span when we see a Task/Agent tool
|
||||
// call. Sub-agent inner messages arrive with parent_tool_use_id === toolUse.id,
|
||||
// and resolveParent() will route them here.
|
||||
if (SUBAGENT_TOOL_NAMES.has(toolUse.name)) {
|
||||
const subagentType = (toolUse.input?.subagent_type as string | undefined) ?? 'agent';
|
||||
const subagentSpan = startSpan({
|
||||
name: `subagent_${subagentType}`,
|
||||
spanType: SpanType.AGENT,
|
||||
parent: toolSpan,
|
||||
inputs: {
|
||||
prompt: toolUse.input?.prompt,
|
||||
description: toolUse.input?.description,
|
||||
subagent_type: subagentType,
|
||||
},
|
||||
attributes: { subagent_type: subagentType },
|
||||
});
|
||||
this.subagentSpans.set(toolUse.id, subagentSpan);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Close any TOOL spans whose tool_use_id matches a tool_result block. */
|
||||
onUserMessage(msg: SDKUserLike): void {
|
||||
const parentKey: ConversationKey = msg.parent_tool_use_id ?? null;
|
||||
const conversation = this.getConversation(parentKey);
|
||||
conversation.push({ role: 'user', content: msg.message.content });
|
||||
|
||||
const content = msg.message.content;
|
||||
if (!Array.isArray(content)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const block of content) {
|
||||
if (
|
||||
typeof block !== 'object' ||
|
||||
block == null ||
|
||||
!('type' in block) ||
|
||||
block.type !== 'tool_result'
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const toolResult = block as {
|
||||
type: 'tool_result';
|
||||
tool_use_id?: string;
|
||||
content?: unknown;
|
||||
is_error?: boolean;
|
||||
};
|
||||
const toolUseId = toolResult.tool_use_id;
|
||||
if (!toolUseId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If this tool result closes a Task/Agent call, finalize its sub-agent
|
||||
// wrapper span first so it nests under the TOOL span correctly.
|
||||
const subagentSpan = this.subagentSpans.get(toolUseId);
|
||||
if (subagentSpan) {
|
||||
const lastText = this.lastAssistantText.get(toolUseId);
|
||||
if (lastText) {
|
||||
subagentSpan.setOutputs({ response: lastText });
|
||||
}
|
||||
subagentSpan.end();
|
||||
this.subagentSpans.delete(toolUseId);
|
||||
this.conversations.delete(toolUseId);
|
||||
this.lastAssistantText.delete(toolUseId);
|
||||
}
|
||||
|
||||
const toolSpan = this.openToolSpans.get(toolUseId);
|
||||
if (!toolSpan) {
|
||||
continue;
|
||||
}
|
||||
|
||||
toolSpan.setOutputs({ result: toolResult.content ?? '' });
|
||||
if (toolResult.is_error) {
|
||||
const errorText =
|
||||
typeof toolResult.content === 'string'
|
||||
? toolResult.content
|
||||
: JSON.stringify(toolResult.content);
|
||||
toolSpan.setStatus(SpanStatusCode.ERROR, errorText || 'Tool execution failed');
|
||||
}
|
||||
toolSpan.end();
|
||||
this.openToolSpans.delete(toolUseId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Record aggregate usage and final result text from the SDK's result message. */
|
||||
onResultMessage(msg: SDKResultLike): void {
|
||||
if (msg.usage) {
|
||||
this.rootSpan.setAttribute(SpanAttributeKey.TOKEN_USAGE, buildUsageDict(msg.usage));
|
||||
}
|
||||
if (msg.subtype !== 'success') {
|
||||
this.rootSpan.setStatus(SpanStatusCode.ERROR, msg.subtype);
|
||||
}
|
||||
const finalText = msg.result ?? this.lastAssistantText.get(null);
|
||||
if (finalText) {
|
||||
this.lastAssistantText.set(null, finalText);
|
||||
}
|
||||
}
|
||||
|
||||
/** Close all open spans with ERROR status on stream throw or interruption. */
|
||||
async finalizeError(error: string): Promise<void> {
|
||||
if (this.ended) {
|
||||
return;
|
||||
}
|
||||
for (const span of this.openToolSpans.values()) {
|
||||
span.setStatus(SpanStatusCode.ERROR, error);
|
||||
span.end();
|
||||
}
|
||||
this.openToolSpans.clear();
|
||||
for (const span of this.subagentSpans.values()) {
|
||||
span.setStatus(SpanStatusCode.ERROR, error);
|
||||
span.end();
|
||||
}
|
||||
this.subagentSpans.clear();
|
||||
this.rootSpan.setStatus(SpanStatusCode.ERROR, error);
|
||||
await this.finalize();
|
||||
}
|
||||
|
||||
/** Close the root span and flush traces to the backend; safe to call multiple times. */
|
||||
async finalize(): Promise<void> {
|
||||
if (this.ended) {
|
||||
return;
|
||||
}
|
||||
this.ended = true;
|
||||
|
||||
// Defensive: close anything still open (shouldn't happen in normal completion).
|
||||
for (const span of this.openToolSpans.values()) {
|
||||
span.end();
|
||||
}
|
||||
this.openToolSpans.clear();
|
||||
for (const span of this.subagentSpans.values()) {
|
||||
span.end();
|
||||
}
|
||||
this.subagentSpans.clear();
|
||||
|
||||
this.applyTraceMetadata();
|
||||
|
||||
const finalResponse = this.lastAssistantText.get(null);
|
||||
const outputs: Record<string, unknown> = { status: 'completed' };
|
||||
if (finalResponse) {
|
||||
outputs.response = finalResponse;
|
||||
}
|
||||
this.rootSpan.setOutputs(outputs);
|
||||
this.rootSpan.end();
|
||||
|
||||
try {
|
||||
await flushTraces();
|
||||
} catch (err) {
|
||||
console.error('[mlflow] Failed to flush traces:', err);
|
||||
}
|
||||
}
|
||||
|
||||
private getConversation(key: ConversationKey): InputMessage[] {
|
||||
let conv = this.conversations.get(key);
|
||||
if (!conv) {
|
||||
conv = [];
|
||||
this.conversations.set(key, conv);
|
||||
}
|
||||
return conv;
|
||||
}
|
||||
|
||||
private resolveParent(parentKey: ConversationKey): LiveSpan {
|
||||
if (parentKey == null) {
|
||||
return this.rootSpan;
|
||||
}
|
||||
const subagent = this.subagentSpans.get(parentKey);
|
||||
if (subagent) {
|
||||
return subagent;
|
||||
}
|
||||
// Sub-agent wrapper hasn't been created yet — fall back to root. This only
|
||||
// happens if a sub-agent message arrives before its Task tool_use, which
|
||||
// the SDK never does in practice. We avoid silently dropping the span.
|
||||
return this.rootSpan;
|
||||
}
|
||||
|
||||
private applyTraceMetadata(): void {
|
||||
try {
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const trace = traceManager.getTrace(this.rootSpan.traceId);
|
||||
if (!trace) {
|
||||
return;
|
||||
}
|
||||
|
||||
const userPrompt = this.conversations.get(null)?.[0]?.content;
|
||||
if (typeof userPrompt === 'string') {
|
||||
trace.info.requestPreview = userPrompt.slice(0, MAX_PREVIEW_LENGTH);
|
||||
}
|
||||
const finalResponse = this.lastAssistantText.get(null);
|
||||
if (finalResponse) {
|
||||
trace.info.responsePreview = finalResponse.slice(0, MAX_PREVIEW_LENGTH);
|
||||
}
|
||||
|
||||
const metadata: Record<string, string> = { ...trace.info.traceMetadata };
|
||||
if (this.sessionId) {
|
||||
metadata[TraceMetadataKey.TRACE_SESSION] = this.sessionId;
|
||||
}
|
||||
const user = process.env.USER;
|
||||
if (user) {
|
||||
metadata[TraceMetadataKey.TRACE_USER] = user;
|
||||
}
|
||||
metadata[METADATA_KEY_WORKING_DIRECTORY] = process.cwd();
|
||||
if (this.permissionMode) {
|
||||
metadata[METADATA_KEY_PERMISSION_MODE] = this.permissionMode;
|
||||
}
|
||||
if (this.claudeCodeVersion) {
|
||||
metadata[METADATA_KEY_CLAUDE_CODE_VERSION] = this.claudeCodeVersion;
|
||||
}
|
||||
trace.info.traceMetadata = metadata;
|
||||
} catch (err) {
|
||||
console.error('[mlflow] Failed to apply trace metadata:', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hasThinking(content: string | ContentBlock[]): boolean {
|
||||
return (
|
||||
Array.isArray(content) &&
|
||||
content.some((b) => typeof b === 'object' && b != null && 'type' in b && b.type === 'thinking')
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip callbacks (functions are not serialisable and may close over large
|
||||
* objects) before recording options on the root span's inputs. The Agent SDK
|
||||
* options can include functions at several depths — `hooks[].hooks[]`,
|
||||
* `canUseTool`, MCP server entries — so we recursively drop any function
|
||||
* value rather than enumerating known fields.
|
||||
*/
|
||||
function sanitizeOptions(options: Record<string, unknown> | undefined): unknown {
|
||||
if (!options) {
|
||||
return undefined;
|
||||
}
|
||||
return sanitizeForSpan(options);
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* Wraps the Claude Agent SDK `query()` function so each call produces an
|
||||
* MLflow trace. The wrapper sets `forwardSubagentText: true` on options by
|
||||
* default so sub-agent activity flows through the parent message stream and
|
||||
* can be attributed via `parent_tool_use_id`.
|
||||
*
|
||||
* The actual span construction lives in LiveTracingContext, which mirrors the
|
||||
* span tree produced by processTranscript (CLI Stop-hook path).
|
||||
*/
|
||||
|
||||
import type { Query, SDKMessage } from '@anthropic-ai/claude-agent-sdk';
|
||||
|
||||
import {
|
||||
LiveTracingContext,
|
||||
type SDKAssistantLike,
|
||||
type SDKResultLike,
|
||||
type SDKSystemInit,
|
||||
type SDKUserLike,
|
||||
} from './liveTracing.js';
|
||||
|
||||
type QueryOptions = {
|
||||
forwardSubagentText?: boolean;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
type QueryParams = {
|
||||
prompt: string | AsyncIterable<unknown>;
|
||||
options?: QueryOptions;
|
||||
};
|
||||
|
||||
type QueryFunction = (params: QueryParams) => Query;
|
||||
|
||||
/**
|
||||
* Wrap a Claude Agent SDK `query` function so each invocation produces an
|
||||
* MLflow trace. The wrapped query has the same call signature and return
|
||||
* type as the original; tracing is fully transparent to the caller.
|
||||
*/
|
||||
export function createTracedQuery(queryFn: QueryFunction): QueryFunction {
|
||||
return (params: QueryParams): Query => {
|
||||
const ctx = new LiveTracingContext(initialPromptFor(params.prompt), params.options);
|
||||
|
||||
// For AsyncIterable prompts, wrap so user messages are observed for prompt
|
||||
// capture without consuming them — the SDK still receives the original
|
||||
// sequence. The wrapper records text content on the root span's inputs.
|
||||
const prompt =
|
||||
typeof params.prompt === 'string'
|
||||
? params.prompt
|
||||
: capturingPromptIterable(params.prompt, (text) => ctx.appendPromptText(text));
|
||||
|
||||
// Force forwardSubagentText: true so sub-agent inner messages flow
|
||||
// through the parent stream. Without it, the SDK only emits tool_use and
|
||||
// tool_result blocks from sub-agents (heartbeat), which strips LLM and
|
||||
// nested tool spans from the resulting trace. Forwarding is local-only
|
||||
// (no network cost), so we always enable it when the wrapper is in use.
|
||||
const options: QueryOptions = {
|
||||
...params.options,
|
||||
forwardSubagentText: true,
|
||||
};
|
||||
|
||||
const sdkQuery = queryFn({ ...params, prompt, options });
|
||||
|
||||
async function* tracedGenerator(): AsyncGenerator<SDKMessage, void> {
|
||||
// finalize() / finalizeError() are idempotent (guarded by `ended`), so
|
||||
// calling finalize() unconditionally in `finally` is safe even when the
|
||||
// error path already called finalizeError(). The `finally` block matters
|
||||
// because consumers can exit early (`break`) and skip the post-loop code.
|
||||
try {
|
||||
for await (const msg of sdkQuery) {
|
||||
dispatch(ctx, msg);
|
||||
yield msg;
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
await ctx.finalizeError(message);
|
||||
throw err;
|
||||
} finally {
|
||||
await ctx.finalize();
|
||||
}
|
||||
}
|
||||
|
||||
const wrappedIterator = tracedGenerator();
|
||||
|
||||
return new Proxy(sdkQuery, {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === Symbol.asyncIterator) {
|
||||
return () => wrappedIterator;
|
||||
}
|
||||
if (prop === 'next' || prop === 'return' || prop === 'throw') {
|
||||
return Reflect.get(wrappedIterator, prop, wrappedIterator);
|
||||
}
|
||||
if (prop === 'interrupt') {
|
||||
const method = Reflect.get(target, prop, receiver) as
|
||||
| ((...a: unknown[]) => unknown)
|
||||
| undefined;
|
||||
return (...args: unknown[]) => {
|
||||
// Fire-and-forget the trace flush so .interrupt() stays sync-compatible.
|
||||
void ctx.finalizeError('interrupted');
|
||||
return typeof method === 'function' ? method.call(target, ...args) : method;
|
||||
};
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function dispatch(ctx: LiveTracingContext, msg: SDKMessage): void {
|
||||
const m = msg as { type: string; subtype?: string };
|
||||
switch (m.type) {
|
||||
case 'system':
|
||||
if (m.subtype === 'init') {
|
||||
ctx.onSystemInit(msg as unknown as SDKSystemInit);
|
||||
}
|
||||
return;
|
||||
case 'assistant':
|
||||
ctx.onAssistantMessage(msg as unknown as SDKAssistantLike);
|
||||
return;
|
||||
case 'user':
|
||||
ctx.onUserMessage(msg as unknown as SDKUserLike);
|
||||
return;
|
||||
case 'result':
|
||||
ctx.onResultMessage(msg as unknown as SDKResultLike);
|
||||
return;
|
||||
default:
|
||||
// Ignore status, partial assistant, hook, task progress, etc. — these
|
||||
// are heartbeat signals that don't change the span tree.
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initial value for the root span's `prompt` input. For string prompts we have
|
||||
* it up front; for streaming prompts we start with a placeholder and append
|
||||
* each user message's text via `capturingPromptIterable`.
|
||||
*/
|
||||
function initialPromptFor(prompt: string | AsyncIterable<unknown>): string {
|
||||
return typeof prompt === 'string' ? prompt : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an AsyncIterable of SDK user messages so each yielded message's text
|
||||
* content is forwarded to `onText`. Used to record streaming prompts on the
|
||||
* root span's inputs as they arrive; the underlying iterable is passed through
|
||||
* unchanged to the SDK.
|
||||
*/
|
||||
function capturingPromptIterable(
|
||||
source: AsyncIterable<unknown>,
|
||||
onText: (text: string) => void,
|
||||
): AsyncIterable<unknown> {
|
||||
return {
|
||||
[Symbol.asyncIterator](): AsyncIterator<unknown> {
|
||||
const sourceIter = source[Symbol.asyncIterator]();
|
||||
return {
|
||||
async next(): Promise<IteratorResult<unknown>> {
|
||||
const result = await sourceIter.next();
|
||||
if (!result.done) {
|
||||
const text = extractUserText(result.value);
|
||||
if (text) {
|
||||
onText(text);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
},
|
||||
return: sourceIter.return ? sourceIter.return.bind(sourceIter) : undefined,
|
||||
throw: sourceIter.throw ? sourceIter.throw.bind(sourceIter) : undefined,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function extractUserText(value: unknown): string | undefined {
|
||||
if (typeof value !== 'object' || value == null) {
|
||||
return undefined;
|
||||
}
|
||||
const v = value as { type?: string; message?: { content?: unknown } };
|
||||
if (v.type !== 'user') {
|
||||
return undefined;
|
||||
}
|
||||
const content = v.message?.content;
|
||||
if (typeof content === 'string') {
|
||||
return content;
|
||||
}
|
||||
if (Array.isArray(content)) {
|
||||
const text = content
|
||||
.filter((b): b is { type: 'text'; text: string } => {
|
||||
return typeof b === 'object' && b != null && (b as { type?: string }).type === 'text';
|
||||
})
|
||||
.map((b) => b.text)
|
||||
.join('');
|
||||
return text || undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,559 @@
|
||||
import { existsSync } from 'node:fs';
|
||||
import { resolve, dirname, basename } from 'node:path';
|
||||
|
||||
import {
|
||||
startSpan,
|
||||
flushTraces,
|
||||
InMemoryTraceManager,
|
||||
SpanType,
|
||||
SpanAttributeKey,
|
||||
TraceMetadataKey,
|
||||
type LiveSpan,
|
||||
} from '@mlflow/core';
|
||||
|
||||
import type { SubagentGroup, TokenUsage, ToolResultInfo, TranscriptEntry } from './types.js';
|
||||
import {
|
||||
extractTextContent,
|
||||
findFinalAssistantResponse,
|
||||
findLastUserMessageIndex,
|
||||
getNextTimestampNs,
|
||||
parseTimestampToNs,
|
||||
readTranscript,
|
||||
} from './transcript.js';
|
||||
import {
|
||||
MAX_PREVIEW_LENGTH,
|
||||
METADATA_KEY_CLAUDE_CODE_VERSION,
|
||||
METADATA_KEY_PERMISSION_MODE,
|
||||
METADATA_KEY_WORKING_DIRECTORY,
|
||||
buildUsageDict,
|
||||
extractContentAndTools,
|
||||
} from './_internal.js';
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
const NANOSECONDS_PER_MS = 1e6;
|
||||
const NANOSECONDS_PER_S = 1e9;
|
||||
|
||||
// ============================================================================
|
||||
// Tool result finding
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Find tool results following the current assistant response.
|
||||
* Returns a mapping from tool_use_id to result info.
|
||||
*/
|
||||
function findToolResults(
|
||||
transcript: TranscriptEntry[],
|
||||
startIdx: number,
|
||||
): Record<string, ToolResultInfo> {
|
||||
const results: Record<string, ToolResultInfo> = {};
|
||||
// Claude Code splits a single assistant turn into multiple JSONL entries
|
||||
// (one per content block) that share the same message.id. Treat them as
|
||||
// one turn so parallel tool_uses in the same turn all find their results.
|
||||
const currentMessageId = transcript[startIdx]?.message?.id;
|
||||
|
||||
for (let i = startIdx + 1; i < transcript.length; i++) {
|
||||
const entry = transcript[i];
|
||||
if (entry.type === 'assistant') {
|
||||
if (currentMessageId && entry.message?.id === currentMessageId) {
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (entry.type !== 'user') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Entry-level toolUseResult (used in real Claude Code transcripts)
|
||||
const entryToolUseResult =
|
||||
entry.toolUseResult && typeof entry.toolUseResult === 'object' ? entry.toolUseResult : {};
|
||||
|
||||
const content = entry.message?.content;
|
||||
if (!Array.isArray(content)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const part of content) {
|
||||
if (typeof part !== 'object' || part == null || !('type' in part)) {
|
||||
continue;
|
||||
}
|
||||
if (part.type !== 'tool_result') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const toolResult = part as {
|
||||
type: 'tool_result';
|
||||
tool_use_id?: string;
|
||||
content?: string;
|
||||
is_error?: boolean;
|
||||
toolUseResult?: { agentId?: string };
|
||||
};
|
||||
|
||||
const toolUseId = toolResult.tool_use_id;
|
||||
if (!toolUseId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check both entry-level and content-level toolUseResult for agentId
|
||||
const partToolUseResult = toolResult.toolUseResult ?? {};
|
||||
const agentId = entryToolUseResult.agentId ?? partToolUseResult.agentId;
|
||||
|
||||
results[toolUseId] = {
|
||||
content: toolResult.content ?? '',
|
||||
isError: toolResult.is_error ?? false,
|
||||
agentId,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Input message reconstruction
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Get all messages between the previous text-bearing assistant response
|
||||
* and the current one, for use as LLM span inputs.
|
||||
*/
|
||||
function getInputMessages(
|
||||
transcript: TranscriptEntry[],
|
||||
currentIdx: number,
|
||||
): Array<{ role: string; content: unknown }> {
|
||||
const messages: Array<{ role: string; content: unknown }> = [];
|
||||
|
||||
for (let i = currentIdx - 1; i >= 0; i--) {
|
||||
const entry = transcript[i];
|
||||
const msg = entry.message;
|
||||
|
||||
// Stop at a previous assistant entry that has text content (previous LLM span)
|
||||
if (entry.type === 'assistant' && msg) {
|
||||
const content = msg.content;
|
||||
let hasText = false;
|
||||
if (typeof content === 'string') {
|
||||
hasText = content.trim().length > 0;
|
||||
} else if (Array.isArray(content)) {
|
||||
hasText = content.some(
|
||||
(p) => typeof p === 'object' && p != null && 'type' in p && p.type === 'text',
|
||||
);
|
||||
}
|
||||
if (hasText) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Include steer messages (queue-operation enqueue) as user messages
|
||||
if (entry.type === 'queue-operation' && entry.operation === 'enqueue' && entry.content) {
|
||||
messages.push({ role: 'user', content: entry.content });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (msg?.role && msg?.content) {
|
||||
messages.push({ role: msg.role, content: msg.content });
|
||||
}
|
||||
}
|
||||
|
||||
messages.reverse();
|
||||
return messages;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Token usage
|
||||
// ============================================================================
|
||||
|
||||
function setTokenUsageAttribute(span: LiveSpan, usage: TokenUsage | undefined): void {
|
||||
if (!usage) {
|
||||
return;
|
||||
}
|
||||
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, buildUsageDict(usage));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Sub-agent handling
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Group progress entries by parentToolUseID.
|
||||
*/
|
||||
function collectSubagentGroups(
|
||||
transcript: TranscriptEntry[],
|
||||
startIdx: number,
|
||||
): Record<string, SubagentGroup> {
|
||||
const groups: Record<string, SubagentGroup> = {};
|
||||
|
||||
for (let i = startIdx; i < transcript.length; i++) {
|
||||
const entry = transcript[i];
|
||||
if (entry.type !== 'progress') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const data = entry.data;
|
||||
if (!data?.message || typeof data.message !== 'object') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const parentToolId = entry.parentToolUseID;
|
||||
if (!parentToolId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!groups[parentToolId]) {
|
||||
groups[parentToolId] = {
|
||||
prompt: data.prompt ?? '',
|
||||
messages: [],
|
||||
timestamp: entry.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
groups[parentToolId].messages.push(data.message);
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the sub-agent transcript file path from the main transcript path.
|
||||
*/
|
||||
function getSubagentTranscriptPath(
|
||||
transcriptPath: string | undefined,
|
||||
agentId: string | undefined,
|
||||
): string | null {
|
||||
if (!transcriptPath || !agentId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Session dir = main transcript path without .jsonl extension
|
||||
const dir = dirname(transcriptPath);
|
||||
const base = basename(transcriptPath, '.jsonl');
|
||||
const subagentPath = resolve(dir, base, 'subagents', `agent-${agentId}.jsonl`);
|
||||
|
||||
if (existsSync(subagentPath)) {
|
||||
return subagentPath;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an AGENT wrapper span for a sub-agent's execution.
|
||||
*/
|
||||
function createAgentWrapperSpan(
|
||||
parentSpan: LiveSpan,
|
||||
toolInput: Record<string, unknown>,
|
||||
startNs: number,
|
||||
): LiveSpan {
|
||||
const subagentType = (toolInput.subagent_type as string) ?? '';
|
||||
const description = (toolInput.description as string) ?? '';
|
||||
const prompt = (toolInput.prompt as string) ?? '';
|
||||
const agentName = subagentType ? `subagent_${subagentType}` : 'subagent';
|
||||
|
||||
return startSpan({
|
||||
name: agentName,
|
||||
parent: parentSpan,
|
||||
spanType: SpanType.AGENT,
|
||||
startTimeNs: startNs,
|
||||
inputs: { prompt, description },
|
||||
attributes: { subagent_type: subagentType },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create LLM and tool spans for a sub-agent's inner messages (progress-based).
|
||||
*/
|
||||
function createSubagentSpans(
|
||||
parentSpan: LiveSpan,
|
||||
group: SubagentGroup,
|
||||
startNs: number,
|
||||
totalDurationNs: number,
|
||||
toolInput: Record<string, unknown>,
|
||||
): void {
|
||||
const innerMessages = group.messages;
|
||||
if (!innerMessages.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const agentSpan = createAgentWrapperSpan(parentSpan, toolInput, startNs);
|
||||
|
||||
// Find first assistant message index
|
||||
let firstAssistantIdx = 0;
|
||||
for (let idx = 0; idx < innerMessages.length; idx++) {
|
||||
if (innerMessages[idx].type === 'assistant') {
|
||||
firstAssistantIdx = idx;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
createLlmAndToolSpans(agentSpan, innerMessages, firstAssistantIdx);
|
||||
agentSpan.end({ endTimeNs: startNs + totalDurationNs });
|
||||
}
|
||||
|
||||
/**
|
||||
* Create LLM and tool spans from a sub-agent's separate transcript file.
|
||||
*/
|
||||
function createSubagentSpansFromFile(
|
||||
parentSpan: LiveSpan,
|
||||
subagentTranscriptPath: string,
|
||||
startNs: number,
|
||||
totalDurationNs: number,
|
||||
toolInput: Record<string, unknown>,
|
||||
): void {
|
||||
try {
|
||||
const subagentTranscript = readTranscript(subagentTranscriptPath);
|
||||
if (!subagentTranscript.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const agentSpan = createAgentWrapperSpan(parentSpan, toolInput, startNs);
|
||||
|
||||
let firstAssistantIdx = 0;
|
||||
for (let idx = 0; idx < subagentTranscript.length; idx++) {
|
||||
if (subagentTranscript[idx].type === 'assistant') {
|
||||
firstAssistantIdx = idx;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
createLlmAndToolSpans(agentSpan, subagentTranscript, firstAssistantIdx, subagentTranscriptPath);
|
||||
agentSpan.end({ endTimeNs: startNs + totalDurationNs });
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[mlflow] Failed to process sub-agent transcript ${subagentTranscriptPath}:`,
|
||||
err,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Core span creation
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Create LLM and tool spans for assistant responses with proper timing.
|
||||
*/
|
||||
function createLlmAndToolSpans(
|
||||
parentSpan: LiveSpan,
|
||||
transcript: TranscriptEntry[],
|
||||
startIdx: number,
|
||||
transcriptPath?: string,
|
||||
): void {
|
||||
const subagentGroups = collectSubagentGroups(transcript, startIdx);
|
||||
|
||||
for (let i = startIdx; i < transcript.length; i++) {
|
||||
const entry = transcript[i];
|
||||
if (entry.type !== 'assistant') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const timestampNs = parseTimestampToNs(entry.timestamp);
|
||||
if (!timestampNs) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const nextTimestampNs = getNextTimestampNs(transcript, i);
|
||||
const durationNs = nextTimestampNs
|
||||
? nextTimestampNs - timestampNs
|
||||
: Math.floor(1000 * NANOSECONDS_PER_MS); // 1 second default
|
||||
|
||||
const msg = entry.message;
|
||||
if (!msg) {
|
||||
continue;
|
||||
}
|
||||
const content = msg.content ?? [];
|
||||
const usage = msg.usage;
|
||||
|
||||
const [textContent, toolUses] = extractContentAndTools(content);
|
||||
|
||||
// Create LLM span if there's text content (no tools)
|
||||
if (textContent.trim() && !toolUses.length) {
|
||||
const messages = getInputMessages(transcript, i);
|
||||
const model = msg.model ?? 'unknown';
|
||||
|
||||
const llmSpan = startSpan({
|
||||
name: 'llm',
|
||||
parent: parentSpan,
|
||||
spanType: SpanType.LLM,
|
||||
startTimeNs: timestampNs,
|
||||
inputs: { model, messages },
|
||||
attributes: {
|
||||
model,
|
||||
'mlflow.llm.model': model,
|
||||
[SpanAttributeKey.MESSAGE_FORMAT]: 'anthropic',
|
||||
},
|
||||
});
|
||||
|
||||
setTokenUsageAttribute(llmSpan, usage);
|
||||
|
||||
llmSpan.setOutputs({
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
content,
|
||||
});
|
||||
llmSpan.end({ endTimeNs: timestampNs + durationNs });
|
||||
}
|
||||
|
||||
// Create tool spans with proportional timing
|
||||
if (toolUses.length) {
|
||||
const toolResults = findToolResults(transcript, i);
|
||||
const toolDurationNs = Math.floor(durationNs / toolUses.length);
|
||||
|
||||
for (let idx = 0; idx < toolUses.length; idx++) {
|
||||
const toolUse = toolUses[idx];
|
||||
const toolStartNs = timestampNs + idx * toolDurationNs;
|
||||
const toolUseId = toolUse.id ?? '';
|
||||
const toolResultInfo = toolResults[toolUseId];
|
||||
const toolResult = toolResultInfo?.content ?? 'No result found';
|
||||
const toolName = toolUse.name ?? 'unknown';
|
||||
|
||||
const toolSpan = startSpan({
|
||||
name: `tool_${toolName}`,
|
||||
parent: parentSpan,
|
||||
spanType: SpanType.TOOL,
|
||||
startTimeNs: toolStartNs,
|
||||
inputs: toolUse.input ?? {},
|
||||
attributes: {
|
||||
tool_name: toolName,
|
||||
tool_id: toolUseId,
|
||||
},
|
||||
});
|
||||
|
||||
// If this is a Task tool, try to read sub-agent transcript
|
||||
const agentId = toolResultInfo?.agentId;
|
||||
const subagentPath = getSubagentTranscriptPath(transcriptPath, agentId);
|
||||
const toolInput = toolUse.input ?? {};
|
||||
|
||||
if (subagentPath) {
|
||||
createSubagentSpansFromFile(
|
||||
toolSpan,
|
||||
subagentPath,
|
||||
toolStartNs,
|
||||
toolDurationNs,
|
||||
toolInput,
|
||||
);
|
||||
} else if (subagentGroups[toolUseId]) {
|
||||
createSubagentSpans(
|
||||
toolSpan,
|
||||
subagentGroups[toolUseId],
|
||||
toolStartNs,
|
||||
toolDurationNs,
|
||||
toolInput,
|
||||
);
|
||||
}
|
||||
|
||||
toolSpan.setOutputs({ result: toolResult });
|
||||
|
||||
if (toolResultInfo?.isError) {
|
||||
const errorMessage = toolResult || 'Tool execution failed';
|
||||
toolSpan.recordException(new Error(errorMessage));
|
||||
}
|
||||
|
||||
toolSpan.end({ endTimeNs: toolStartNs + toolDurationNs });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main entry point
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Process a Claude conversation transcript and create an MLflow trace with spans.
|
||||
*/
|
||||
export async function processTranscript(transcriptPath: string, sessionId?: string): Promise<void> {
|
||||
try {
|
||||
const transcript = readTranscript(transcriptPath);
|
||||
if (!transcript.length) {
|
||||
console.error('[mlflow] Empty transcript, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
const lastUserIdx = findLastUserMessageIndex(transcript);
|
||||
if (lastUserIdx == null) {
|
||||
console.error('[mlflow] No user message found in transcript');
|
||||
return;
|
||||
}
|
||||
|
||||
const lastUserEntry = transcript[lastUserIdx];
|
||||
const lastUserPrompt = lastUserEntry.message?.content ?? '';
|
||||
const userPromptText = extractTextContent(lastUserPrompt);
|
||||
|
||||
if (!sessionId) {
|
||||
sessionId = `claude-${new Date().toISOString().replace(/[:.]/g, '').slice(0, 15)}`;
|
||||
}
|
||||
|
||||
const convStartNs = parseTimestampToNs(lastUserEntry.timestamp);
|
||||
|
||||
const parentSpan = startSpan({
|
||||
name: 'claude_code_conversation',
|
||||
inputs: { prompt: userPromptText },
|
||||
startTimeNs: convStartNs ?? undefined,
|
||||
spanType: SpanType.AGENT,
|
||||
});
|
||||
|
||||
// Create spans for all assistant responses and tool uses
|
||||
createLlmAndToolSpans(parentSpan, transcript, lastUserIdx + 1, transcriptPath);
|
||||
|
||||
// Find final response for preview
|
||||
const finalResponse = findFinalAssistantResponse(transcript, lastUserIdx + 1);
|
||||
|
||||
// Set trace previews and metadata
|
||||
try {
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const trace = traceManager.getTrace(parentSpan.traceId);
|
||||
if (trace) {
|
||||
if (userPromptText) {
|
||||
trace.info.requestPreview = userPromptText.slice(0, MAX_PREVIEW_LENGTH);
|
||||
}
|
||||
if (finalResponse) {
|
||||
trace.info.responsePreview = finalResponse.slice(0, MAX_PREVIEW_LENGTH);
|
||||
}
|
||||
|
||||
const metadata: Record<string, string> = {
|
||||
...trace.info.traceMetadata,
|
||||
[TraceMetadataKey.TRACE_SESSION]: sessionId,
|
||||
[TraceMetadataKey.TRACE_USER]: process.env.USER ?? '',
|
||||
[METADATA_KEY_WORKING_DIRECTORY]: process.cwd(),
|
||||
};
|
||||
|
||||
// Capture permission mode
|
||||
const permissionMode = lastUserEntry.permissionMode;
|
||||
if (permissionMode) {
|
||||
metadata[METADATA_KEY_PERMISSION_MODE] = permissionMode;
|
||||
}
|
||||
|
||||
// Extract Claude Code version from transcript entries
|
||||
const claudeCodeVersion = transcript.reduce<string | undefined>(
|
||||
(found, entry) => found ?? entry.version,
|
||||
undefined,
|
||||
);
|
||||
if (claudeCodeVersion) {
|
||||
metadata[METADATA_KEY_CLAUDE_CODE_VERSION] = claudeCodeVersion;
|
||||
}
|
||||
|
||||
trace.info.traceMetadata = metadata;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[mlflow] Failed to update trace metadata:', err);
|
||||
}
|
||||
|
||||
// Calculate end time
|
||||
const lastEntry = transcript[transcript.length - 1];
|
||||
let convEndNs = parseTimestampToNs(lastEntry.timestamp);
|
||||
if (!convEndNs || (convStartNs && convEndNs <= convStartNs)) {
|
||||
convEndNs = (convStartNs ?? 0) + Math.floor(10 * NANOSECONDS_PER_S);
|
||||
}
|
||||
|
||||
const outputs: Record<string, string> = { status: 'completed' };
|
||||
if (finalResponse) {
|
||||
outputs.response = finalResponse;
|
||||
}
|
||||
parentSpan.setOutputs(outputs);
|
||||
parentSpan.end({ endTimeNs: convEndNs });
|
||||
|
||||
await flushTraces();
|
||||
} catch (err) {
|
||||
console.error('[mlflow] Error processing transcript:', err);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
import type { TranscriptEntry } from './types.js';
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
const NANOSECONDS_PER_MS = 1e6;
|
||||
const NANOSECONDS_PER_S = 1e9;
|
||||
|
||||
// ============================================================================
|
||||
// JSONL parsing
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Read and parse a Claude Code transcript from a JSONL file.
|
||||
*/
|
||||
export function readTranscript(path: string): TranscriptEntry[] {
|
||||
const content = readFileSync(path, 'utf-8');
|
||||
return content
|
||||
.split('\n')
|
||||
.filter((line) => line.trim())
|
||||
.map((line) => JSON.parse(line) as TranscriptEntry);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Timestamp utilities
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Convert various timestamp formats to nanoseconds since Unix epoch.
|
||||
* Handles ISO strings, Unix seconds, milliseconds, and nanoseconds.
|
||||
*/
|
||||
export function parseTimestampToNs(timestamp: string | number | undefined | null): number | null {
|
||||
if (!timestamp) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof timestamp === 'string') {
|
||||
try {
|
||||
const dt = new Date(timestamp);
|
||||
if (isNaN(dt.getTime())) {
|
||||
return null;
|
||||
}
|
||||
return Math.floor(dt.getTime() * NANOSECONDS_PER_MS);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof timestamp === 'number') {
|
||||
// Unix seconds (< 1e10, e.g. 1705312245)
|
||||
if (timestamp < 1e10) {
|
||||
return Math.floor(timestamp * NANOSECONDS_PER_S);
|
||||
}
|
||||
// Milliseconds (< 1e13, e.g. 1705312245123)
|
||||
if (timestamp < 1e13) {
|
||||
return Math.floor(timestamp * NANOSECONDS_PER_MS);
|
||||
}
|
||||
// Already nanoseconds
|
||||
return Math.floor(timestamp);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Content extraction
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Extract text content from Claude message content (string or content block array).
|
||||
*/
|
||||
export function extractTextContent(content: unknown): string {
|
||||
if (Array.isArray(content)) {
|
||||
const textParts = content
|
||||
.filter(
|
||||
(part): part is { type: 'text'; text: string } =>
|
||||
typeof part === 'object' &&
|
||||
part != null &&
|
||||
'type' in part &&
|
||||
(part as { type: string }).type === 'text',
|
||||
)
|
||||
.map((part) => part.text);
|
||||
return textParts.join('\n');
|
||||
}
|
||||
if (typeof content === 'string') {
|
||||
return content;
|
||||
}
|
||||
return String(content);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Transcript navigation
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Find the index of the last actual user message, skipping tool results,
|
||||
* skill injections, and empty messages.
|
||||
*/
|
||||
export function findLastUserMessageIndex(transcript: TranscriptEntry[]): number | null {
|
||||
for (let i = transcript.length - 1; i >= 0; i--) {
|
||||
const entry = transcript[i];
|
||||
if (entry.type !== 'user' || entry.toolUseResult || entry.isCompactSummary) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip skill content injections: a user message immediately following
|
||||
// a Skill tool result (which has toolUseResult with commandName)
|
||||
if (i > 0) {
|
||||
const prevToolResult = transcript[i - 1].toolUseResult;
|
||||
if (
|
||||
prevToolResult &&
|
||||
typeof prevToolResult === 'object' &&
|
||||
'commandName' in prevToolResult &&
|
||||
prevToolResult.commandName
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const msg = entry.message;
|
||||
if (!msg) {
|
||||
continue;
|
||||
}
|
||||
const content = msg.content;
|
||||
|
||||
// Skip tool result messages
|
||||
if (Array.isArray(content) && content.length > 0) {
|
||||
const first = content[0];
|
||||
if (typeof first === 'object' && first != null && 'type' in first) {
|
||||
if ((first as { type: string }).type === 'tool_result') {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Skip local command stdout
|
||||
if (typeof content === 'string' && content.includes('<local-command-stdout>')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip empty content
|
||||
if (!content || (typeof content === 'string' && content.trim() === '')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return i;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the final text response from the assistant after the given index.
|
||||
*/
|
||||
export function findFinalAssistantResponse(
|
||||
transcript: TranscriptEntry[],
|
||||
startIdx: number,
|
||||
): string | null {
|
||||
let finalResponse: string | null = null;
|
||||
|
||||
for (let i = startIdx; i < transcript.length; i++) {
|
||||
const entry = transcript[i];
|
||||
if (entry.type !== 'assistant') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = entry.message?.content;
|
||||
if (!Array.isArray(content)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const part of content) {
|
||||
if (
|
||||
typeof part === 'object' &&
|
||||
part != null &&
|
||||
'type' in part &&
|
||||
part.type === 'text' &&
|
||||
'text' in part
|
||||
) {
|
||||
const text = (part as { type: 'text'; text: string }).text;
|
||||
if (text.trim()) {
|
||||
finalResponse = text;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return finalResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the timestamp (in ns) of the next transcript entry that has one.
|
||||
*/
|
||||
export function getNextTimestampNs(
|
||||
transcript: TranscriptEntry[],
|
||||
currentIdx: number,
|
||||
): number | null {
|
||||
for (let i = currentIdx + 1; i < transcript.length; i++) {
|
||||
const ts = transcript[i].timestamp;
|
||||
if (ts) {
|
||||
return parseTimestampToNs(ts);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* TypeScript interfaces for Claude Code transcript entries.
|
||||
*/
|
||||
|
||||
// ============================================================================
|
||||
// Content block types
|
||||
// ============================================================================
|
||||
|
||||
export interface TextBlock {
|
||||
type: 'text';
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface ThinkingBlock {
|
||||
type: 'thinking';
|
||||
thinking: string;
|
||||
}
|
||||
|
||||
export interface ToolUseBlock {
|
||||
type: 'tool_use';
|
||||
id: string;
|
||||
name: string;
|
||||
input: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ToolResultBlock {
|
||||
type: 'tool_result';
|
||||
tool_use_id: string;
|
||||
content: string;
|
||||
is_error?: boolean;
|
||||
toolUseResult?: {
|
||||
status?: string;
|
||||
agentId?: string;
|
||||
totalDurationMs?: number;
|
||||
};
|
||||
}
|
||||
|
||||
export type ContentBlock = TextBlock | ThinkingBlock | ToolUseBlock | ToolResultBlock;
|
||||
|
||||
// ============================================================================
|
||||
// Token usage
|
||||
// ============================================================================
|
||||
|
||||
export interface TokenUsage {
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
cache_creation_input_tokens?: number;
|
||||
cache_read_input_tokens?: number;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Message content
|
||||
// ============================================================================
|
||||
|
||||
export interface MessageContent {
|
||||
role: 'user' | 'assistant';
|
||||
content: string | ContentBlock[];
|
||||
id?: string;
|
||||
model?: string;
|
||||
usage?: TokenUsage;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Transcript entries
|
||||
// ============================================================================
|
||||
|
||||
export interface TranscriptEntry {
|
||||
type: 'user' | 'assistant' | 'progress' | 'queue-operation';
|
||||
message?: MessageContent;
|
||||
timestamp?: string | number;
|
||||
version?: string;
|
||||
permissionMode?: string;
|
||||
toolUseResult?: ToolUseResultInfo;
|
||||
sessionId?: string;
|
||||
parentToolUseID?: string;
|
||||
data?: ProgressData;
|
||||
operation?: string;
|
||||
content?: string;
|
||||
agentId?: string;
|
||||
isCompactSummary?: boolean;
|
||||
}
|
||||
|
||||
export interface ToolUseResultInfo {
|
||||
success?: boolean;
|
||||
commandName?: string;
|
||||
agentId?: string;
|
||||
status?: string;
|
||||
totalDurationMs?: number;
|
||||
}
|
||||
|
||||
export interface ProgressData {
|
||||
type?: string;
|
||||
agentId?: string;
|
||||
prompt?: string;
|
||||
message?: TranscriptEntry;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Hook input/output
|
||||
// ============================================================================
|
||||
|
||||
export interface StopHookInput {
|
||||
session_id: string;
|
||||
transcript_path: string;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Internal types
|
||||
// ============================================================================
|
||||
|
||||
export interface ToolResultInfo {
|
||||
content: string;
|
||||
isError: boolean;
|
||||
agentId?: string;
|
||||
}
|
||||
|
||||
export interface SubagentGroup {
|
||||
prompt: string;
|
||||
messages: TranscriptEntry[];
|
||||
timestamp?: string | number;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* Read all data from stdin and parse as JSON.
|
||||
*/
|
||||
export function readStdin<T>(): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
process.stdin.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
process.stdin.on('end', () => {
|
||||
try {
|
||||
const raw = Buffer.concat(chunks).toString('utf-8');
|
||||
resolve(JSON.parse(raw) as T);
|
||||
} catch (err) {
|
||||
reject(new Error(`Failed to parse stdin as JSON: ${String(err)}`));
|
||||
}
|
||||
});
|
||||
process.stdin.on('error', reject);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import {
|
||||
ensureInitialized,
|
||||
getEffectiveTracingConfig,
|
||||
parseTraceLocation,
|
||||
resolveSettingsPath,
|
||||
writeTracingSettings,
|
||||
} from '../src/config';
|
||||
|
||||
const initMock = jest.fn();
|
||||
const getExperimentByNameMock = jest.fn();
|
||||
const createExperimentMock = jest.fn();
|
||||
const createAuthProviderMock = jest.fn((_options: unknown) => ({
|
||||
getHost: () => 'http://localhost:5000',
|
||||
}));
|
||||
|
||||
jest.mock('@mlflow/core', () => ({
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
init: jest.fn((...args: unknown[]) => initMock(...(args as Parameters<typeof initMock>))),
|
||||
createAuthProvider: jest.fn((...args: unknown[]) =>
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
createAuthProviderMock(...(args as Parameters<typeof createAuthProviderMock>)),
|
||||
),
|
||||
MlflowClient: jest.fn().mockImplementation(() => ({
|
||||
getExperimentByName: jest.fn((...args: unknown[]) =>
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
getExperimentByNameMock(...(args as Parameters<typeof getExperimentByNameMock>)),
|
||||
),
|
||||
createExperiment: jest.fn((...args: unknown[]) =>
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
createExperimentMock(...(args as Parameters<typeof createExperimentMock>)),
|
||||
),
|
||||
})),
|
||||
}));
|
||||
|
||||
describe('Claude Code tracing config', () => {
|
||||
let tmpHome: string;
|
||||
let tmpCwd: string;
|
||||
let originalCwd: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpHome = mkdtempSync(join(tmpdir(), 'claude-config-home-'));
|
||||
tmpCwd = mkdtempSync(join(tmpdir(), 'claude-config-cwd-'));
|
||||
originalCwd = process.cwd();
|
||||
delete process.env.MLFLOW_CLAUDE_TRACING_ENABLED;
|
||||
delete process.env.MLFLOW_TRACKING_URI;
|
||||
delete process.env.MLFLOW_EXPERIMENT_ID;
|
||||
delete process.env.MLFLOW_EXPERIMENT_NAME;
|
||||
delete process.env.MLFLOW_TRACE_LOCATION;
|
||||
initMock.mockReset();
|
||||
getExperimentByNameMock.mockReset();
|
||||
createExperimentMock.mockReset();
|
||||
createAuthProviderMock.mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.chdir(originalCwd);
|
||||
rmSync(tmpHome, { recursive: true, force: true });
|
||||
rmSync(tmpCwd, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('reads project settings and preserves experiment name when writing config', () => {
|
||||
const settingsPath = resolveSettingsPath(true, { home: tmpHome, cwd: tmpCwd });
|
||||
writeTracingSettings(settingsPath, {
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '42',
|
||||
experimentName: 'claude-code-traces',
|
||||
});
|
||||
|
||||
const stored = JSON.parse(readFileSync(settingsPath, 'utf-8')) as {
|
||||
env: Record<string, string>;
|
||||
};
|
||||
expect(stored.env).toMatchObject({
|
||||
MLFLOW_CLAUDE_TRACING_ENABLED: 'true',
|
||||
MLFLOW_TRACKING_URI: 'http://localhost:5000',
|
||||
MLFLOW_EXPERIMENT_ID: '42',
|
||||
MLFLOW_EXPERIMENT_NAME: 'claude-code-traces',
|
||||
});
|
||||
|
||||
expect(getEffectiveTracingConfig({ home: tmpHome, cwd: tmpCwd })).toMatchObject({
|
||||
enabled: true,
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '42',
|
||||
experimentName: 'claude-code-traces',
|
||||
source: 'project',
|
||||
});
|
||||
});
|
||||
|
||||
it('lets environment variables override saved settings', () => {
|
||||
writeTracingSettings(resolveSettingsPath(false, { home: tmpHome, cwd: tmpCwd }), {
|
||||
trackingUri: 'http://saved.example',
|
||||
experimentId: '7',
|
||||
experimentName: 'saved-exp',
|
||||
});
|
||||
|
||||
process.env.MLFLOW_TRACKING_URI = 'http://override.example';
|
||||
process.env.MLFLOW_EXPERIMENT_ID = '99';
|
||||
process.env.MLFLOW_CLAUDE_TRACING_ENABLED = 'true';
|
||||
|
||||
expect(getEffectiveTracingConfig({ home: tmpHome, cwd: tmpCwd })).toMatchObject({
|
||||
enabled: true,
|
||||
trackingUri: 'http://override.example',
|
||||
experimentId: '99',
|
||||
experimentName: 'saved-exp',
|
||||
source: 'environment',
|
||||
});
|
||||
});
|
||||
|
||||
it('resolves experiment name to an ID and initializes the SDK', async () => {
|
||||
writeTracingSettings(resolveSettingsPath(true, { home: tmpHome, cwd: tmpCwd }), {
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '',
|
||||
experimentName: 'claude-code-traces',
|
||||
});
|
||||
process.chdir(tmpCwd);
|
||||
getExperimentByNameMock.mockResolvedValueOnce({
|
||||
experimentId: '123',
|
||||
name: 'claude-code-traces',
|
||||
});
|
||||
|
||||
await expect(ensureInitialized()).resolves.toBe(true);
|
||||
expect(createAuthProviderMock).toHaveBeenCalledWith({ trackingUri: 'http://localhost:5000' });
|
||||
expect(initMock).toHaveBeenCalledWith({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '123',
|
||||
});
|
||||
});
|
||||
|
||||
it('creates the experiment when a configured experiment name does not exist', async () => {
|
||||
writeTracingSettings(resolveSettingsPath(true, { home: tmpHome, cwd: tmpCwd }), {
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '',
|
||||
experimentName: 'new-experiment',
|
||||
});
|
||||
process.chdir(tmpCwd);
|
||||
getExperimentByNameMock.mockResolvedValueOnce(null);
|
||||
createExperimentMock.mockResolvedValueOnce('456');
|
||||
|
||||
await expect(ensureInitialized()).resolves.toBe(true);
|
||||
expect(createExperimentMock).toHaveBeenCalledWith('new-experiment');
|
||||
expect(initMock).toHaveBeenCalledWith({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '456',
|
||||
});
|
||||
});
|
||||
|
||||
it('parses a catalog.schema.table_prefix trace location', () => {
|
||||
expect(parseTraceLocation('cat.sch.pfx')).toEqual({
|
||||
catalogName: 'cat',
|
||||
schemaName: 'sch',
|
||||
tablePrefix: 'pfx',
|
||||
});
|
||||
expect(parseTraceLocation(' cat.sch.pfx ')).toEqual({
|
||||
catalogName: 'cat',
|
||||
schemaName: 'sch',
|
||||
tablePrefix: 'pfx',
|
||||
});
|
||||
expect(parseTraceLocation(undefined)).toBeNull();
|
||||
expect(parseTraceLocation('')).toBeNull();
|
||||
expect(parseTraceLocation('cat.sch')).toBeNull();
|
||||
expect(parseTraceLocation('cat.sch.pfx.extra')).toBeNull();
|
||||
expect(parseTraceLocation('cat..pfx')).toBeNull();
|
||||
});
|
||||
|
||||
it('persists and surfaces the trace location and passes it to init', async () => {
|
||||
const settingsPath = resolveSettingsPath(true, { home: tmpHome, cwd: tmpCwd });
|
||||
writeTracingSettings(settingsPath, {
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '42',
|
||||
traceLocation: 'my_catalog.my_schema.my_prefix',
|
||||
});
|
||||
process.chdir(tmpCwd);
|
||||
|
||||
const stored = JSON.parse(readFileSync(settingsPath, 'utf-8')) as {
|
||||
env: Record<string, string>;
|
||||
};
|
||||
expect(stored.env.MLFLOW_TRACE_LOCATION).toBe('my_catalog.my_schema.my_prefix');
|
||||
|
||||
expect(getEffectiveTracingConfig({ home: tmpHome, cwd: tmpCwd })).toMatchObject({
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '42',
|
||||
traceLocation: 'my_catalog.my_schema.my_prefix',
|
||||
});
|
||||
|
||||
await expect(ensureInitialized()).resolves.toBe(true);
|
||||
expect(initMock).toHaveBeenCalledWith({
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '42',
|
||||
traceLocation: {
|
||||
catalogName: 'my_catalog',
|
||||
schemaName: 'my_schema',
|
||||
tablePrefix: 'my_prefix',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('refuses to initialize when the trace location is malformed', async () => {
|
||||
writeTracingSettings(resolveSettingsPath(true, { home: tmpHome, cwd: tmpCwd }), {
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '42',
|
||||
traceLocation: 'not-a-valid-location',
|
||||
});
|
||||
process.chdir(tmpCwd);
|
||||
|
||||
await expect(ensureInitialized()).resolves.toBe(false);
|
||||
expect(initMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
{"type":"user","message":{"role":"user","content":"What is 2 + 2?"},"timestamp":"2025-01-15T10:00:00.000Z","sessionId":"test-session-123"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Let me calculate that for you."}]},"timestamp":"2025-01-15T10:00:01.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"tool_123","name":"Bash","input":{"command":"echo $((2 + 2))"}}]},"timestamp":"2025-01-15T10:00:02.000Z"}
|
||||
{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool_123","content":"4"}]},"timestamp":"2025-01-15T10:00:03.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"The answer is 4."}]},"timestamp":"2025-01-15T10:00:04.000Z"}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
{"type":"user","message":{"role":"user","content":"Search for auth"},"timestamp":"2025-01-15T10:00:02.500Z","agentId":"abc1234"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"sub_tool_grep","name":"Grep","input":{"pattern":"auth"}}]},"timestamp":"2025-01-15T10:00:03.000Z"}
|
||||
{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"sub_tool_grep","content":"auth.py:1: def auth()"}]},"timestamp":"2025-01-15T10:00:04.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"sub_tool_read","name":"Read","input":{"file_path":"auth.py"}}]},"timestamp":"2025-01-15T10:00:04.500Z"}
|
||||
{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"sub_tool_read","content":"def auth():\n pass"}]},"timestamp":"2025-01-15T10:00:05.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Found auth.py with auth function."}]},"timestamp":"2025-01-15T10:00:05.500Z"}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{"type":"user","message":{"role":"user","content":"Spawn four agents in parallel"},"timestamp":"2025-01-15T10:00:00.000Z"}
|
||||
{"type":"assistant","message":{"id":"msg_turn1","role":"assistant","content":[{"type":"text","text":"Launching agents."}]},"timestamp":"2025-01-15T10:00:01.000Z"}
|
||||
{"type":"assistant","message":{"id":"msg_turn1","role":"assistant","content":[{"type":"tool_use","id":"toolu_a","name":"Task","input":{"prompt":"Agent A","subagent_type":"Explore"}}]},"timestamp":"2025-01-15T10:00:02.000Z"}
|
||||
{"type":"assistant","message":{"id":"msg_turn1","role":"assistant","content":[{"type":"tool_use","id":"toolu_b","name":"Task","input":{"prompt":"Agent B","subagent_type":"Explore"}}]},"timestamp":"2025-01-15T10:00:03.000Z"}
|
||||
{"type":"assistant","message":{"id":"msg_turn1","role":"assistant","content":[{"type":"tool_use","id":"toolu_c","name":"Task","input":{"prompt":"Agent C","subagent_type":"Explore"}}]},"timestamp":"2025-01-15T10:00:04.000Z"}
|
||||
{"type":"assistant","message":{"id":"msg_turn1","role":"assistant","content":[{"type":"tool_use","id":"toolu_d","name":"Task","input":{"prompt":"Agent D","subagent_type":"Explore"}}]},"timestamp":"2025-01-15T10:00:05.000Z"}
|
||||
{"type":"user","toolUseResult":{"status":"completed","agentId":"agentA","totalDurationMs":1000},"message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_a","content":"A done"}]},"timestamp":"2025-01-15T10:00:10.000Z"}
|
||||
{"type":"user","toolUseResult":{"status":"completed","agentId":"agentB","totalDurationMs":1000},"message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_b","content":"B done"}]},"timestamp":"2025-01-15T10:00:11.000Z"}
|
||||
{"type":"user","toolUseResult":{"status":"completed","agentId":"agentC","totalDurationMs":1000},"message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_c","content":"C done"}]},"timestamp":"2025-01-15T10:00:12.000Z"}
|
||||
{"type":"user","toolUseResult":{"status":"completed","agentId":"agentD","totalDurationMs":1000},"message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_d","content":"D done"}]},"timestamp":"2025-01-15T10:00:13.000Z"}
|
||||
{"type":"assistant","message":{"id":"msg_turn2","role":"assistant","content":[{"type":"text","text":"All four done."}]},"timestamp":"2025-01-15T10:00:14.000Z"}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
{"type":"user","message":{"role":"user","content":"Search the codebase for auth"},"timestamp":"2025-01-15T10:00:00.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"I'll search for that."}]},"timestamp":"2025-01-15T10:00:01.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_task_file_001","name":"Task","input":{"prompt":"Search for auth","subagent_type":"Explore"}}]},"timestamp":"2025-01-15T10:00:02.000Z"}
|
||||
{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_task_file_001","content":"Found auth.py with auth function.","toolUseResult":{"status":"completed","agentId":"abc1234","totalDurationMs":5000}}]},"timestamp":"2025-01-15T10:00:06.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"I found the auth module."}]},"timestamp":"2025-01-15T10:00:07.000Z"}
|
||||
@@ -0,0 +1,8 @@
|
||||
{"type":"user","message":{"role":"user","content":"Search the codebase for auth"},"timestamp":"2025-01-15T10:00:00.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"I'll search for that."}]},"timestamp":"2025-01-15T10:00:01.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_task_001","name":"Task","input":{"prompt":"Search for auth","subagent_type":"Explore"}}]},"timestamp":"2025-01-15T10:00:02.000Z"}
|
||||
{"type":"progress","parentToolUseID":"toolu_task_001","toolUseID":"agent_msg_001","timestamp":"2025-01-15T10:00:03.000Z","data":{"type":"agent_progress","agentId":"sub_1","prompt":"Search for auth","message":{"type":"assistant","timestamp":"2025-01-15T10:00:03.000Z","message":{"role":"assistant","content":[{"type":"tool_use","id":"sub_tool_1","name":"Grep","input":{"pattern":"auth"}}]}}}}
|
||||
{"type":"progress","parentToolUseID":"toolu_task_001","toolUseID":"agent_msg_001","timestamp":"2025-01-15T10:00:04.000Z","data":{"type":"agent_progress","agentId":"sub_1","prompt":"Search for auth","message":{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"sub_tool_1","content":"auth.py:1: def auth()"}]}}}}
|
||||
{"type":"progress","parentToolUseID":"toolu_task_001","toolUseID":"agent_msg_001","timestamp":"2025-01-15T10:00:05.000Z","data":{"type":"agent_progress","agentId":"sub_1","prompt":"Search for auth","message":{"type":"assistant","timestamp":"2025-01-15T10:00:05.000Z","message":{"role":"assistant","content":[{"type":"text","text":"Found auth.py with auth function."}]}}}}
|
||||
{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_task_001","content":"Found auth.py"}]},"timestamp":"2025-01-15T10:00:06.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"I found the auth module."}]},"timestamp":"2025-01-15T10:00:07.000Z"}
|
||||
@@ -0,0 +1,4 @@
|
||||
{"type":"user","message":{"role":"user","content":"Delete all files"},"timestamp":"2025-01-15T10:00:00.000Z","permissionMode":"default"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_rejected_001","name":"Bash","input":{"command":"rm -rf /"}}]},"timestamp":"2025-01-15T10:00:01.000Z"}
|
||||
{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_rejected_001","content":"The user doesn't want to proceed with this tool use.","is_error":true}]},"timestamp":"2025-01-15T10:00:02.000Z"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"I understand, I won't proceed with that."}]},"timestamp":"2025-01-15T10:00:03.000Z"}
|
||||
@@ -0,0 +1,2 @@
|
||||
{"type":"user","message":{"role":"user","content":"Hello Claude!"},"timestamp":"2025-01-15T10:00:00.000Z","sessionId":"test-session-usage"}
|
||||
{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Hello! How can I help you today?"}],"model":"claude-sonnet-4-20250514","usage":{"input_tokens":10,"cache_creation_input_tokens":100,"cache_read_input_tokens":40,"output_tokens":25}},"timestamp":"2025-01-15T10:00:01.000Z"}
|
||||
@@ -0,0 +1,187 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { parseSetupArgs, runSetup } from '../src/commands/setup';
|
||||
|
||||
jest.mock('../src/config', () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
||||
const actual = jest.requireActual('../src/config');
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return {
|
||||
...actual,
|
||||
resolveExperiment: jest.fn(
|
||||
(_trackingUri: string, experimentId?: string, experimentName?: string) =>
|
||||
Promise.resolve({
|
||||
experimentId: experimentId ?? 'resolved-id',
|
||||
experimentName,
|
||||
created: false,
|
||||
}),
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
describe('mlflow-claude-code setup', () => {
|
||||
let tmpHome: string;
|
||||
let tmpCwd: string;
|
||||
let originalExitCode: number | string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpHome = mkdtempSync(join(tmpdir(), 'claude-setup-home-'));
|
||||
tmpCwd = mkdtempSync(join(tmpdir(), 'claude-setup-cwd-'));
|
||||
originalExitCode = process.exitCode;
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpHome, { recursive: true, force: true });
|
||||
rmSync(tmpCwd, { recursive: true, force: true });
|
||||
process.exitCode = originalExitCode;
|
||||
});
|
||||
|
||||
it('parses scope and experiment flags', () => {
|
||||
expect(
|
||||
parseSetupArgs([
|
||||
'--project',
|
||||
'--tracking-uri',
|
||||
'http://localhost:5000',
|
||||
'--experiment-name',
|
||||
'claude-code-traces',
|
||||
]),
|
||||
).toMatchObject({
|
||||
projectLocal: true,
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentName: 'claude-code-traces',
|
||||
});
|
||||
});
|
||||
|
||||
it('writes project settings from flags', async () => {
|
||||
await runSetup(
|
||||
[
|
||||
'--project',
|
||||
'--tracking-uri',
|
||||
'http://localhost:5000',
|
||||
'--experiment-name',
|
||||
'my-experiment',
|
||||
],
|
||||
{ home: tmpHome, cwd: tmpCwd },
|
||||
);
|
||||
|
||||
const settingsPath = join(tmpCwd, '.claude', 'settings.json');
|
||||
expect(existsSync(settingsPath)).toBe(true);
|
||||
expect(JSON.parse(readFileSync(settingsPath, 'utf-8'))).toMatchObject({
|
||||
env: {
|
||||
MLFLOW_CLAUDE_TRACING_ENABLED: 'true',
|
||||
MLFLOW_TRACKING_URI: 'http://localhost:5000',
|
||||
MLFLOW_EXPERIMENT_ID: 'resolved-id',
|
||||
MLFLOW_EXPERIMENT_NAME: 'my-experiment',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('writes user settings when --user is passed', async () => {
|
||||
await runSetup(['--user', '--tracking-uri', 'http://mlflow.example', '--experiment-id', '42'], {
|
||||
home: tmpHome,
|
||||
cwd: tmpCwd,
|
||||
});
|
||||
|
||||
expect(existsSync(join(tmpHome, '.claude', 'settings.json'))).toBe(true);
|
||||
expect(existsSync(join(tmpCwd, '.claude', 'settings.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects missing scope flag', async () => {
|
||||
await runSetup(['--tracking-uri', 'http://localhost:5000', '--experiment-name', 'x'], {
|
||||
home: tmpHome,
|
||||
cwd: tmpCwd,
|
||||
});
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it('rejects missing tracking URI', async () => {
|
||||
await runSetup(['--project', '--experiment-name', 'x'], { home: tmpHome, cwd: tmpCwd });
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it('rejects missing experiment', async () => {
|
||||
await runSetup(['--project', '--tracking-uri', 'http://localhost:5000'], {
|
||||
home: tmpHome,
|
||||
cwd: tmpCwd,
|
||||
});
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it('rejects conflicting experiment flags', async () => {
|
||||
await runSetup(
|
||||
[
|
||||
'--project',
|
||||
'--tracking-uri',
|
||||
'http://localhost:5000',
|
||||
'--experiment-id',
|
||||
'1',
|
||||
'--experiment-name',
|
||||
'duplicate',
|
||||
],
|
||||
{ home: tmpHome, cwd: tmpCwd },
|
||||
);
|
||||
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it('rejects invalid tracking URI', async () => {
|
||||
await runSetup(['--project', '--tracking-uri', 'not-a-uri', '--experiment-name', 'x'], {
|
||||
home: tmpHome,
|
||||
cwd: tmpCwd,
|
||||
});
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it('writes the trace location when --trace-location is passed', async () => {
|
||||
await runSetup(
|
||||
[
|
||||
'--user',
|
||||
'--tracking-uri',
|
||||
'databricks',
|
||||
'--experiment-id',
|
||||
'42',
|
||||
'--trace-location',
|
||||
'my_catalog.my_schema.my_prefix',
|
||||
],
|
||||
{ home: tmpHome, cwd: tmpCwd },
|
||||
);
|
||||
|
||||
const settingsPath = join(tmpHome, '.claude', 'settings.json');
|
||||
expect(JSON.parse(readFileSync(settingsPath, 'utf-8'))).toMatchObject({
|
||||
env: {
|
||||
MLFLOW_TRACKING_URI: 'databricks',
|
||||
MLFLOW_EXPERIMENT_ID: '42',
|
||||
MLFLOW_TRACE_LOCATION: 'my_catalog.my_schema.my_prefix',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects --trace-location with a missing value instead of ignoring it', async () => {
|
||||
await runSetup(
|
||||
['--user', '--tracking-uri', 'databricks', '--experiment-id', '42', '--trace-location'],
|
||||
{ home: tmpHome, cwd: tmpCwd },
|
||||
);
|
||||
expect(process.exitCode).toBe(1);
|
||||
expect(existsSync(join(tmpHome, '.claude', 'settings.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects an invalid --trace-location', async () => {
|
||||
await runSetup(
|
||||
[
|
||||
'--user',
|
||||
'--tracking-uri',
|
||||
'databricks',
|
||||
'--experiment-id',
|
||||
'42',
|
||||
'--trace-location',
|
||||
'not-valid',
|
||||
],
|
||||
{ home: tmpHome, cwd: tmpCwd },
|
||||
);
|
||||
expect(process.exitCode).toBe(1);
|
||||
expect(existsSync(join(tmpHome, '.claude', 'settings.json'))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,591 @@
|
||||
/**
|
||||
* Integration tests for createTracedQuery against a real MLflow tracking server.
|
||||
*
|
||||
* Mirrors core/tests/core/api.test.ts: the global-server-setup spawns a
|
||||
* local MLflow server, we create a real experiment, drive the wrapper with
|
||||
* scripted SDKMessage sequences, flush, then fetch the trace via the client
|
||||
* API and assert on the real span tree.
|
||||
*
|
||||
* No mocks of @mlflow/core, LiveSpan, or InMemoryTraceManager — the same
|
||||
* exporter, span processor, and storage path that production uses.
|
||||
*/
|
||||
|
||||
import * as mlflow from '@mlflow/core';
|
||||
import { createAuthProvider } from '@mlflow/core/auth';
|
||||
import { MlflowClient } from '@mlflow/core/clients';
|
||||
import { SpanType } from '@mlflow/core/core/constants';
|
||||
import { SpanStatusCode } from '@mlflow/core/core/entities/span_status';
|
||||
import { TraceState } from '@mlflow/core/core/entities/trace_state';
|
||||
|
||||
import { createTracedQuery } from '../src/tracedClaudeAgent';
|
||||
|
||||
// The real `Query` type has 20+ methods (setModel, setPermissionMode, etc.)
|
||||
// that the wrapper doesn't touch; cast the minimal stub for tests.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type AnyQueryFn = any;
|
||||
|
||||
const TEST_TRACKING_URI = process.env.MLFLOW_TRACKING_URI ?? 'http://localhost:5000';
|
||||
|
||||
// ============================================================================
|
||||
// Mock query (the only thing we mock — the SDK transport itself)
|
||||
// ============================================================================
|
||||
|
||||
interface MockQueryParams {
|
||||
prompt: string | AsyncIterable<unknown>;
|
||||
options?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface MockQueryReturn {
|
||||
[Symbol.asyncIterator]: () => AsyncIterator<unknown>;
|
||||
next: () => Promise<IteratorResult<unknown>>;
|
||||
return: (v?: unknown) => Promise<IteratorResult<unknown>>;
|
||||
throw: (e?: unknown) => Promise<IteratorResult<unknown>>;
|
||||
interrupt: jest.Mock;
|
||||
}
|
||||
|
||||
function mockQuery(messages: unknown[]) {
|
||||
const optionsSeen: Record<string, unknown>[] = [];
|
||||
const fn = (params: MockQueryParams): MockQueryReturn => {
|
||||
optionsSeen.push(params.options ?? {});
|
||||
const iter = (async function* () {
|
||||
// If the prompt is an AsyncIterable, drain it (mirrors real SDK).
|
||||
if (typeof params.prompt !== 'string') {
|
||||
for await (const _item of params.prompt) {
|
||||
// Discard — the wrapper observes via its capturing iterable.
|
||||
}
|
||||
}
|
||||
for (const m of messages) {
|
||||
yield m;
|
||||
}
|
||||
})();
|
||||
return {
|
||||
[Symbol.asyncIterator]: () => iter,
|
||||
next: () => iter.next(),
|
||||
return: (v?: unknown) => iter.return(v as never),
|
||||
throw: (e?: unknown) => iter.throw(e),
|
||||
interrupt: jest.fn(),
|
||||
};
|
||||
};
|
||||
return { fn, optionsSeen };
|
||||
}
|
||||
|
||||
function mockQueryThatThrows(messages: unknown[], error: Error) {
|
||||
const fn = (): MockQueryReturn => {
|
||||
// eslint-disable-next-line @typescript-eslint/require-await
|
||||
const iter = (async function* () {
|
||||
for (const m of messages) {
|
||||
yield m;
|
||||
}
|
||||
throw error;
|
||||
})();
|
||||
return {
|
||||
[Symbol.asyncIterator]: () => iter,
|
||||
next: () => iter.next(),
|
||||
return: (v?: unknown) => iter.return(v as never),
|
||||
throw: (e?: unknown) => iter.throw(e),
|
||||
interrupt: jest.fn(),
|
||||
};
|
||||
};
|
||||
return fn;
|
||||
}
|
||||
|
||||
async function drain(query: unknown) {
|
||||
for await (const _msg of query as AsyncIterable<unknown>) {
|
||||
// discard
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Trace fetch helpers — operate on the real trace returned by the server
|
||||
// ============================================================================
|
||||
|
||||
interface FetchedSpan {
|
||||
name: string;
|
||||
spanId: string;
|
||||
parentId: string | null;
|
||||
spanType: string;
|
||||
inputs: unknown;
|
||||
outputs: unknown;
|
||||
attributes: Record<string, unknown>;
|
||||
status: { statusCode: string; description?: string };
|
||||
}
|
||||
|
||||
function rootSpan(spans: FetchedSpan[]): FetchedSpan {
|
||||
const root = spans.find((s) => s.parentId == null);
|
||||
if (!root) {
|
||||
throw new Error('No root span in trace');
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function spansByName(spans: FetchedSpan[], name: string): FetchedSpan[] {
|
||||
return spans.filter((s) => s.name === name);
|
||||
}
|
||||
|
||||
function spansByType(spans: FetchedSpan[], type: string): FetchedSpan[] {
|
||||
return spans.filter((s) => s.spanType === type);
|
||||
}
|
||||
|
||||
function childrenOf(spans: FetchedSpan[], spanId: string): FetchedSpan[] {
|
||||
return spans.filter((s) => s.parentId === spanId);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Suite setup — one experiment shared across tests
|
||||
// ============================================================================
|
||||
|
||||
describe('createTracedQuery (integration)', () => {
|
||||
let client: MlflowClient;
|
||||
let experimentId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
const authProvider = createAuthProvider({ trackingUri: TEST_TRACKING_URI });
|
||||
client = new MlflowClient({ trackingUri: TEST_TRACKING_URI, authProvider });
|
||||
const experimentName = `claude-code-agent-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
experimentId = await client.createExperiment(experimentName);
|
||||
mlflow.init({ trackingUri: TEST_TRACKING_URI, experimentId });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (experimentId) {
|
||||
await client.deleteExperiment(experimentId);
|
||||
}
|
||||
});
|
||||
|
||||
async function runAndFetchTrace(
|
||||
wrappedQuery: ReturnType<typeof createTracedQuery>,
|
||||
params: MockQueryParams,
|
||||
) {
|
||||
await drain(wrappedQuery(params));
|
||||
await mlflow.flushTraces();
|
||||
const traceId = mlflow.getLastActiveTraceId();
|
||||
if (!traceId) {
|
||||
throw new Error('No active trace id after run');
|
||||
}
|
||||
const trace = await client.getTrace(traceId);
|
||||
return {
|
||||
trace,
|
||||
spans: trace.data.spans as unknown as FetchedSpan[],
|
||||
};
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Basic span tree
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
it('produces an AGENT root span with LLM and TOOL children for a basic run', async () => {
|
||||
const { fn } = mockQuery([
|
||||
{ type: 'system', subtype: 'init', session_id: 'sess-abc', model: 'claude-haiku-4-5' },
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'claude-haiku-4-5',
|
||||
content: [{ type: 'text', text: "I'll list the files." }],
|
||||
usage: { input_tokens: 10, output_tokens: 20 },
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'claude-haiku-4-5',
|
||||
content: [{ type: 'tool_use', id: 'tool-1', name: 'Bash', input: { command: 'ls' } }],
|
||||
usage: { input_tokens: 30, output_tokens: 5 },
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'tool-1',
|
||||
content: 'file1\nfile2',
|
||||
is_error: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'claude-haiku-4-5',
|
||||
content: [{ type: 'text', text: 'Two files: file1, file2.' }],
|
||||
usage: { input_tokens: 40, output_tokens: 10 },
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
duration_ms: 1234,
|
||||
usage: {
|
||||
input_tokens: 80,
|
||||
output_tokens: 35,
|
||||
cache_read_input_tokens: 100,
|
||||
cache_creation_input_tokens: 5,
|
||||
},
|
||||
result: 'Two files: file1, file2.',
|
||||
},
|
||||
]);
|
||||
|
||||
const { trace, spans } = await runAndFetchTrace(createTracedQuery(fn as AnyQueryFn), {
|
||||
prompt: 'list files',
|
||||
});
|
||||
|
||||
expect(trace.info.state).toBe(TraceState.OK);
|
||||
|
||||
const root = rootSpan(spans);
|
||||
expect(root.spanType).toBe(SpanType.AGENT);
|
||||
expect(root.name).toBe('claude_code_conversation');
|
||||
|
||||
const llmSpans = spansByType(spans, SpanType.LLM);
|
||||
expect(llmSpans).toHaveLength(2);
|
||||
expect(llmSpans.every((s) => s.parentId === root.spanId)).toBe(true);
|
||||
|
||||
const toolSpans = spansByType(spans, SpanType.TOOL);
|
||||
expect(toolSpans).toHaveLength(1);
|
||||
expect(toolSpans[0].name).toBe('tool_Bash');
|
||||
expect(toolSpans[0].outputs).toEqual({ result: 'file1\nfile2' });
|
||||
|
||||
// Aggregate token usage (with separate cache keys) is recorded on the root.
|
||||
const rootUsage = root.attributes['mlflow.chat.tokenUsage'] as Record<string, number>;
|
||||
expect(rootUsage).toEqual({
|
||||
input_tokens: 80,
|
||||
output_tokens: 35,
|
||||
total_tokens: 115,
|
||||
cache_read_input_tokens: 100,
|
||||
cache_creation_input_tokens: 5,
|
||||
});
|
||||
|
||||
// Per-turn token usage on the first LLM span.
|
||||
expect(llmSpans[0].attributes['mlflow.chat.tokenUsage']).toEqual({
|
||||
input_tokens: 10,
|
||||
output_tokens: 20,
|
||||
total_tokens: 30,
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Sub-agent nesting
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
it('nests an AGENT(subagent) span under the Task tool for sub-agent invocations', async () => {
|
||||
const { fn } = mockQuery([
|
||||
{ type: 'system', subtype: 'init', session_id: 's', model: 'claude-haiku-4-5' },
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'claude-haiku-4-5',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_use',
|
||||
id: 'task-1',
|
||||
name: 'Agent',
|
||||
input: {
|
||||
subagent_type: 'general-purpose',
|
||||
description: 'List files',
|
||||
prompt: 'List files in /tmp',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
parent_tool_use_id: 'task-1',
|
||||
message: { role: 'user', content: [{ type: 'text', text: 'List files in /tmp' }] },
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: 'task-1',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'claude-haiku-4-5',
|
||||
content: [
|
||||
{ type: 'tool_use', id: 'bash-1', name: 'Bash', input: { command: 'ls /tmp' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
parent_tool_use_id: 'task-1',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'tool_result', tool_use_id: 'bash-1', content: 'a\nb', is_error: false },
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: 'task-1',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'claude-haiku-4-5',
|
||||
content: [{ type: 'text', text: 'Found 2 files.' }],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'task-1',
|
||||
content: 'Found 2 files.',
|
||||
is_error: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
duration_ms: 100,
|
||||
usage: { input_tokens: 0, output_tokens: 0 },
|
||||
},
|
||||
]);
|
||||
|
||||
const { spans } = await runAndFetchTrace(createTracedQuery(fn as AnyQueryFn), {
|
||||
prompt: 'use agent',
|
||||
});
|
||||
|
||||
const root = rootSpan(spans);
|
||||
const taskTool = spansByName(spans, 'tool_Agent')[0];
|
||||
expect(taskTool).toBeDefined();
|
||||
expect(taskTool.parentId).toBe(root.spanId);
|
||||
|
||||
const subagent = spansByName(spans, 'subagent_general-purpose')[0];
|
||||
expect(subagent).toBeDefined();
|
||||
expect(subagent.spanType).toBe(SpanType.AGENT);
|
||||
expect(subagent.parentId).toBe(taskTool.spanId);
|
||||
|
||||
// Inner Bash span nests under the sub-agent wrapper, not under root.
|
||||
const innerBash = spansByName(spans, 'tool_Bash')[0];
|
||||
expect(innerBash.parentId).toBe(subagent.spanId);
|
||||
expect(innerBash.outputs).toEqual({ result: 'a\nb' });
|
||||
|
||||
// At least one inner LLM span lives under the sub-agent.
|
||||
const innerLlms = childrenOf(spans, subagent.spanId).filter((s) => s.spanType === SpanType.LLM);
|
||||
expect(innerLlms.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Error handling
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
it('marks a TOOL span ERROR when its tool_result has is_error=true', async () => {
|
||||
const { fn } = mockQuery([
|
||||
{ type: 'system', subtype: 'init', session_id: 's', model: 'm' },
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'm',
|
||||
content: [{ type: 'tool_use', id: 'tool-x', name: 'Bash', input: {} }],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'tool-x',
|
||||
content: 'command failed',
|
||||
is_error: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
duration_ms: 1,
|
||||
usage: { input_tokens: 0, output_tokens: 0 },
|
||||
},
|
||||
]);
|
||||
|
||||
const { spans } = await runAndFetchTrace(createTracedQuery(fn as AnyQueryFn), { prompt: 'p' });
|
||||
|
||||
const tool = spansByName(spans, 'tool_Bash')[0];
|
||||
expect(tool.status.statusCode).toBe(SpanStatusCode.ERROR);
|
||||
expect(tool.status.description).toBe('command failed');
|
||||
});
|
||||
|
||||
it('closes every open span with ERROR and marks the trace ERROR on stream throw', async () => {
|
||||
const fn = mockQueryThatThrows(
|
||||
[
|
||||
{ type: 'system', subtype: 'init', session_id: 's', model: 'm' },
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'm',
|
||||
content: [{ type: 'tool_use', id: 'tool-x', name: 'Bash', input: {} }],
|
||||
},
|
||||
},
|
||||
],
|
||||
new Error('rate limit'),
|
||||
);
|
||||
|
||||
const traced = createTracedQuery(fn as AnyQueryFn);
|
||||
await expect(drain(traced({ prompt: 'p' }))).rejects.toThrow('rate limit');
|
||||
await mlflow.flushTraces();
|
||||
const traceId = mlflow.getLastActiveTraceId();
|
||||
if (!traceId) {
|
||||
throw new Error('No trace id');
|
||||
}
|
||||
const trace = await client.getTrace(traceId);
|
||||
const spans = trace.data.spans as unknown as FetchedSpan[];
|
||||
|
||||
expect(trace.info.state).toBe(TraceState.ERROR);
|
||||
const tool = spansByName(spans, 'tool_Bash')[0];
|
||||
expect(tool.status.statusCode).toBe(SpanStatusCode.ERROR);
|
||||
});
|
||||
|
||||
it('marks the trace ERROR when result.subtype is not "success"', async () => {
|
||||
const { fn } = mockQuery([
|
||||
{ type: 'system', subtype: 'init', session_id: 's', model: 'm' },
|
||||
{
|
||||
type: 'result',
|
||||
subtype: 'error_max_turns',
|
||||
duration_ms: 1,
|
||||
usage: { input_tokens: 1, output_tokens: 1 },
|
||||
},
|
||||
]);
|
||||
|
||||
const { trace } = await runAndFetchTrace(createTracedQuery(fn as AnyQueryFn), { prompt: 'p' });
|
||||
expect(trace.info.state).toBe(TraceState.ERROR);
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Wrapper behavior
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
it('forces forwardSubagentText: true even when the caller passes false', async () => {
|
||||
const { fn, optionsSeen } = mockQuery([
|
||||
{
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
duration_ms: 1,
|
||||
usage: { input_tokens: 0, output_tokens: 0 },
|
||||
},
|
||||
]);
|
||||
|
||||
await drain(
|
||||
createTracedQuery(fn as AnyQueryFn)({ prompt: 'p', options: { forwardSubagentText: false } }),
|
||||
);
|
||||
await mlflow.flushTraces();
|
||||
|
||||
expect(optionsSeen[0]?.forwardSubagentText).toBe(true);
|
||||
});
|
||||
|
||||
it('replaces callbacks in options with a sanitized placeholder on the span', async () => {
|
||||
const { fn } = mockQuery([
|
||||
{
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
duration_ms: 1,
|
||||
usage: { input_tokens: 0, output_tokens: 0 },
|
||||
},
|
||||
]);
|
||||
const hookFn = jest.fn();
|
||||
const canUseTool = jest.fn();
|
||||
|
||||
const { spans } = await runAndFetchTrace(createTracedQuery(fn as AnyQueryFn), {
|
||||
prompt: 'p',
|
||||
options: {
|
||||
hooks: { PreToolUse: [hookFn] },
|
||||
canUseTool,
|
||||
permissionMode: 'bypassPermissions',
|
||||
},
|
||||
});
|
||||
|
||||
const opts = (rootSpan(spans).inputs as { options: Record<string, unknown> }).options;
|
||||
expect((opts.hooks as { PreToolUse: unknown[] }).PreToolUse[0]).toBe('[function]');
|
||||
expect(opts.canUseTool).toBe('[function]');
|
||||
expect(opts.permissionMode).toBe('bypassPermissions');
|
||||
});
|
||||
|
||||
it('finalizes the trace even when the consumer breaks out of the iterator early', async () => {
|
||||
const { fn } = mockQuery([
|
||||
{ type: 'system', subtype: 'init', session_id: 's', model: 'm' },
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: null,
|
||||
message: { role: 'assistant', model: 'm', content: [{ type: 'text', text: 'hi' }] },
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'm',
|
||||
content: [{ type: 'text', text: 'never reached' }],
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const stream = createTracedQuery(fn as AnyQueryFn)({ prompt: 'p' });
|
||||
for await (const msg of stream as AsyncIterable<{ type: string }>) {
|
||||
if (msg.type === 'assistant') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
await mlflow.flushTraces();
|
||||
|
||||
const traceId = mlflow.getLastActiveTraceId();
|
||||
if (!traceId) {
|
||||
throw new Error('No trace id after early break');
|
||||
}
|
||||
// If the post-loop finalize hadn't run, the root would never end and the
|
||||
// exporter wouldn't have pushed the trace — getTrace would 404.
|
||||
const trace = await client.getTrace(traceId);
|
||||
expect(trace.info.traceId).toBe(traceId);
|
||||
expect(trace.info.state).toBe(TraceState.OK);
|
||||
});
|
||||
|
||||
it('captures streaming-prompt text on the root span as the SDK consumes it', async () => {
|
||||
const { fn } = mockQuery([
|
||||
{
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
duration_ms: 1,
|
||||
usage: { input_tokens: 0, output_tokens: 0 },
|
||||
},
|
||||
]);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/require-await
|
||||
async function* streamingPrompt() {
|
||||
yield { type: 'user', message: { role: 'user', content: 'first chunk' } };
|
||||
yield {
|
||||
type: 'user',
|
||||
message: { role: 'user', content: [{ type: 'text', text: 'second chunk' }] },
|
||||
};
|
||||
}
|
||||
|
||||
const { spans } = await runAndFetchTrace(createTracedQuery(fn as AnyQueryFn), {
|
||||
prompt: streamingPrompt(),
|
||||
});
|
||||
|
||||
const recorded = (rootSpan(spans).inputs as { prompt: string }).prompt;
|
||||
expect(recorded).toContain('first chunk');
|
||||
expect(recorded).toContain('second chunk');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,572 @@
|
||||
import { resolve } from 'node:path';
|
||||
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
import type { TranscriptEntry } from '../src/types';
|
||||
|
||||
// ============================================================================
|
||||
// Mock @mlflow/core
|
||||
// ============================================================================
|
||||
|
||||
const mockSpans: Record<
|
||||
string,
|
||||
{
|
||||
name: string;
|
||||
traceId: string;
|
||||
spanId: string;
|
||||
parentId: string | null;
|
||||
spanType: string;
|
||||
inputs: any;
|
||||
outputs: any;
|
||||
attributes: Record<string, any>;
|
||||
startTimeNs?: number;
|
||||
endTimeNs?: number;
|
||||
exceptions: Error[];
|
||||
}
|
||||
> = {};
|
||||
|
||||
let spanCounter = 0;
|
||||
|
||||
function resetMocks() {
|
||||
for (const key of Object.keys(mockSpans)) {
|
||||
delete mockSpans[key];
|
||||
}
|
||||
spanCounter = 0;
|
||||
}
|
||||
|
||||
const mockTraceInfo: {
|
||||
traceMetadata: Record<string, string>;
|
||||
requestPreview?: string;
|
||||
responsePreview?: string;
|
||||
} = {
|
||||
traceMetadata: {},
|
||||
};
|
||||
|
||||
jest.mock('@mlflow/core', () => {
|
||||
return {
|
||||
init: jest.fn(),
|
||||
startSpan: jest.fn((options: any) => {
|
||||
const id = `span-${++spanCounter}`;
|
||||
const parentId = options.parent ? options.parent.spanId : null;
|
||||
const span = {
|
||||
name: options.name,
|
||||
traceId: 'mock-trace-id',
|
||||
spanId: id,
|
||||
parentId,
|
||||
spanType: options.spanType ?? 'UNKNOWN',
|
||||
inputs: options.inputs ?? {},
|
||||
outputs: {},
|
||||
attributes: { ...(options.attributes ?? {}) },
|
||||
startTimeNs: options.startTimeNs,
|
||||
endTimeNs: undefined as number | undefined,
|
||||
exceptions: [] as Error[],
|
||||
setAttribute: jest.fn((key: string, value: any) => {
|
||||
span.attributes[key] = value;
|
||||
}),
|
||||
setOutputs: jest.fn((outputs: any) => {
|
||||
span.outputs = outputs;
|
||||
}),
|
||||
end: jest.fn((opts?: { endTimeNs?: number }) => {
|
||||
span.endTimeNs = opts?.endTimeNs;
|
||||
}),
|
||||
recordException: jest.fn((err: Error) => {
|
||||
span.exceptions.push(err);
|
||||
}),
|
||||
};
|
||||
mockSpans[id] = span;
|
||||
return span;
|
||||
}),
|
||||
flushTraces: jest.fn().mockResolvedValue(undefined),
|
||||
SpanType: {
|
||||
LLM: 'LLM',
|
||||
CHAIN: 'CHAIN',
|
||||
AGENT: 'AGENT',
|
||||
TOOL: 'TOOL',
|
||||
UNKNOWN: 'UNKNOWN',
|
||||
},
|
||||
SpanAttributeKey: {
|
||||
TOKEN_USAGE: 'mlflow.chat.tokenUsage',
|
||||
MESSAGE_FORMAT: 'mlflow.message.format',
|
||||
},
|
||||
TraceMetadataKey: {
|
||||
TRACE_SESSION: 'mlflow.trace.session',
|
||||
TRACE_USER: 'mlflow.trace.user',
|
||||
TOKEN_USAGE: 'mlflow.trace.tokenUsage',
|
||||
},
|
||||
TokenUsageKey: {
|
||||
INPUT_TOKENS: 'input_tokens',
|
||||
OUTPUT_TOKENS: 'output_tokens',
|
||||
TOTAL_TOKENS: 'total_tokens',
|
||||
CACHE_READ_INPUT_TOKENS: 'cache_read_input_tokens',
|
||||
CACHE_CREATION_INPUT_TOKENS: 'cache_creation_input_tokens',
|
||||
},
|
||||
InMemoryTraceManager: {
|
||||
getInstance: jest.fn(() => ({
|
||||
getTrace: jest.fn(() => ({
|
||||
info: mockTraceInfo,
|
||||
})),
|
||||
})),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
// Import after mock
|
||||
import { processTranscript } from '../src/tracing';
|
||||
import { startSpan, flushTraces } from '@mlflow/core';
|
||||
|
||||
const FIXTURES_DIR = resolve(__dirname, 'fixtures');
|
||||
|
||||
// ============================================================================
|
||||
// Helpers
|
||||
// ============================================================================
|
||||
|
||||
function getSpans() {
|
||||
return Object.values(mockSpans);
|
||||
}
|
||||
|
||||
function getSpansByType(type: string) {
|
||||
return getSpans().filter((s) => s.spanType === type);
|
||||
}
|
||||
|
||||
function getSpansByName(name: string) {
|
||||
return getSpans().filter((s) => s.name === name);
|
||||
}
|
||||
|
||||
function getChildSpans(parentId: string) {
|
||||
return getSpans().filter((s) => s.parentId === parentId);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test suite
|
||||
// ============================================================================
|
||||
|
||||
beforeEach(() => {
|
||||
resetMocks();
|
||||
mockTraceInfo.traceMetadata = {};
|
||||
mockTraceInfo.requestPreview = undefined;
|
||||
mockTraceInfo.responsePreview = undefined;
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('processTranscript', () => {
|
||||
// --------------------------------------------------------------------------
|
||||
// Basic span hierarchy
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('basic transcript', () => {
|
||||
it('creates root AGENT span with LLM and TOOL children', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
|
||||
const agents = getSpansByType('AGENT');
|
||||
const llms = getSpansByType('LLM');
|
||||
const tools = getSpansByType('TOOL');
|
||||
|
||||
expect(agents).toHaveLength(1);
|
||||
expect(agents[0].name).toBe('claude_code_conversation');
|
||||
expect(llms).toHaveLength(2);
|
||||
expect(tools).toHaveLength(1);
|
||||
expect(tools[0].name).toBe('tool_Bash');
|
||||
});
|
||||
|
||||
it('sets correct root span inputs and outputs', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
|
||||
const root = getSpansByName('claude_code_conversation')[0];
|
||||
expect(root.inputs.prompt).toBe('What is 2 + 2?');
|
||||
expect(root.outputs.status).toBe('completed');
|
||||
expect(root.outputs.response).toBe('The answer is 4.');
|
||||
});
|
||||
|
||||
it('sets LLM span inputs with messages and outputs in Anthropic format', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
|
||||
const llms = getSpansByType('LLM');
|
||||
const firstLlm = llms.find((s) => s.inputs?.messages?.length > 0);
|
||||
expect(firstLlm).toBeDefined();
|
||||
|
||||
// Outputs should be in Anthropic response format
|
||||
expect(firstLlm!.outputs.type).toBe('message');
|
||||
expect(firstLlm!.outputs.role).toBe('assistant');
|
||||
expect(firstLlm!.outputs.content).toBeDefined();
|
||||
});
|
||||
|
||||
it('sets MESSAGE_FORMAT attribute on LLM spans', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
|
||||
const llms = getSpansByType('LLM');
|
||||
for (const llm of llms) {
|
||||
expect(llm.attributes['mlflow.message.format']).toBe('anthropic');
|
||||
}
|
||||
});
|
||||
|
||||
it('sets tool span inputs and outputs correctly', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
|
||||
const toolSpan = getSpansByName('tool_Bash')[0];
|
||||
expect(toolSpan.inputs).toEqual({ command: 'echo $((2 + 2))' });
|
||||
expect(toolSpan.outputs).toEqual({ result: '4' });
|
||||
expect(toolSpan.attributes.tool_name).toBe('Bash');
|
||||
expect(toolSpan.attributes.tool_id).toBe('tool_123');
|
||||
});
|
||||
|
||||
it('all child spans have root span as parent', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
|
||||
const root = getSpansByName('claude_code_conversation')[0];
|
||||
const children = getChildSpans(root.spanId);
|
||||
// 2 LLM + 1 TOOL
|
||||
expect(children).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('calls flushTraces after processing', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
expect(flushTraces).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Token usage
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('token usage', () => {
|
||||
it('preserves cache tokens as separate fields and excludes cache from total', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'with-usage.jsonl'), 'test-session-usage');
|
||||
|
||||
const llms = getSpansByType('LLM');
|
||||
expect(llms).toHaveLength(1);
|
||||
|
||||
const tokenUsage = llms[0].attributes['mlflow.chat.tokenUsage'];
|
||||
expect(tokenUsage).toBeDefined();
|
||||
// input_tokens stays as the non-cached input the API reports.
|
||||
expect(tokenUsage.input_tokens).toBe(10);
|
||||
expect(tokenUsage.output_tokens).toBe(25);
|
||||
// total = input + output, cache excluded (matches mlflow.anthropic.autolog).
|
||||
expect(tokenUsage.total_tokens).toBe(35);
|
||||
// Cache fields are surfaced as separate optional keys.
|
||||
expect(tokenUsage.cache_read_input_tokens).toBe(40);
|
||||
expect(tokenUsage.cache_creation_input_tokens).toBe(100);
|
||||
});
|
||||
|
||||
it('omits cache token keys when the API does not report them', async () => {
|
||||
const tmpDir = mkdtempSync(resolve(tmpdir(), 'cc-test-'));
|
||||
const transcriptPath = resolve(tmpDir, 'no-cache.jsonl');
|
||||
const entries: TranscriptEntry[] = [
|
||||
{
|
||||
type: 'user',
|
||||
message: { role: 'user', content: 'Hi' },
|
||||
timestamp: '2025-01-15T10:00:00.000Z',
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: 'Hello!' }],
|
||||
usage: { input_tokens: 7, output_tokens: 3 },
|
||||
},
|
||||
timestamp: '2025-01-15T10:00:01.000Z',
|
||||
},
|
||||
];
|
||||
writeFileSync(transcriptPath, entries.map((e) => JSON.stringify(e)).join('\n') + '\n');
|
||||
|
||||
await processTranscript(transcriptPath, 'no-cache-session');
|
||||
|
||||
const tokenUsage = getSpansByType('LLM')[0].attributes['mlflow.chat.tokenUsage'];
|
||||
expect(tokenUsage.input_tokens).toBe(7);
|
||||
expect(tokenUsage.output_tokens).toBe(3);
|
||||
expect(tokenUsage.total_tokens).toBe(10);
|
||||
expect(tokenUsage).not.toHaveProperty('cache_read_input_tokens');
|
||||
expect(tokenUsage).not.toHaveProperty('cache_creation_input_tokens');
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Metadata
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('metadata', () => {
|
||||
it('sets trace session metadata', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
expect(mockTraceInfo.traceMetadata['mlflow.trace.session']).toBe('test-session-123');
|
||||
});
|
||||
|
||||
it('sets trace user from environment', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
expect(mockTraceInfo.traceMetadata['mlflow.trace.user']).toBe(process.env.USER ?? '');
|
||||
});
|
||||
|
||||
it('sets working directory', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
expect(mockTraceInfo.traceMetadata['mlflow.trace.working_directory']).toBe(process.cwd());
|
||||
});
|
||||
|
||||
it('sets request and response previews', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
expect(mockTraceInfo.requestPreview).toBe('What is 2 + 2?');
|
||||
expect(mockTraceInfo.responsePreview).toBe('The answer is 4.');
|
||||
});
|
||||
|
||||
it('captures permission mode from user entry', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'with-tool-error.jsonl'), 'test-perm');
|
||||
expect(mockTraceInfo.traceMetadata['mlflow.trace.permission_mode']).toBe('default');
|
||||
});
|
||||
|
||||
it('captures Claude Code version from transcript', async () => {
|
||||
const tmpDir = mkdtempSync(resolve(tmpdir(), 'cc-test-'));
|
||||
const transcriptPath = resolve(tmpDir, 'version.jsonl');
|
||||
const entries: TranscriptEntry[] = [
|
||||
{
|
||||
type: 'user',
|
||||
version: '2.1.34',
|
||||
message: { role: 'user', content: 'Hello!' },
|
||||
timestamp: '2025-01-15T10:00:00.000Z',
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
version: '2.1.34',
|
||||
message: { role: 'assistant', content: [{ type: 'text', text: 'Hi!' }] },
|
||||
timestamp: '2025-01-15T10:00:01.000Z',
|
||||
},
|
||||
];
|
||||
writeFileSync(transcriptPath, entries.map((e) => JSON.stringify(e)).join('\n') + '\n');
|
||||
|
||||
await processTranscript(transcriptPath, 'version-test');
|
||||
expect(mockTraceInfo.traceMetadata['mlflow.claude_code_version']).toBe('2.1.34');
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Sub-agent (progress-based)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('sub-agent spans (progress-based)', () => {
|
||||
it('creates nested AGENT span under tool_Task', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'with-subagent.jsonl'), 'test-subagent');
|
||||
|
||||
const agents = getSpansByType('AGENT');
|
||||
// Root + subagent_Explore
|
||||
expect(agents.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const taskTools = getSpansByName('tool_Task');
|
||||
expect(taskTools).toHaveLength(1);
|
||||
|
||||
const taskChildren = getChildSpans(taskTools[0].spanId);
|
||||
const subAgents = taskChildren.filter((s) => s.spanType === 'AGENT');
|
||||
expect(subAgents).toHaveLength(1);
|
||||
expect(subAgents[0].name).toBe('subagent_Explore');
|
||||
});
|
||||
|
||||
it('creates LLM and tool spans under sub-agent', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'with-subagent.jsonl'), 'test-subagent');
|
||||
|
||||
const subAgentSpan = getSpansByName('subagent_Explore')[0];
|
||||
const agentChildren = getChildSpans(subAgentSpan.spanId);
|
||||
|
||||
const childLlm = agentChildren.filter((s) => s.spanType === 'LLM');
|
||||
const childTool = agentChildren.filter((s) => s.spanType === 'TOOL');
|
||||
|
||||
expect(childLlm.length).toBeGreaterThanOrEqual(1);
|
||||
expect(childTool.length).toBeGreaterThanOrEqual(1);
|
||||
expect(childTool[0].name).toBe('tool_Grep');
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Parallel tool_uses in one assistant turn (split across JSONL entries)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('parallel tool_uses in single turn', () => {
|
||||
it('creates a tool span for every parallel sub-agent call', async () => {
|
||||
await processTranscript(
|
||||
resolve(FIXTURES_DIR, 'with-parallel-subagents.jsonl'),
|
||||
'test-parallel',
|
||||
);
|
||||
|
||||
const taskTools = getSpansByName('tool_Task');
|
||||
expect(taskTools).toHaveLength(4);
|
||||
|
||||
const outputs = taskTools.map((s) => (s.outputs as { result: string }).result).sort();
|
||||
expect(outputs).toEqual(['A done', 'B done', 'C done', 'D done']);
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Sub-agent (file-based)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('sub-agent spans (file-based)', () => {
|
||||
it('reads sub-agent transcript from separate file', async () => {
|
||||
// Set up file structure: main.jsonl + main/subagents/agent-abc1234.jsonl
|
||||
const tmpDir = mkdtempSync(resolve(tmpdir(), 'cc-test-'));
|
||||
const mainPath = resolve(tmpDir, 'session-123.jsonl');
|
||||
const subagentDir = resolve(tmpDir, 'session-123', 'subagents');
|
||||
mkdirSync(subagentDir, { recursive: true });
|
||||
|
||||
// Copy fixtures
|
||||
const mainContent = readFileSync(resolve(FIXTURES_DIR, 'with-subagent-file.jsonl'), 'utf-8');
|
||||
writeFileSync(mainPath, mainContent);
|
||||
|
||||
const subagentContent = readFileSync(
|
||||
resolve(FIXTURES_DIR, 'subagent-abc1234.jsonl'),
|
||||
'utf-8',
|
||||
);
|
||||
writeFileSync(resolve(subagentDir, 'agent-abc1234.jsonl'), subagentContent);
|
||||
|
||||
await processTranscript(mainPath, 'test-subagent-file');
|
||||
|
||||
// Verify sub-agent span hierarchy
|
||||
const taskTools = getSpansByName('tool_Task');
|
||||
expect(taskTools).toHaveLength(1);
|
||||
|
||||
const taskChildren = getChildSpans(taskTools[0].spanId);
|
||||
const subAgents = taskChildren.filter((s) => s.spanType === 'AGENT');
|
||||
expect(subAgents).toHaveLength(1);
|
||||
expect(subAgents[0].name).toBe('subagent_Explore');
|
||||
|
||||
// Sub-agent should have Grep and Read tool children
|
||||
const agentChildren = getChildSpans(subAgents[0].spanId);
|
||||
const childTools = agentChildren.filter((s) => s.spanType === 'TOOL');
|
||||
const toolNames = new Set(childTools.map((s) => s.name));
|
||||
expect(toolNames).toContain('tool_Grep');
|
||||
expect(toolNames).toContain('tool_Read');
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Tool errors
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('tool errors', () => {
|
||||
it('records exception on rejected tool', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'with-tool-error.jsonl'), 'test-error');
|
||||
|
||||
const bashTools = getSpansByName('tool_Bash');
|
||||
expect(bashTools).toHaveLength(1);
|
||||
|
||||
const toolSpan = bashTools[0];
|
||||
expect(toolSpan.exceptions).toHaveLength(1);
|
||||
expect(toolSpan.exceptions[0].message).toContain("doesn't want to proceed");
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Edge cases
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('edge cases', () => {
|
||||
it('handles empty transcript gracefully', async () => {
|
||||
const tmpDir = mkdtempSync(resolve(tmpdir(), 'cc-test-'));
|
||||
const emptyPath = resolve(tmpDir, 'empty.jsonl');
|
||||
writeFileSync(emptyPath, '');
|
||||
|
||||
await processTranscript(emptyPath, 'empty-session');
|
||||
expect(startSpan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('handles transcript with no user message', async () => {
|
||||
const tmpDir = mkdtempSync(resolve(tmpdir(), 'cc-test-'));
|
||||
const noUserPath = resolve(tmpDir, 'no-user.jsonl');
|
||||
const entries = [
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { role: 'assistant', content: [{ type: 'text', text: 'Hi' }] },
|
||||
timestamp: '2025-01-15T10:00:00.000Z',
|
||||
},
|
||||
];
|
||||
writeFileSync(noUserPath, entries.map((e) => JSON.stringify(e)).join('\n') + '\n');
|
||||
|
||||
await processTranscript(noUserPath, 'no-user-session');
|
||||
expect(startSpan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('handles nonexistent file gracefully', async () => {
|
||||
await processTranscript('/nonexistent/path/transcript.jsonl', 'test-session');
|
||||
expect(startSpan).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Timing
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('timing', () => {
|
||||
it('sets start and end times on root span', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
|
||||
const root = getSpansByName('claude_code_conversation')[0];
|
||||
expect(root.startTimeNs).toBeDefined();
|
||||
expect(root.endTimeNs).toBeDefined();
|
||||
expect(root.endTimeNs!).toBeGreaterThan(root.startTimeNs!);
|
||||
});
|
||||
|
||||
it('sets start and end times on child spans', async () => {
|
||||
await processTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'), 'test-session-123');
|
||||
|
||||
const llms = getSpansByType('LLM');
|
||||
for (const llm of llms) {
|
||||
expect(llm.startTimeNs).toBeDefined();
|
||||
expect(llm.endTimeNs).toBeDefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Steer messages
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('steer messages', () => {
|
||||
it('includes queue-operation enqueue as user messages in LLM inputs', async () => {
|
||||
const tmpDir = mkdtempSync(resolve(tmpdir(), 'cc-test-'));
|
||||
const steerPath = resolve(tmpDir, 'steer.jsonl');
|
||||
const entries: TranscriptEntry[] = [
|
||||
{
|
||||
type: 'user',
|
||||
message: { role: 'user', content: 'Tell me about Python.' },
|
||||
timestamp: '2025-01-15T10:00:00.000Z',
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: 'Python is a programming language.' }],
|
||||
},
|
||||
timestamp: '2025-01-15T10:00:01.000Z',
|
||||
},
|
||||
{
|
||||
type: 'queue-operation',
|
||||
operation: 'enqueue',
|
||||
content: 'also tell me about Java',
|
||||
timestamp: '2025-01-15T10:00:02.000Z',
|
||||
},
|
||||
{
|
||||
type: 'queue-operation',
|
||||
operation: 'remove' as any,
|
||||
timestamp: '2025-01-15T10:00:03.000Z',
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: 'Java is also a programming language.' }],
|
||||
},
|
||||
timestamp: '2025-01-15T10:00:04.000Z',
|
||||
},
|
||||
];
|
||||
writeFileSync(steerPath, entries.map((e) => JSON.stringify(e)).join('\n') + '\n');
|
||||
|
||||
await processTranscript(steerPath, 'steer-session');
|
||||
|
||||
const llms = getSpansByType('LLM');
|
||||
expect(llms).toHaveLength(2);
|
||||
|
||||
// Second LLM span should have steer message in inputs
|
||||
const secondLlm = llms[1];
|
||||
const inputMessages = secondLlm.inputs.messages as Array<{
|
||||
role: string;
|
||||
content: unknown;
|
||||
}>;
|
||||
const steerMessages = inputMessages.filter((m) => m.content === 'also tell me about Java');
|
||||
expect(steerMessages).toHaveLength(1);
|
||||
expect(steerMessages[0].role).toBe('user');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,239 @@
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
import {
|
||||
readTranscript,
|
||||
parseTimestampToNs,
|
||||
extractTextContent,
|
||||
findLastUserMessageIndex,
|
||||
findFinalAssistantResponse,
|
||||
} from '../src/transcript';
|
||||
|
||||
import type { TranscriptEntry } from '../src/types';
|
||||
|
||||
const FIXTURES_DIR = resolve(__dirname, 'fixtures');
|
||||
|
||||
// ============================================================================
|
||||
// readTranscript
|
||||
// ============================================================================
|
||||
|
||||
describe('readTranscript', () => {
|
||||
it('parses a basic JSONL file', () => {
|
||||
const entries = readTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'));
|
||||
expect(entries.length).toBe(5);
|
||||
expect(entries[0].type).toBe('user');
|
||||
expect(entries[1].type).toBe('assistant');
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// parseTimestampToNs
|
||||
// ============================================================================
|
||||
|
||||
describe('parseTimestampToNs', () => {
|
||||
it('parses ISO string', () => {
|
||||
const result = parseTimestampToNs('2024-01-15T10:30:45.123Z');
|
||||
expect(typeof result).toBe('number');
|
||||
expect(result).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('converts Unix seconds to nanoseconds', () => {
|
||||
const unixTs = 1705312245.123456;
|
||||
const result = parseTimestampToNs(unixTs);
|
||||
const expected = Math.floor(unixTs * 1e9);
|
||||
expect(result).toBe(expected);
|
||||
});
|
||||
|
||||
it('converts milliseconds to nanoseconds', () => {
|
||||
const msTs = 1705312245123;
|
||||
const result = parseTimestampToNs(msTs);
|
||||
expect(result).toBe(Math.floor(msTs * 1e6));
|
||||
});
|
||||
|
||||
it('returns nanoseconds as-is for large numbers', () => {
|
||||
// Use a value that's >= 1e13 (routes through the ns branch) and
|
||||
// below Number.MAX_SAFE_INTEGER (~9e15) so no precision is lost.
|
||||
const nsTs = 1705312245123456;
|
||||
const result = parseTimestampToNs(nsTs);
|
||||
expect(result).toBe(Math.floor(nsTs));
|
||||
});
|
||||
|
||||
it('returns null for empty/null input', () => {
|
||||
expect(parseTimestampToNs(null)).toBeNull();
|
||||
expect(parseTimestampToNs(undefined)).toBeNull();
|
||||
expect(parseTimestampToNs('')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for invalid string', () => {
|
||||
expect(parseTimestampToNs('not-a-date')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// extractTextContent
|
||||
// ============================================================================
|
||||
|
||||
describe('extractTextContent', () => {
|
||||
it('extracts text from content block array', () => {
|
||||
const content = [
|
||||
{ type: 'text' as const, text: 'Hello' },
|
||||
{ type: 'tool_use' as const, id: 'x', name: 'Bash', input: {} },
|
||||
{ type: 'text' as const, text: 'World' },
|
||||
];
|
||||
expect(extractTextContent(content)).toBe('Hello\nWorld');
|
||||
});
|
||||
|
||||
it('returns string content directly', () => {
|
||||
expect(extractTextContent('plain text')).toBe('plain text');
|
||||
});
|
||||
|
||||
it('handles empty array', () => {
|
||||
expect(extractTextContent([])).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// findLastUserMessageIndex
|
||||
// ============================================================================
|
||||
|
||||
describe('findLastUserMessageIndex', () => {
|
||||
it('finds the last user message in basic transcript', () => {
|
||||
const transcript: TranscriptEntry[] = [
|
||||
{
|
||||
type: 'user',
|
||||
message: { role: 'user', content: 'First question' },
|
||||
timestamp: '2025-01-01T00:00:00Z',
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { role: 'assistant', content: [{ type: 'text', text: 'First answer' }] },
|
||||
timestamp: '2025-01-01T00:00:01Z',
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
message: { role: 'user', content: 'Second question' },
|
||||
timestamp: '2025-01-01T00:00:02Z',
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { role: 'assistant', content: [{ type: 'text', text: 'Second answer' }] },
|
||||
timestamp: '2025-01-01T00:00:03Z',
|
||||
},
|
||||
];
|
||||
|
||||
const idx = findLastUserMessageIndex(transcript);
|
||||
expect(idx).toBe(2);
|
||||
});
|
||||
|
||||
it('skips tool result messages', () => {
|
||||
const entries = readTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'));
|
||||
const idx = findLastUserMessageIndex(entries);
|
||||
// Entry 0 is the real user message; entry 3 is a tool_result → skipped
|
||||
expect(idx).toBe(0);
|
||||
});
|
||||
|
||||
it('skips skill injection messages', () => {
|
||||
const transcript: TranscriptEntry[] = [
|
||||
{
|
||||
type: 'user',
|
||||
message: { role: 'user', content: 'Enable tracing on the agent.' },
|
||||
timestamp: '2025-01-01T00:00:00Z',
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'tool_use', id: 'toolu_abc', name: 'Skill', input: { skill: 'my-skill' } },
|
||||
],
|
||||
},
|
||||
timestamp: '2025-01-01T00:00:01Z',
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
toolUseResult: { success: true, commandName: 'my-skill' },
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'toolu_abc',
|
||||
content: 'Launching skill: my-skill',
|
||||
},
|
||||
],
|
||||
},
|
||||
timestamp: '2025-01-01T00:00:02Z',
|
||||
},
|
||||
// Skill content injection — should be skipped
|
||||
{
|
||||
type: 'user',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: 'Base directory: /skill\n# Guide' }],
|
||||
},
|
||||
timestamp: '2025-01-01T00:00:03Z',
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { role: 'assistant', content: [{ type: 'text', text: 'Done.' }] },
|
||||
timestamp: '2025-01-01T00:00:04Z',
|
||||
},
|
||||
];
|
||||
|
||||
const idx = findLastUserMessageIndex(transcript);
|
||||
expect(idx).toBe(0);
|
||||
expect(transcript[idx!].message!.content as string).toBe('Enable tracing on the agent.');
|
||||
});
|
||||
|
||||
it('skips compaction summary messages', () => {
|
||||
const transcript: TranscriptEntry[] = [
|
||||
{
|
||||
type: 'user',
|
||||
message: { role: 'user', content: 'Real question after context reset' },
|
||||
timestamp: '2025-01-01T00:00:00Z',
|
||||
},
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { role: 'assistant', content: [{ type: 'text', text: 'Answer' }] },
|
||||
timestamp: '2025-01-01T00:00:01Z',
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
isCompactSummary: true,
|
||||
message: { role: 'user', content: 'Summary of prior conversation...' },
|
||||
timestamp: '2025-01-01T00:00:02Z',
|
||||
},
|
||||
];
|
||||
|
||||
const idx = findLastUserMessageIndex(transcript);
|
||||
expect(idx).toBe(0);
|
||||
});
|
||||
|
||||
it('returns null for empty transcript', () => {
|
||||
expect(findLastUserMessageIndex([])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// findFinalAssistantResponse
|
||||
// ============================================================================
|
||||
|
||||
describe('findFinalAssistantResponse', () => {
|
||||
it('finds the last text response', () => {
|
||||
const entries = readTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'));
|
||||
const response = findFinalAssistantResponse(entries, 1);
|
||||
expect(response).toBe('The answer is 4.');
|
||||
});
|
||||
|
||||
it('returns null when no text response found', () => {
|
||||
const transcript: TranscriptEntry[] = [
|
||||
{
|
||||
type: 'assistant',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: [{ type: 'tool_use', id: 'x', name: 'Bash', input: {} }],
|
||||
},
|
||||
},
|
||||
];
|
||||
expect(findFinalAssistantResponse(transcript, 0)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src"
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "bundle", "tests"]
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
# MLflow Typescript SDK - Codex CLI
|
||||
|
||||
Seamlessly integrate [MLflow Tracing](https://github.com/mlflow/mlflow/tree/main/libs/typescript) with [Codex CLI](https://github.com/openai/codex) to automatically trace your Codex coding-agent conversations, including user prompts, assistant responses, tool usage, and token consumption.
|
||||
|
||||
| Package | NPM | Description |
|
||||
| ------------------- | ------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ |
|
||||
| [@mlflow/codex](./) | [](https://www.npmjs.com/package/@mlflow/codex) | Auto-instrumentation integration for OpenAI Codex CLI. |
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install -g @mlflow/codex
|
||||
```
|
||||
|
||||
This installs the `mlflow-codex` CLI globally. If you'd rather not install globally, you can invoke it via `npx @mlflow/codex` (every command below works the same way).
|
||||
|
||||
## Quickstart
|
||||
|
||||
Start MLflow Tracking Server if you don't have one already:
|
||||
|
||||
```bash
|
||||
pip install mlflow
|
||||
mlflow server --port 5000
|
||||
```
|
||||
|
||||
Self-hosting MLflow server requires Python 3.10 or higher. If you don't have one, you can also use [managed MLflow service](https://mlflow.org/#get-started) for free to get started quickly.
|
||||
|
||||
Run the interactive setup. It registers the Codex `notify` hook and writes your tracking URI / experiment ID into Codex's config directory:
|
||||
|
||||
```bash
|
||||
mlflow-codex setup
|
||||
```
|
||||
|
||||
The setup command prompts you to choose between a project-local install (`./.codex/`) or a user-level install (`~/.codex/`), then writes:
|
||||
|
||||
- `config.toml`: adds `notify = ["mlflow-codex", "notify-hook"]` so Codex invokes the hook after every turn.
|
||||
- `mlflow-tracing.json`: persists your MLflow tracking URI and experiment ID.
|
||||
|
||||
Pass `--non-interactive` / `-y` to skip prompts and use defaults, or override values with `--tracking-uri` and `--experiment-id`:
|
||||
|
||||
```bash
|
||||
mlflow-codex setup -y --tracking-uri http://localhost:5000 --experiment-id 0
|
||||
```
|
||||
|
||||
Use Codex normally:
|
||||
|
||||
```bash
|
||||
codex "help me refactor this function"
|
||||
```
|
||||
|
||||
After each conversation turn, MLflow records a trace with the message history, tool calls and results, and token usage. You don't need to wait for the session to end.
|
||||
|
||||
## Configuration
|
||||
|
||||
The `mlflow-codex` hook resolves configuration in this order (first match wins):
|
||||
|
||||
1. `MLFLOW_TRACKING_URI` / `MLFLOW_EXPERIMENT_ID` environment variables
|
||||
2. `./.codex/mlflow-tracing.json` (project-local)
|
||||
3. `~/.codex/mlflow-tracing.json` (user-level)
|
||||
|
||||
Environment variables are convenient for one-off overrides, e.g. switching between a local server and a Databricks workspace:
|
||||
|
||||
```bash
|
||||
MLFLOW_TRACKING_URI=databricks MLFLOW_EXPERIMENT_ID=123456789 codex "..."
|
||||
```
|
||||
|
||||
## Documentation 📘
|
||||
|
||||
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart). For the full Codex CLI tracing guide including troubleshooting and OTLP support, see the [Codex CLI integration page](https://mlflow.org/docs/latest/genai/tracing/integrations/listing/codex).
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
|
||||
@@ -0,0 +1,21 @@
|
||||
import { build } from 'esbuild';
|
||||
import { chmodSync } from 'node:fs';
|
||||
|
||||
await build({
|
||||
entryPoints: ['dist/cli.js'],
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
outfile: 'bundle/cli.js',
|
||||
external: ['node:*'],
|
||||
banner: {
|
||||
// Create a require function for CJS dependencies that use bare node specifiers
|
||||
js: [
|
||||
'#!/usr/bin/env node',
|
||||
'import { createRequire as __createRequire } from "node:module";',
|
||||
'const require = __createRequire(import.meta.url);',
|
||||
].join('\n'),
|
||||
},
|
||||
});
|
||||
|
||||
chmodSync('bundle/cli.js', 0o755);
|
||||
@@ -0,0 +1,41 @@
|
||||
const path = require('path');
|
||||
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
roots: ['<rootDir>/tests'],
|
||||
testMatch: ['**/*.test.ts'],
|
||||
moduleFileExtensions: ['ts', 'js', 'json', 'node'],
|
||||
modulePaths: [path.resolve(__dirname, '../../node_modules')],
|
||||
transform: {
|
||||
'^.+\\.tsx?$': [
|
||||
'ts-jest',
|
||||
{
|
||||
tsconfig: {
|
||||
target: 'ES2022',
|
||||
module: 'CommonJS',
|
||||
moduleResolution: 'Node',
|
||||
esModuleInterop: true,
|
||||
strict: true,
|
||||
skipLibCheck: true,
|
||||
types: ['jest', 'node'],
|
||||
baseUrl: '.',
|
||||
paths: {
|
||||
'@mlflow/core': ['../../core/src/index.ts'],
|
||||
'@mlflow/core/*': ['../../core/src/*'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
moduleNameMapper: {
|
||||
'^@mlflow/core$': '<rootDir>/../../core/src',
|
||||
'^@mlflow/core/(.*)$': '<rootDir>/../../core/src/$1',
|
||||
// Strip .js extensions for ESM → CJS test resolution
|
||||
'^(\\.{1,2}/.*)\\.js$': '$1',
|
||||
},
|
||||
testTimeout: 30000,
|
||||
forceExit: true,
|
||||
detectOpenHandles: true,
|
||||
};
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"name": "@mlflow/codex",
|
||||
"version": "0.3.0",
|
||||
"description": "Codex CLI integration package for MLflow Tracing",
|
||||
"type": "module",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mlflow/mlflow.git"
|
||||
},
|
||||
"homepage": "https://mlflow.org/",
|
||||
"author": {
|
||||
"name": "MLflow",
|
||||
"url": "https://mlflow.org/"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/mlflow/mlflow/issues"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"keywords": [
|
||||
"mlflow",
|
||||
"tracing",
|
||||
"observability",
|
||||
"codex",
|
||||
"openai",
|
||||
"llm",
|
||||
"agent",
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"files": [
|
||||
"dist",
|
||||
"bundle"
|
||||
],
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"bin": {
|
||||
"mlflow-codex": "./bundle/cli.js"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc && npm run build:bundle",
|
||||
"build:bundle": "node esbuild.config.mjs",
|
||||
"prepublishOnly": "npm run build",
|
||||
"test": "jest --config jest.config.cjs",
|
||||
"lint": "eslint src --ext .ts",
|
||||
"lint:fix": "eslint src --ext .ts --fix",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlflow/core": "^0.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.14",
|
||||
"esbuild": "^0.25.4",
|
||||
"jest": "^29.7.0",
|
||||
"ts-jest": "^29.3.2",
|
||||
"typescript": "^5.8.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* CLI dispatcher for `@mlflow/codex`.
|
||||
*
|
||||
* Installed as the `mlflow-codex` bin. Subcommands:
|
||||
* - `setup` → registers the notify hook and writes mlflow-tracing.json
|
||||
* - `notify-hook` → runs the Codex notify handler (Codex appends the JSON
|
||||
* payload as the final argv entry)
|
||||
* - `--help`/`-h` → prints usage
|
||||
*
|
||||
* Codex invokes `mlflow-codex notify-hook` as the registered notify handler.
|
||||
* Codex appends the turn JSON as the final argument.
|
||||
*/
|
||||
|
||||
import { runNotifyHook } from './hooks/stop.js';
|
||||
import { runSetup } from './commands/setup.js';
|
||||
import { FAIL, bold, dim } from './ui.js';
|
||||
|
||||
function printUsage(): void {
|
||||
console.error(`${bold('Usage:')} mlflow-codex <command> [options]`);
|
||||
console.error('');
|
||||
console.error(bold('Commands:'));
|
||||
console.error(
|
||||
` ${bold('setup')} Register the notify hook in ~/.codex/config.toml and configure`,
|
||||
);
|
||||
console.error(' the MLflow tracking URI / experiment ID. Runs interactively');
|
||||
console.error(' by default.');
|
||||
console.error('');
|
||||
console.error(` ${dim('Flags:')}`);
|
||||
console.error(
|
||||
dim(
|
||||
' --project, -p Write to ./.codex/ (skip the interactive scope prompt)',
|
||||
),
|
||||
);
|
||||
console.error(
|
||||
dim(' --non-interactive, -y Skip prompts; use flag values or defaults'),
|
||||
);
|
||||
console.error(
|
||||
dim(' --tracking-uri <url> Bypass the prompt for the tracking URI'),
|
||||
);
|
||||
console.error(
|
||||
dim(' --experiment-id <id> Bypass the prompt for the experiment ID'),
|
||||
);
|
||||
console.error(
|
||||
dim(
|
||||
' --trace-location <loc> Databricks Unity Catalog trace location as\n' +
|
||||
" 'catalog.schema.table_prefix' (routes to UC ingestion)",
|
||||
),
|
||||
);
|
||||
console.error('');
|
||||
console.error(
|
||||
` ${bold('notify-hook')} Run the Codex notify handler. Codex appends the turn JSON as`,
|
||||
);
|
||||
console.error(' the final argument — this is the form Codex itself uses via');
|
||||
console.error(' config.toml.');
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const [, , command, ...rest] = process.argv;
|
||||
|
||||
if (command === undefined || command === '--help' || command === '-h' || command === 'help') {
|
||||
printUsage();
|
||||
if (command === undefined) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === 'setup') {
|
||||
await runSetup(rest);
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === 'notify-hook') {
|
||||
const payload = rest[0];
|
||||
if (payload === undefined) {
|
||||
console.error(`${FAIL} notify-hook expects a JSON payload as the first argument`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
await runNotifyHook(payload);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(`${FAIL} Unknown command: ${bold(command)}\n`);
|
||||
printUsage();
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -0,0 +1,293 @@
|
||||
/**
|
||||
* `mlflow-codex setup` — interactively register the notify hook in the user's
|
||||
* Codex config and write an MLflow tracing config alongside it.
|
||||
*
|
||||
* Writes:
|
||||
* - `config.toml` — prepends `notify = ["mlflow-codex", "notify-hook"]`
|
||||
* ahead of any `[section]` headers. Refuses to modify a pre-existing
|
||||
* `notify = ...` entry to avoid mangling the user's config.
|
||||
* - `mlflow-tracing.json` — persists the tracking URI and experiment ID
|
||||
* so the hook can run without shell exports.
|
||||
*
|
||||
* Scope selection:
|
||||
* - Interactive runs prompt for project-local (`./.codex/`) vs user-level
|
||||
* (`~/.codex/`), defaulting to project-local.
|
||||
* - `--project` / `-p` forces project-local and skips the prompt.
|
||||
* - `--non-interactive` defaults to project-local, matching the interactive
|
||||
* default.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { createInterface } from 'node:readline';
|
||||
|
||||
import { parseTraceLocation } from '../config.js';
|
||||
import { FAIL, OK, WARN, bold, cyan, dim } from '../ui.js';
|
||||
import { selectPrompt } from '../ui-select.js';
|
||||
|
||||
const HOOK_LINE = 'notify = ["mlflow-codex", "notify-hook"]';
|
||||
const NOTIFY_LINE_RE = /^\s*notify\s*=.*$/m;
|
||||
const NOTIFY_HAS_MLFLOW_RE = /^\s*notify\s*=.*["']mlflow-codex["']/m;
|
||||
const DEFAULT_TRACKING_URI = 'http://localhost:5000';
|
||||
const DEFAULT_EXPERIMENT_ID = '0';
|
||||
|
||||
/**
|
||||
* Returns true only when `raw` parses as a URL with an http(s) scheme.
|
||||
* Rejects scheme-less inputs like `localhost:5000` (which `new URL` otherwise
|
||||
* interprets as a custom scheme with an empty port).
|
||||
*/
|
||||
export function isValidTrackingUri(raw: string): boolean {
|
||||
if (raw === 'databricks' || raw.startsWith('databricks://')) {
|
||||
return true;
|
||||
}
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return parsed.protocol === 'http:' || parsed.protocol === 'https:';
|
||||
}
|
||||
|
||||
export interface SetupOptions {
|
||||
/** Override the user home directory. Defaults to `os.homedir()`. */
|
||||
home?: string;
|
||||
/** Override the current working directory. Defaults to `process.cwd()`. */
|
||||
cwd?: string;
|
||||
/** Pre-supplied tracking URI. Skips the interactive prompt. */
|
||||
trackingUri?: string;
|
||||
/** Pre-supplied experiment ID. Skips the interactive prompt. */
|
||||
experimentId?: string;
|
||||
/**
|
||||
* Optional Databricks Unity Catalog trace location as
|
||||
* `catalog.schema.table_prefix`. When set, traces are routed to the UC
|
||||
* table-prefix destination instead of the experiment-backed path.
|
||||
*/
|
||||
traceLocation?: string;
|
||||
/** Suppress prompts entirely. Uses defaults for any unset values. */
|
||||
nonInteractive?: boolean;
|
||||
}
|
||||
|
||||
export function resolveConfigPath(projectLocal: boolean, options: SetupOptions = {}): string {
|
||||
return projectLocal
|
||||
? resolve(options.cwd ?? process.cwd(), '.codex', 'config.toml')
|
||||
: resolve(options.home ?? homedir(), '.codex', 'config.toml');
|
||||
}
|
||||
|
||||
function resolveTracingConfigPath(projectLocal: boolean, options: SetupOptions = {}): string {
|
||||
return projectLocal
|
||||
? resolve(options.cwd ?? process.cwd(), '.codex', 'mlflow-tracing.json')
|
||||
: resolve(options.home ?? homedir(), '.codex', 'mlflow-tracing.json');
|
||||
}
|
||||
|
||||
function writeConfigWithHook(path: string, original: string | null): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
const prefix = `# Added by \`mlflow-codex setup\` — forwards each Codex turn to MLflow Tracing.\n${HOOK_LINE}\n`;
|
||||
const content = original ? prefix + '\n' + original : prefix;
|
||||
writeFileSync(path, content, 'utf-8');
|
||||
}
|
||||
|
||||
function writeTracingConfig(
|
||||
path: string,
|
||||
config: { trackingUri: string; experimentId: string; traceLocation?: string },
|
||||
): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, JSON.stringify(config, null, 2) + '\n', 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse raw CLI args for the setup command. Supports:
|
||||
* --project / -p
|
||||
* --non-interactive / -y
|
||||
* --tracking-uri <url>
|
||||
* --experiment-id <id>
|
||||
* --trace-location <catalog.schema.table_prefix>
|
||||
*
|
||||
* `projectLocal` is left undefined when `--project` is not passed so the
|
||||
* caller can apply its own default (interactive prompts; non-interactive
|
||||
* defaults to project-local).
|
||||
*/
|
||||
export function parseSetupArgs(
|
||||
args: string[],
|
||||
): SetupOptions & { projectLocal: boolean | undefined } {
|
||||
const out: SetupOptions & { projectLocal: boolean | undefined } = { projectLocal: undefined };
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
if (arg === '--project' || arg === '-p') {
|
||||
out.projectLocal = true;
|
||||
} else if (arg === '--non-interactive' || arg === '-y') {
|
||||
out.nonInteractive = true;
|
||||
} else if (arg === '--tracking-uri') {
|
||||
out.trackingUri = args[++i];
|
||||
} else if (arg === '--experiment-id') {
|
||||
out.experimentId = args[++i];
|
||||
} else if (arg === '--trace-location') {
|
||||
out.traceLocation = args[++i];
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
type Readline = ReturnType<typeof createInterface>;
|
||||
|
||||
function askOn(
|
||||
rl: Readline,
|
||||
label: string,
|
||||
defaultValue: string,
|
||||
validate?: (value: string) => string | null,
|
||||
): Promise<string> {
|
||||
return new Promise((resolvePromise) => {
|
||||
const ask = (): void => {
|
||||
rl.question(` ${label} ${dim(`[${defaultValue}]`)} `, (answer) => {
|
||||
const value = answer.trim() || defaultValue;
|
||||
const err = validate?.(value);
|
||||
if (err) {
|
||||
console.error(` ${FAIL} ${err}`);
|
||||
ask();
|
||||
return;
|
||||
}
|
||||
resolvePromise(value);
|
||||
});
|
||||
};
|
||||
ask();
|
||||
});
|
||||
}
|
||||
|
||||
const validateTrackingUri = (value: string): string | null =>
|
||||
isValidTrackingUri(value) ? null : 'Must be an absolute http:// or https:// URL.';
|
||||
|
||||
function promptScope(): Promise<boolean> {
|
||||
return selectPrompt<boolean>({
|
||||
question: 'Where should MLflow tracing be installed?',
|
||||
options: [
|
||||
{ value: true, label: 'Project', hint: './.codex/' },
|
||||
{ value: false, label: 'User', hint: '~/.codex/' },
|
||||
],
|
||||
defaultIndex: 0,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runSetup(args: string[], options: SetupOptions = {}): Promise<void> {
|
||||
const parsed = parseSetupArgs(args);
|
||||
const merged: SetupOptions & { projectLocal: boolean | undefined } = {
|
||||
...parsed,
|
||||
...options,
|
||||
projectLocal: parsed.projectLocal,
|
||||
};
|
||||
|
||||
const interactive = !merged.nonInteractive;
|
||||
const needsBanner =
|
||||
interactive &&
|
||||
(merged.projectLocal === undefined || !merged.trackingUri || !merged.experimentId);
|
||||
if (needsBanner) {
|
||||
console.error(`\n${bold('Configure MLflow tracing for Codex CLI')}`);
|
||||
}
|
||||
|
||||
const projectLocal =
|
||||
merged.projectLocal !== undefined
|
||||
? merged.projectLocal
|
||||
: interactive
|
||||
? await promptScope()
|
||||
: true;
|
||||
|
||||
const configPath = resolveConfigPath(projectLocal, merged);
|
||||
const tracingConfigPath = resolveTracingConfigPath(projectLocal, merged);
|
||||
|
||||
let hookRegistered = false;
|
||||
if (!existsSync(configPath)) {
|
||||
writeConfigWithHook(configPath, null);
|
||||
console.error(`${OK} Created ${cyan(configPath)} with notify hook`);
|
||||
hookRegistered = true;
|
||||
} else {
|
||||
const content = readFileSync(configPath, 'utf-8');
|
||||
if (NOTIFY_LINE_RE.test(content)) {
|
||||
if (NOTIFY_HAS_MLFLOW_RE.test(content)) {
|
||||
console.error(`${WARN} Notify hook already registered in ${cyan(configPath)}`);
|
||||
hookRegistered = true;
|
||||
} else {
|
||||
console.error(`${FAIL} ${cyan(configPath)} already has a \`notify = ...\` entry.`);
|
||||
console.error(` Update it manually to: ${bold(HOOK_LINE)}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
writeConfigWithHook(configPath, content);
|
||||
console.error(`${OK} Added notify hook to ${cyan(configPath)}`);
|
||||
hookRegistered = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!hookRegistered) {
|
||||
return;
|
||||
}
|
||||
console.error('');
|
||||
|
||||
const needsTextPrompt = interactive && (!merged.trackingUri || !merged.experimentId);
|
||||
const rl = needsTextPrompt
|
||||
? createInterface({ input: process.stdin, output: process.stdout })
|
||||
: null;
|
||||
try {
|
||||
const trackingUri =
|
||||
merged.trackingUri ??
|
||||
(rl
|
||||
? await askOn(rl, 'MLflow tracking URI', DEFAULT_TRACKING_URI, validateTrackingUri)
|
||||
: DEFAULT_TRACKING_URI);
|
||||
const experimentId =
|
||||
merged.experimentId ??
|
||||
(rl ? await askOn(rl, 'MLflow experiment ID', DEFAULT_EXPERIMENT_ID) : DEFAULT_EXPERIMENT_ID);
|
||||
|
||||
writeTracingConfigIfValid(tracingConfigPath, trackingUri, experimentId, merged.traceLocation);
|
||||
} finally {
|
||||
rl?.close();
|
||||
}
|
||||
}
|
||||
|
||||
function writeTracingConfigIfValid(
|
||||
tracingConfigPath: string,
|
||||
trackingUri: string,
|
||||
experimentId: string,
|
||||
traceLocation?: string,
|
||||
): void {
|
||||
if (!isValidTrackingUri(trackingUri)) {
|
||||
console.error(
|
||||
`${FAIL} Invalid tracking URI: ${bold(trackingUri)} - must be an absolute http:// or https:// URL, or a databricks URI.`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (traceLocation && !parseTraceLocation(traceLocation)) {
|
||||
console.error(
|
||||
`${FAIL} Invalid trace location: ${bold(traceLocation)} - must be in 'catalog.schema.table_prefix' format.`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
writeTracingConfig(tracingConfigPath, {
|
||||
trackingUri,
|
||||
experimentId,
|
||||
...(traceLocation ? { traceLocation } : {}),
|
||||
});
|
||||
console.error(`\n${OK} Wrote tracing config to ${cyan(tracingConfigPath)}`);
|
||||
if (traceLocation) {
|
||||
console.error(` Trace location: ${bold(traceLocation)}`);
|
||||
}
|
||||
|
||||
console.error(`\n${bold('Next steps')}`);
|
||||
if (trackingUri.startsWith('databricks')) {
|
||||
const destination = traceLocation ? `UC location ${bold(traceLocation)}` : bold(trackingUri);
|
||||
console.error(
|
||||
` 1. Launch ${cyan('codex')} - traces appear in ${destination} after each turn.`,
|
||||
);
|
||||
} else {
|
||||
const port = new URL(trackingUri).port || '5000';
|
||||
console.error(' 1. Start the MLflow tracking server in a separate terminal:');
|
||||
console.error(` ${cyan(`mlflow server --port ${port}`)}`);
|
||||
console.error(
|
||||
` 2. Launch ${cyan('codex')} - traces appear at ${bold(trackingUri)} after each turn.`,
|
||||
);
|
||||
}
|
||||
console.error(
|
||||
`\n${dim('Override per-shell with $MLFLOW_TRACKING_URI / $MLFLOW_EXPERIMENT_ID / $MLFLOW_TRACE_LOCATION.')}`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
import { init } from '@mlflow/core';
|
||||
|
||||
let initialized = false;
|
||||
|
||||
const TRACING_CONFIG_FILE = 'mlflow-tracing.json';
|
||||
|
||||
export interface TracingConfig {
|
||||
trackingUri?: string;
|
||||
experimentId?: string;
|
||||
/** Raw `catalog.schema.table_prefix` UC trace location, if configured. */
|
||||
traceLocation?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A Databricks Unity Catalog trace location parsed from a
|
||||
* `catalog.schema.table_prefix` string.
|
||||
*/
|
||||
export interface UnityCatalogTraceLocation {
|
||||
catalogName: string;
|
||||
schemaName: string;
|
||||
tablePrefix: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a `catalog.schema.table_prefix` string into a UC trace location.
|
||||
* Returns null when the value is empty or not exactly three non-empty,
|
||||
* dot-separated parts. All three parts are required because the SDK does not
|
||||
* create UC trace locations - the customer must point at a provisioned one.
|
||||
*/
|
||||
export function parseTraceLocation(value: string | undefined): UnityCatalogTraceLocation | null {
|
||||
if (!value || value.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
const parts = value.trim().split('.');
|
||||
if (parts.length !== 3 || parts.some((part) => part.trim().length === 0)) {
|
||||
return null;
|
||||
}
|
||||
const [catalogName, schemaName, tablePrefix] = parts.map((part) => part.trim());
|
||||
return { catalogName, schemaName, tablePrefix };
|
||||
}
|
||||
|
||||
export interface ResolveConfigOptions {
|
||||
/** Override the user home directory. Defaults to `os.homedir()`. */
|
||||
home?: string;
|
||||
/** Override the current working directory. Defaults to `process.cwd()`. */
|
||||
cwd?: string;
|
||||
}
|
||||
|
||||
function readTracingConfigFile(path: string): TracingConfig {
|
||||
if (!existsSync(path)) {
|
||||
return {};
|
||||
}
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, 'utf-8')) as TracingConfig;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the effective MLflow tracing config. Precedence (highest first):
|
||||
* 1. `MLFLOW_TRACKING_URI` / `MLFLOW_EXPERIMENT_ID` / `MLFLOW_TRACE_LOCATION`
|
||||
* environment variables
|
||||
* 2. `./.codex/mlflow-tracing.json` (project-local)
|
||||
* 3. `~/.codex/mlflow-tracing.json` (user-level)
|
||||
*/
|
||||
export function resolveTracingConfig(options: ResolveConfigOptions = {}): TracingConfig {
|
||||
const projectPath = resolve(options.cwd ?? process.cwd(), '.codex', TRACING_CONFIG_FILE);
|
||||
const userPath = resolve(options.home ?? homedir(), '.codex', TRACING_CONFIG_FILE);
|
||||
const projectConfig = readTracingConfigFile(projectPath);
|
||||
const userConfig = readTracingConfigFile(userPath);
|
||||
return {
|
||||
trackingUri:
|
||||
process.env.MLFLOW_TRACKING_URI ?? projectConfig.trackingUri ?? userConfig.trackingUri,
|
||||
experimentId:
|
||||
process.env.MLFLOW_EXPERIMENT_ID ?? projectConfig.experimentId ?? userConfig.experimentId,
|
||||
traceLocation:
|
||||
process.env.MLFLOW_TRACE_LOCATION ?? projectConfig.traceLocation ?? userConfig.traceLocation,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the MLflow SDK with tracking URI and experiment settings.
|
||||
* No-ops if already initialized or if required config is missing.
|
||||
*/
|
||||
export function ensureInitialized(): boolean {
|
||||
if (initialized) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const { trackingUri, experimentId, traceLocation: rawTraceLocation } = resolveTracingConfig();
|
||||
if (!trackingUri) {
|
||||
console.error(
|
||||
'[mlflow] MLflow tracking URI is not configured - checked $MLFLOW_TRACKING_URI,',
|
||||
'./.codex/mlflow-tracing.json, and ~/.codex/mlflow-tracing.json.',
|
||||
);
|
||||
console.error('[mlflow] Run `mlflow-codex setup` to configure.');
|
||||
return false;
|
||||
}
|
||||
|
||||
let traceLocation: UnityCatalogTraceLocation | null = null;
|
||||
if (rawTraceLocation && rawTraceLocation.trim().length > 0) {
|
||||
traceLocation = parseTraceLocation(rawTraceLocation);
|
||||
if (!traceLocation) {
|
||||
console.error(
|
||||
`[mlflow] MLFLOW_TRACE_LOCATION must be in 'catalog.schema.table_prefix' format, ` +
|
||||
`got '${rawTraceLocation}'`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
init({
|
||||
trackingUri,
|
||||
experimentId,
|
||||
...(traceLocation ? { traceLocation } : {}),
|
||||
});
|
||||
initialized = true;
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* Codex notify hook handler.
|
||||
*
|
||||
* Codex passes the turn data as a JSON string in the first CLI argument:
|
||||
* node cli.js '{"type":"agent-turn-complete","thread-id":"...","input-messages":[...],...}'
|
||||
*
|
||||
* Configured in ~/.codex/config.toml:
|
||||
* notify = ["mlflow-codex"]
|
||||
*/
|
||||
|
||||
import { ensureInitialized } from '../config.js';
|
||||
import { processNotify } from '../tracing.js';
|
||||
import type { NotifyPayload } from '../types.js';
|
||||
|
||||
export async function runNotifyHook(rawPayload: string): Promise<void> {
|
||||
try {
|
||||
if (!ensureInitialized()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const payload = JSON.parse(rawPayload) as NotifyPayload;
|
||||
if (payload.type !== 'agent-turn-complete') {
|
||||
return;
|
||||
}
|
||||
|
||||
await processNotify(payload);
|
||||
} catch (err) {
|
||||
console.error('[mlflow]', err);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
export { processNotify } from './tracing.js';
|
||||
export { ensureInitialized } from './config.js';
|
||||
export {
|
||||
readTranscript,
|
||||
parseTimestampToNs,
|
||||
extractTextFromContent,
|
||||
findLastUserPrompt,
|
||||
getLastTurnRecords,
|
||||
getTokenUsage,
|
||||
getModel,
|
||||
getSessionId,
|
||||
buildToolResultMap,
|
||||
findTranscriptForThread,
|
||||
} from './transcript.js';
|
||||
export type {
|
||||
NotifyPayload,
|
||||
RolloutLine,
|
||||
SessionMetaPayload,
|
||||
ResponseItemPayload,
|
||||
ContentBlock,
|
||||
EventMsgPayload,
|
||||
TokenCountInfo,
|
||||
TokenUsage,
|
||||
} from './types.js';
|
||||
@@ -0,0 +1,400 @@
|
||||
/**
|
||||
* MLflow tracing integration for Codex CLI.
|
||||
*
|
||||
* Two integration modes:
|
||||
*
|
||||
* 1. **Notify hook** (recommended): Codex passes turn data as a JSON CLI arg
|
||||
* after each agent turn. Simple, no transcript parsing needed.
|
||||
* Configured via `notify` in config.toml.
|
||||
*
|
||||
* 2. **Transcript parsing**: Reads the rollout JSONL file for richer data
|
||||
* (tool calls, token usage). Used when transcript_path is available.
|
||||
*
|
||||
* References:
|
||||
* - Notify hook: developers.openai.com/codex/hooks
|
||||
* - Protocol types: github.com/openai/codex codex-rs/protocol/src/protocol.rs
|
||||
* - Rollout recorder: github.com/openai/codex codex-rs/rollout/src/recorder.rs
|
||||
*/
|
||||
|
||||
import {
|
||||
startSpan,
|
||||
flushTraces,
|
||||
InMemoryTraceManager,
|
||||
SpanStatusCode,
|
||||
SpanType,
|
||||
SpanAttributeKey,
|
||||
TraceMetadataKey,
|
||||
TokenUsageKey,
|
||||
type LiveSpan,
|
||||
} from '@mlflow/core';
|
||||
|
||||
import type {
|
||||
ChatMessage,
|
||||
EventMsgPayload,
|
||||
NotifyPayload,
|
||||
RolloutLine,
|
||||
ResponseItemPayload,
|
||||
} from './types.js';
|
||||
import {
|
||||
parseTimestampToNs,
|
||||
extractTextFromContent,
|
||||
getTokenUsage,
|
||||
getModel,
|
||||
buildToolResultMap,
|
||||
findTranscriptForThread,
|
||||
getLastTurnRecords,
|
||||
readTranscript,
|
||||
} from './transcript.js';
|
||||
|
||||
/**
|
||||
* Process a Codex notify hook payload and create an MLflow trace.
|
||||
*
|
||||
* The notify payload has the user prompt and assistant response directly,
|
||||
* so we create a simple AGENT → LLM trace. If a transcript file is found,
|
||||
* we also parse it for tool calls and token usage.
|
||||
*/
|
||||
export async function processNotify(payload: NotifyPayload): Promise<void> {
|
||||
// input-messages accumulates all prompts in the session; take only the last one
|
||||
const inputMessages = payload['input-messages'] ?? [];
|
||||
const userPrompt = inputMessages[inputMessages.length - 1] ?? '';
|
||||
const assistantResponse = payload['last-assistant-message'] ?? '';
|
||||
const sessionId = payload['thread-id'];
|
||||
|
||||
if (!userPrompt) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Try to find and parse the transcript for richer data (tool calls, tokens)
|
||||
const transcriptPath = findTranscriptForThread(sessionId);
|
||||
let turnRecords: RolloutLine[] | null = null;
|
||||
let model = 'unknown';
|
||||
|
||||
if (transcriptPath) {
|
||||
const records = readTranscript(transcriptPath);
|
||||
if (records.length > 0) {
|
||||
turnRecords = getLastTurnRecords(records);
|
||||
model = getModel(records);
|
||||
}
|
||||
}
|
||||
|
||||
// Root span bracket: use task_started / task_complete from the transcript
|
||||
// so the root span covers the full turn. Without this, the root span would
|
||||
// only cover the hook's own wall-clock execution time (a few ms), which is
|
||||
// AFTER the transcript timestamps on the children. That makes the waterfall
|
||||
// scale to ~turn_duration + hook_delay with the root as a sliver at the
|
||||
// right edge. Opencode uses the same pattern with message.time.created /
|
||||
// .completed.
|
||||
const rootStartNs = turnRecords ? findTaskStartedNs(turnRecords) : null;
|
||||
const rootEndNs = turnRecords ? findTaskCompleteNs(turnRecords) : null;
|
||||
|
||||
// Create root AGENT span. Pass the user prompt as a raw string so MLflow
|
||||
// can auto-generate the request preview and the session view renders the
|
||||
// message cleanly.
|
||||
const rootSpan = startSpan({
|
||||
name: 'codex_conversation',
|
||||
spanType: SpanType.AGENT,
|
||||
inputs: userPrompt,
|
||||
attributes: { model },
|
||||
...(rootStartNs != null ? { startTimeNs: rootStartNs } : {}),
|
||||
});
|
||||
|
||||
// If we have transcript data, create detailed child spans
|
||||
if (turnRecords && turnRecords.length > 0) {
|
||||
createChildSpans(rootSpan, turnRecords, model);
|
||||
|
||||
const tokenUsage = getTokenUsage(turnRecords);
|
||||
if (tokenUsage) {
|
||||
rootSpan.setAttribute(SpanAttributeKey.TOKEN_USAGE, {
|
||||
[TokenUsageKey.INPUT_TOKENS]: tokenUsage.input_tokens,
|
||||
[TokenUsageKey.OUTPUT_TOKENS]: tokenUsage.output_tokens,
|
||||
[TokenUsageKey.TOTAL_TOKENS]: tokenUsage.total_tokens,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Fallback: create a simple LLM span from the notify data using the same
|
||||
// OpenAI chat format the transcript path produces.
|
||||
const llmSpan = startSpan({
|
||||
name: 'llm_call',
|
||||
parent: rootSpan,
|
||||
spanType: SpanType.LLM,
|
||||
inputs: {
|
||||
model,
|
||||
messages: [{ role: 'user', content: userPrompt }],
|
||||
},
|
||||
attributes: { model },
|
||||
});
|
||||
llmSpan.end({
|
||||
outputs: {
|
||||
choices: [{ message: { role: 'assistant', content: assistantResponse } }],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Attach session/user metadata to the trace. We use InMemoryTraceManager
|
||||
// directly because `updateCurrentTrace()` requires an active OTel span
|
||||
// context, which hook-based integrations don't have — spans are created
|
||||
// via `startSpan()` without OTel context propagation.
|
||||
const traceId = rootSpan.traceId;
|
||||
if (traceId) {
|
||||
const traceManager = InMemoryTraceManager.getInstance();
|
||||
const trace = traceManager.getTrace(traceId);
|
||||
if (trace) {
|
||||
trace.info.traceMetadata = {
|
||||
...trace.info.traceMetadata,
|
||||
[TraceMetadataKey.TRACE_SESSION]: sessionId,
|
||||
[TraceMetadataKey.TRACE_USER]: process.env.USER ?? '',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
rootSpan.end({
|
||||
outputs: assistantResponse,
|
||||
...(rootEndNs != null ? { endTimeNs: rootEndNs } : {}),
|
||||
});
|
||||
|
||||
await flushTraces();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconstruct OpenAI chat-format message history from response_items preceding
|
||||
* the current index. Used to populate LLM span inputs so the MLflow Chat view
|
||||
* shows the full conversation context that led up to each assistant call.
|
||||
*
|
||||
* Maps Codex's Responses-API-style records to standard chat messages:
|
||||
* - `message` (user/assistant/system) → `{role, content}`
|
||||
* - `function_call` → assistant message with `tool_calls: [{id, type, function}]`
|
||||
* - `function_call_output` → `{role: 'tool', tool_call_id, content}`
|
||||
*/
|
||||
export function reconstructMessages(
|
||||
responseItems: RolloutLine[],
|
||||
uptoIndex: number,
|
||||
): ChatMessage[] {
|
||||
const messages: ChatMessage[] = [];
|
||||
for (let i = 0; i < uptoIndex; i++) {
|
||||
const payload = responseItems[i].payload as ResponseItemPayload;
|
||||
|
||||
if (payload.type === 'message') {
|
||||
const text = extractTextFromContent(payload.content);
|
||||
if (!text.trim()) {
|
||||
continue;
|
||||
}
|
||||
if (payload.role === 'user' || payload.role === 'assistant') {
|
||||
messages.push({ role: payload.role, content: text });
|
||||
} else if (payload.role === 'developer') {
|
||||
// Codex uses "developer" for system-style instructions; render as system
|
||||
messages.push({ role: 'system', content: text });
|
||||
}
|
||||
} else if (payload.type === 'function_call') {
|
||||
messages.push({
|
||||
role: 'assistant',
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: payload.call_id ?? '',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: payload.name ?? 'unknown',
|
||||
arguments: payload.arguments ?? '{}',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
} else if (payload.type === 'function_call_output') {
|
||||
messages.push({
|
||||
role: 'tool',
|
||||
tool_call_id: payload.call_id ?? '',
|
||||
content: payload.output ?? '',
|
||||
});
|
||||
}
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create LLM and TOOL child spans from transcript turn records.
|
||||
*
|
||||
* Timing model:
|
||||
* - LLM span covers "LLM thinking": from the last boundary (turn start or
|
||||
* the previous `function_call_output`) to the `message/assistant` record.
|
||||
* - TOOL span covers the actual tool call: from the `function_call` record
|
||||
* to the matching `function_call_output` record (matched by call_id).
|
||||
*
|
||||
* Using the record's own timestamp as both start and end — or chaining to
|
||||
* the next response_item — would produce spans that represent "time between
|
||||
* records" rather than the work each span describes. The record's timestamp
|
||||
* marks when the event was logged, which for an assistant message is when
|
||||
* generation *finished*, not when it started.
|
||||
*/
|
||||
export function createChildSpans(
|
||||
parentSpan: LiveSpan,
|
||||
turnRecords: RolloutLine[],
|
||||
model: string,
|
||||
): void {
|
||||
const toolResults = buildToolResultMap(turnRecords);
|
||||
const toolEndTimes = buildToolEndTimes(turnRecords);
|
||||
const toolStatuses = buildToolStatuses(turnRecords);
|
||||
|
||||
// Initial boundary for the first LLM span: the turn's task_started event,
|
||||
// if present. Falls back to null so the LLM span omits startTimeNs.
|
||||
let prevBoundaryNs: number | null = findTaskStartedNs(turnRecords);
|
||||
|
||||
const responseItems = turnRecords.filter((record) => record.type === 'response_item');
|
||||
|
||||
for (let i = 0; i < responseItems.length; i++) {
|
||||
const record = responseItems[i];
|
||||
const payload = record.payload as ResponseItemPayload;
|
||||
const timestampNs = parseTimestampToNs(record.timestamp);
|
||||
if (timestampNs == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (payload.type === 'message' && payload.role === 'assistant') {
|
||||
const text = extractTextFromContent(payload.content);
|
||||
if (text.trim()) {
|
||||
const messages = reconstructMessages(responseItems, i);
|
||||
const llmSpan = startSpan({
|
||||
name: 'llm_call',
|
||||
parent: parentSpan,
|
||||
spanType: SpanType.LLM,
|
||||
startTimeNs: prevBoundaryNs ?? timestampNs,
|
||||
inputs: { model, messages },
|
||||
attributes: { model },
|
||||
});
|
||||
llmSpan.end({
|
||||
outputs: {
|
||||
choices: [{ message: { role: 'assistant', content: text } }],
|
||||
},
|
||||
endTimeNs: timestampNs,
|
||||
});
|
||||
prevBoundaryNs = timestampNs;
|
||||
}
|
||||
} else if (payload.type === 'function_call') {
|
||||
const callId = payload.call_id ?? '';
|
||||
const funcName = payload.name ?? 'unknown';
|
||||
let args: Record<string, unknown> = {};
|
||||
try {
|
||||
args = JSON.parse(payload.arguments ?? '{}');
|
||||
} catch {
|
||||
// keep empty
|
||||
}
|
||||
|
||||
const toolSpan = startSpan({
|
||||
name: `tool_${funcName}`,
|
||||
parent: parentSpan,
|
||||
spanType: SpanType.TOOL,
|
||||
startTimeNs: timestampNs,
|
||||
inputs: args,
|
||||
attributes: { tool_name: funcName, tool_id: callId },
|
||||
});
|
||||
// Reflect tool failure in the span status so failed calls are visible
|
||||
// in the trace UI. Codex emits `exec_command_end` event_msg records
|
||||
// with structured status/exit_code — see buildToolStatuses.
|
||||
const toolStatus = toolStatuses[callId];
|
||||
if (toolStatus && toolStatus.failed) {
|
||||
toolSpan.setStatus(
|
||||
SpanStatusCode.ERROR,
|
||||
toolStatus.exitCode != null
|
||||
? `Tool call failed (exit code ${toolStatus.exitCode})`
|
||||
: 'Tool call failed',
|
||||
);
|
||||
}
|
||||
toolSpan.end({
|
||||
outputs: { result: toolResults[callId] ?? '' },
|
||||
endTimeNs: toolEndTimes[callId] ?? timestampNs,
|
||||
});
|
||||
} else if (payload.type === 'function_call_output') {
|
||||
// Tool result logged; the next LLM span should start from here, since
|
||||
// the LLM is waiting on tool output until this point.
|
||||
prevBoundaryNs = timestampNs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the turn's `task_started` event_msg timestamp in nanoseconds.
|
||||
* Returns null if the turn doesn't include a task_started event.
|
||||
*/
|
||||
export function findTaskStartedNs(turnRecords: RolloutLine[]): number | null {
|
||||
for (const record of turnRecords) {
|
||||
if (record.type === 'event_msg') {
|
||||
const payload = record.payload as EventMsgPayload;
|
||||
if (payload.type === 'task_started') {
|
||||
return parseTimestampToNs(record.timestamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the turn's `task_complete` event_msg timestamp in nanoseconds.
|
||||
* Returns null if the turn doesn't include a task_complete event (in-progress
|
||||
* turns may not have one yet).
|
||||
*/
|
||||
export function findTaskCompleteNs(turnRecords: RolloutLine[]): number | null {
|
||||
for (let i = turnRecords.length - 1; i >= 0; i--) {
|
||||
const record = turnRecords[i];
|
||||
if (record.type === 'event_msg') {
|
||||
const payload = record.payload as EventMsgPayload;
|
||||
if (payload.type === 'task_complete') {
|
||||
return parseTimestampToNs(record.timestamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a lookup from function call_id to its `function_call_output`
|
||||
* timestamp (ns). Used to derive accurate TOOL span end times instead of
|
||||
* chaining to the next response_item, which may not be the matching output.
|
||||
*/
|
||||
function buildToolEndTimes(turnRecords: RolloutLine[]): Record<string, number> {
|
||||
const endTimes: Record<string, number> = {};
|
||||
for (const record of turnRecords) {
|
||||
if (record.type !== 'response_item') {
|
||||
continue;
|
||||
}
|
||||
const payload = record.payload as ResponseItemPayload;
|
||||
if (payload.type === 'function_call_output' && payload.call_id) {
|
||||
const ts = parseTimestampToNs(record.timestamp);
|
||||
if (ts != null) {
|
||||
endTimes[payload.call_id] = ts;
|
||||
}
|
||||
}
|
||||
}
|
||||
return endTimes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a lookup from function call_id to its outcome, derived from the
|
||||
* Codex `exec_command_end` event_msg which has structured `status` and
|
||||
* `exit_code` fields. Codex uses `status: 'failed'` for failed commands
|
||||
* (e.g. exit_code 127 for command-not-found).
|
||||
*
|
||||
* Non-exec_command tools don't emit exec_command_end, so their call_ids
|
||||
* won't appear in the map and their spans stay in the default OK state.
|
||||
*/
|
||||
export function buildToolStatuses(
|
||||
turnRecords: RolloutLine[],
|
||||
): Record<string, { failed: boolean; exitCode: number | null }> {
|
||||
const statuses: Record<string, { failed: boolean; exitCode: number | null }> = {};
|
||||
for (const record of turnRecords) {
|
||||
if (record.type !== 'event_msg') {
|
||||
continue;
|
||||
}
|
||||
const payload = record.payload as EventMsgPayload & {
|
||||
call_id?: string;
|
||||
exit_code?: number;
|
||||
status?: string;
|
||||
};
|
||||
if (payload.type === 'exec_command_end' && payload.call_id) {
|
||||
const failed = payload.status === 'failed' || (payload.exit_code ?? 0) !== 0;
|
||||
statuses[payload.call_id] = {
|
||||
failed,
|
||||
exitCode: payload.exit_code ?? null,
|
||||
};
|
||||
}
|
||||
}
|
||||
return statuses;
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* Transcript parsing utilities for Codex CLI rollout JSONL files.
|
||||
*
|
||||
* Codex CLI transcripts use a RolloutLine format defined in
|
||||
* codex-rs/protocol/src/protocol.rs. Each line is:
|
||||
* {"timestamp": "...", "type": "<variant>", "payload": {...}}
|
||||
*
|
||||
* Turns are delimited by event_msg task_started / task_complete pairs.
|
||||
*
|
||||
* References:
|
||||
* - Protocol types: github.com/openai/codex codex-rs/protocol/src/protocol.rs
|
||||
* - Rollout recorder: github.com/openai/codex codex-rs/rollout/src/recorder.rs
|
||||
*/
|
||||
|
||||
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { homedir } from 'node:os';
|
||||
import type {
|
||||
RolloutLine,
|
||||
ResponseItemPayload,
|
||||
EventMsgPayload,
|
||||
SessionMetaPayload,
|
||||
TokenUsage,
|
||||
ContentBlock,
|
||||
} from './types.js';
|
||||
|
||||
export const NANOSECONDS_PER_MS = 1e6;
|
||||
|
||||
/**
|
||||
* Read and parse a Codex JSONL transcript file.
|
||||
*/
|
||||
export function readTranscript(path: string): RolloutLine[] {
|
||||
const content = readFileSync(path, 'utf-8');
|
||||
return content
|
||||
.split('\n')
|
||||
.filter((line) => line.trim())
|
||||
.map((line) => JSON.parse(line) as RolloutLine);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an ISO timestamp string to nanoseconds since Unix epoch.
|
||||
*/
|
||||
export function parseTimestampToNs(timestamp: string | undefined | null): number | null {
|
||||
if (!timestamp) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const ms = new Date(timestamp).getTime();
|
||||
if (isNaN(ms)) {
|
||||
return null;
|
||||
}
|
||||
return ms * NANOSECONDS_PER_MS;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract text from a response_item content field.
|
||||
* Content is an array of ContentBlock objects with type "input_text" or "output_text".
|
||||
*/
|
||||
export function extractTextFromContent(content: ContentBlock[] | string | undefined): string {
|
||||
if (!content) {
|
||||
return '';
|
||||
}
|
||||
if (typeof content === 'string') {
|
||||
return content;
|
||||
}
|
||||
if (!Array.isArray(content)) {
|
||||
return '';
|
||||
}
|
||||
return content
|
||||
.filter((block) => block.type === 'input_text' || block.type === 'output_text')
|
||||
.map((block) => block.text)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the last user prompt in the transcript.
|
||||
* User prompts are response_item records with payload.type=message and payload.role=user
|
||||
* whose content has input_text blocks that aren't system/developer injections.
|
||||
*/
|
||||
export function findLastUserPrompt(records: RolloutLine[]): { text: string; index: number } | null {
|
||||
for (let i = records.length - 1; i >= 0; i--) {
|
||||
const record = records[i];
|
||||
if (record.type !== 'response_item') {
|
||||
continue;
|
||||
}
|
||||
const payload = record.payload as ResponseItemPayload;
|
||||
if (payload.type !== 'message' || payload.role !== 'user') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const text = extractTextFromContent(payload.content);
|
||||
// Skip system/developer context injections (start with XML-like tags)
|
||||
if (text && !text.startsWith('<')) {
|
||||
return { text, index: i };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract records belonging to the last turn.
|
||||
* Turns are delimited by event_msg records with type=task_started / task_complete.
|
||||
*/
|
||||
export function getLastTurnRecords(records: RolloutLine[]): RolloutLine[] {
|
||||
let lastStart: number | null = null;
|
||||
let lastEnd: number | null = null;
|
||||
|
||||
for (let i = 0; i < records.length; i++) {
|
||||
if (records[i].type !== 'event_msg') {
|
||||
continue;
|
||||
}
|
||||
const payload = records[i].payload as EventMsgPayload;
|
||||
if (payload.type === 'task_started') {
|
||||
lastStart = i;
|
||||
} else if (payload.type === 'task_complete') {
|
||||
lastEnd = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (lastStart != null) {
|
||||
// If lastEnd is before lastStart (or missing), the turn is in-progress — slice to end of file
|
||||
const end = lastEnd != null && lastEnd >= lastStart ? lastEnd + 1 : records.length;
|
||||
return records.slice(lastStart, end);
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract cumulative token usage from the last token_count event in a set of records.
|
||||
*/
|
||||
export function getTokenUsage(records: RolloutLine[]): TokenUsage | null {
|
||||
let usage: TokenUsage | null = null;
|
||||
for (const record of records) {
|
||||
if (record.type !== 'event_msg') {
|
||||
continue;
|
||||
}
|
||||
const payload = record.payload as EventMsgPayload;
|
||||
if (payload.type !== 'token_count') {
|
||||
continue;
|
||||
}
|
||||
if (payload.info?.last_token_usage) {
|
||||
usage = payload.info.last_token_usage;
|
||||
}
|
||||
}
|
||||
return usage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract model name from session_meta or turn_context records.
|
||||
*/
|
||||
export function getModel(records: RolloutLine[]): string {
|
||||
for (const record of records) {
|
||||
if (record.type === 'session_meta' || record.type === 'turn_context') {
|
||||
const model = (record.payload as Record<string, unknown>).model;
|
||||
if (typeof model === 'string') {
|
||||
return model;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract session ID from the session_meta record.
|
||||
*/
|
||||
export function getSessionId(records: RolloutLine[]): string | null {
|
||||
for (const record of records) {
|
||||
if (record.type === 'session_meta') {
|
||||
return (record.payload as SessionMetaPayload).id ?? null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a map from function_call call_id to function_call_output output.
|
||||
*/
|
||||
export function buildToolResultMap(records: RolloutLine[]): Record<string, string> {
|
||||
const results: Record<string, string> = {};
|
||||
for (const record of records) {
|
||||
if (record.type !== 'response_item') {
|
||||
continue;
|
||||
}
|
||||
const payload = record.payload as ResponseItemPayload;
|
||||
if (payload.type === 'function_call_output' && payload.call_id) {
|
||||
results[payload.call_id] = payload.output ?? '';
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the transcript rollout file for a given thread ID.
|
||||
*
|
||||
* Codex stores transcripts at:
|
||||
* ~/.codex/sessions/YYYY/MM/DD/rollout-<timestamp>-<thread-id>.jsonl
|
||||
*
|
||||
* This is optional enrichment — if not found, tracing still works
|
||||
* from the notify payload alone.
|
||||
*/
|
||||
export function findTranscriptForThread(threadId: string): string | null {
|
||||
try {
|
||||
const sessionsDir = join(homedir(), '.codex', 'sessions');
|
||||
if (!existsSync(sessionsDir)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Fast path: check today's directory first since the hook fires
|
||||
// right after a turn completes — the transcript is almost always
|
||||
// from the current date.
|
||||
const now = new Date();
|
||||
const year = String(now.getFullYear());
|
||||
const month = String(now.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(now.getDate()).padStart(2, '0');
|
||||
const todayDir = join(sessionsDir, year, month, day);
|
||||
|
||||
if (existsSync(todayDir)) {
|
||||
const files = readdirSync(todayDir).filter(
|
||||
(f) => f.endsWith('.jsonl') && f.includes(threadId),
|
||||
);
|
||||
if (files.length > 0) {
|
||||
return join(todayDir, files[0]);
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: walk year/month/day directories in reverse order.
|
||||
// Only needed if the session started before midnight and the hook
|
||||
// fires after, or the clock is off.
|
||||
const years = readdirSync(sessionsDir).sort().reverse();
|
||||
for (const y of years) {
|
||||
const yearDir = join(sessionsDir, y);
|
||||
const months = readdirSync(yearDir).sort().reverse();
|
||||
for (const m of months) {
|
||||
const monthDir = join(yearDir, m);
|
||||
const days = readdirSync(monthDir).sort().reverse();
|
||||
for (const d of days) {
|
||||
// Skip today's dir — already checked above
|
||||
if (y === year && m === month && d === day) {
|
||||
continue;
|
||||
}
|
||||
const dayDir = join(monthDir, d);
|
||||
const files = readdirSync(dayDir).filter(
|
||||
(f) => f.endsWith('.jsonl') && f.includes(threadId),
|
||||
);
|
||||
if (files.length > 0) {
|
||||
return join(dayDir, files[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Transcript lookup is best-effort
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* Types for Codex CLI notify hook integration.
|
||||
*
|
||||
* Codex CLI fires a `notify` hook after each agent turn, passing a JSON
|
||||
* argument with the turn data. This is configured in ~/.codex/config.toml:
|
||||
* notify = ["node", "/path/to/stop.js"]
|
||||
*
|
||||
* The JSON is passed as the first command-line argument (argv[2]).
|
||||
*
|
||||
* Reference: https://developers.openai.com/codex/hooks
|
||||
*/
|
||||
|
||||
/**
|
||||
* Notify hook payload — passed as a CLI argument JSON string.
|
||||
* Fired after each agent turn completes.
|
||||
*/
|
||||
export interface NotifyPayload {
|
||||
type: 'agent-turn-complete';
|
||||
'thread-id': string;
|
||||
'turn-id': string;
|
||||
cwd: string;
|
||||
client: string;
|
||||
'input-messages': string[];
|
||||
'last-assistant-message': string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Types below are for transcript parsing (rollout JSONL files).
|
||||
* Defined in codex-rs/protocol/src/protocol.rs (tagged enum `RolloutItem`).
|
||||
* Stored at ~/.codex/sessions/YYYY/MM/DD/rollout-<timestamp>-<session_id>.jsonl.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A single line in the Codex rollout JSONL transcript.
|
||||
*/
|
||||
export interface RolloutLine {
|
||||
timestamp: string;
|
||||
type: 'session_meta' | 'response_item' | 'event_msg' | 'turn_context' | 'compacted';
|
||||
payload: SessionMetaPayload | ResponseItemPayload | EventMsgPayload | Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface SessionMetaPayload {
|
||||
id: string;
|
||||
timestamp: string;
|
||||
cwd: string;
|
||||
originator: string;
|
||||
cli_version: string;
|
||||
source: string;
|
||||
model_provider?: string;
|
||||
}
|
||||
|
||||
export interface ResponseItemPayload {
|
||||
type: 'message' | 'function_call' | 'function_call_output' | 'reasoning';
|
||||
role?: 'user' | 'assistant' | 'developer';
|
||||
content?: ContentBlock[];
|
||||
name?: string;
|
||||
call_id?: string;
|
||||
arguments?: string;
|
||||
output?: string;
|
||||
}
|
||||
|
||||
export interface ContentBlock {
|
||||
type: 'input_text' | 'output_text';
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface EventMsgPayload {
|
||||
type: string;
|
||||
info?: TokenCountInfo;
|
||||
}
|
||||
|
||||
export interface TokenCountInfo {
|
||||
last_token_usage?: TokenUsage;
|
||||
total_token_usage?: TokenUsage;
|
||||
}
|
||||
|
||||
export interface TokenUsage {
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
total_tokens: number;
|
||||
cached_input_tokens?: number;
|
||||
reasoning_output_tokens?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenAI chat-format tool call, used on assistant messages.
|
||||
*/
|
||||
export interface ToolCall {
|
||||
id: string;
|
||||
type: 'function';
|
||||
function: {
|
||||
name: string;
|
||||
arguments: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenAI chat-format message used in LLM span inputs. Matches the message
|
||||
* structure the MLflow UI Chat view renders.
|
||||
*/
|
||||
export interface ChatMessage {
|
||||
role: 'user' | 'assistant' | 'system' | 'tool';
|
||||
content: string | null;
|
||||
tool_calls?: ToolCall[];
|
||||
tool_call_id?: string;
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Minimal arrow-key select prompt for the `mlflow-codex` setup CLI.
|
||||
*
|
||||
* No runtime dependency on `inquirer`/`prompts`: we manage raw-mode stdin
|
||||
* ourselves so the bundled binary stays small, matching the policy in
|
||||
* `ui.ts`. Falls back to the default option when stdin is not a TTY so
|
||||
* piped input (CI, `echo "" | mlflow-codex setup`) never hangs.
|
||||
*/
|
||||
|
||||
import { emitKeypressEvents } from 'node:readline';
|
||||
|
||||
import { bold, cyan, dim } from './ui.js';
|
||||
|
||||
export interface SelectOption<T> {
|
||||
value: T;
|
||||
label: string;
|
||||
hint?: string;
|
||||
}
|
||||
|
||||
export interface SelectPromptOptions<T> {
|
||||
question: string;
|
||||
options: SelectOption<T>[];
|
||||
defaultIndex?: number;
|
||||
input?: NodeJS.ReadStream;
|
||||
output?: NodeJS.WriteStream;
|
||||
}
|
||||
|
||||
export function selectPrompt<T>(opts: SelectPromptOptions<T>): Promise<T> {
|
||||
const input = opts.input ?? process.stdin;
|
||||
const output = opts.output ?? process.stdout;
|
||||
const defaultIndex = opts.defaultIndex ?? 0;
|
||||
const { options, question } = opts;
|
||||
|
||||
if (!input.isTTY) {
|
||||
return Promise.resolve(options[defaultIndex].value);
|
||||
}
|
||||
|
||||
let current = defaultIndex;
|
||||
let linesRendered = 0;
|
||||
|
||||
const formatLines = (): string[] => [
|
||||
`${bold('?')} ${question}`,
|
||||
...options.map((o, i) => {
|
||||
const marker = i === current ? cyan('●') : dim('○');
|
||||
const label = i === current ? cyan(o.label) : o.label;
|
||||
const hint = o.hint ? ` ${dim(o.hint)}` : '';
|
||||
const defaultTag = i === defaultIndex ? dim(' (default)') : '';
|
||||
return ` ${marker} ${label}${hint}${defaultTag}`;
|
||||
}),
|
||||
'',
|
||||
` ${dim('↑/↓ to move, enter to select')}`,
|
||||
];
|
||||
|
||||
const render = (): void => {
|
||||
if (linesRendered > 0) {
|
||||
output.write(`\x1b[${linesRendered}A\x1b[0J`);
|
||||
}
|
||||
const lines = formatLines();
|
||||
output.write(lines.join('\n') + '\n');
|
||||
linesRendered = lines.length;
|
||||
};
|
||||
|
||||
emitKeypressEvents(input);
|
||||
const wasRaw = input.isRaw;
|
||||
input.setRawMode(true);
|
||||
input.resume();
|
||||
|
||||
return new Promise<T>((resolvePromise) => {
|
||||
const cleanup = (): void => {
|
||||
input.removeListener('keypress', onKeypress);
|
||||
if (!wasRaw) {
|
||||
input.setRawMode(false);
|
||||
}
|
||||
input.pause();
|
||||
};
|
||||
|
||||
const onKeypress = (_str: string, key: { name?: string; ctrl?: boolean }): void => {
|
||||
if (key.ctrl && key.name === 'c') {
|
||||
cleanup();
|
||||
process.exit(130);
|
||||
} else if (key.name === 'up' || key.name === 'k') {
|
||||
current = (current - 1 + options.length) % options.length;
|
||||
render();
|
||||
} else if (key.name === 'down' || key.name === 'j') {
|
||||
current = (current + 1) % options.length;
|
||||
render();
|
||||
} else if (key.name === 'return') {
|
||||
cleanup();
|
||||
resolvePromise(options[current].value);
|
||||
}
|
||||
};
|
||||
|
||||
input.on('keypress', onKeypress);
|
||||
render();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Tiny ANSI styling helpers for the `mlflow-codex` CLI.
|
||||
*
|
||||
* No runtime dependency on `chalk`/`kleur`: we keep the bundle small and
|
||||
* respect the usual opt-outs (non-TTY stdout, `NO_COLOR`, `FORCE_COLOR=0`).
|
||||
*/
|
||||
|
||||
const COLOR_ENABLED = (() => {
|
||||
if (process.env.FORCE_COLOR === '0' || process.env.NO_COLOR) {
|
||||
return false;
|
||||
}
|
||||
if (process.env.FORCE_COLOR) {
|
||||
return true;
|
||||
}
|
||||
return Boolean(process.stdout.isTTY);
|
||||
})();
|
||||
|
||||
function wrap(code: string): (s: string) => string {
|
||||
return (s: string) => (COLOR_ENABLED ? `\x1b[${code}m${s}\x1b[0m` : s);
|
||||
}
|
||||
|
||||
export const bold = wrap('1');
|
||||
export const dim = wrap('2');
|
||||
export const green = wrap('32');
|
||||
export const red = wrap('31');
|
||||
export const cyan = wrap('36');
|
||||
export const yellow = wrap('33');
|
||||
|
||||
export const OK = green('✓');
|
||||
export const FAIL = red('✗');
|
||||
export const WARN = yellow('!');
|
||||
@@ -0,0 +1,169 @@
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { parseTraceLocation, resolveTracingConfig } from '../src/config';
|
||||
|
||||
jest.mock('@mlflow/core', () => ({ init: jest.fn() }));
|
||||
|
||||
describe('parseTraceLocation', () => {
|
||||
it('parses a catalog.schema.table_prefix value', () => {
|
||||
expect(parseTraceLocation('cat.sch.pfx')).toEqual({
|
||||
catalogName: 'cat',
|
||||
schemaName: 'sch',
|
||||
tablePrefix: 'pfx',
|
||||
});
|
||||
expect(parseTraceLocation(' cat.sch.pfx ')).toEqual({
|
||||
catalogName: 'cat',
|
||||
schemaName: 'sch',
|
||||
tablePrefix: 'pfx',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns null for empty or malformed values', () => {
|
||||
expect(parseTraceLocation(undefined)).toBeNull();
|
||||
expect(parseTraceLocation('')).toBeNull();
|
||||
expect(parseTraceLocation('cat.sch')).toBeNull();
|
||||
expect(parseTraceLocation('cat.sch.pfx.extra')).toBeNull();
|
||||
expect(parseTraceLocation('cat..pfx')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveTracingConfig', () => {
|
||||
let tmpHome: string;
|
||||
let tmpCwd: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpHome = mkdtempSync(join(tmpdir(), 'codex-config-home-'));
|
||||
tmpCwd = mkdtempSync(join(tmpdir(), 'codex-config-cwd-'));
|
||||
delete process.env.MLFLOW_TRACKING_URI;
|
||||
delete process.env.MLFLOW_EXPERIMENT_ID;
|
||||
delete process.env.MLFLOW_TRACE_LOCATION;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpHome, { recursive: true, force: true });
|
||||
rmSync(tmpCwd, { recursive: true, force: true });
|
||||
delete process.env.MLFLOW_TRACKING_URI;
|
||||
delete process.env.MLFLOW_EXPERIMENT_ID;
|
||||
delete process.env.MLFLOW_TRACE_LOCATION;
|
||||
});
|
||||
|
||||
function writeUserConfig(config: Record<string, string>): void {
|
||||
mkdirSync(join(tmpHome, '.codex'), { recursive: true });
|
||||
writeFileSync(join(tmpHome, '.codex', 'mlflow-tracing.json'), JSON.stringify(config), 'utf-8');
|
||||
}
|
||||
|
||||
it('reads the trace location from mlflow-tracing.json', () => {
|
||||
writeUserConfig({
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '42',
|
||||
traceLocation: 'my_catalog.my_schema.my_prefix',
|
||||
});
|
||||
|
||||
expect(resolveTracingConfig({ home: tmpHome, cwd: tmpCwd })).toEqual({
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '42',
|
||||
traceLocation: 'my_catalog.my_schema.my_prefix',
|
||||
});
|
||||
});
|
||||
|
||||
it('prefers MLFLOW_TRACE_LOCATION env over the config file', () => {
|
||||
writeUserConfig({
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '42',
|
||||
traceLocation: 'file_catalog.file_schema.file_prefix',
|
||||
});
|
||||
process.env.MLFLOW_TRACE_LOCATION = 'env_catalog.env_schema.env_prefix';
|
||||
|
||||
expect(resolveTracingConfig({ home: tmpHome, cwd: tmpCwd })).toMatchObject({
|
||||
traceLocation: 'env_catalog.env_schema.env_prefix',
|
||||
});
|
||||
});
|
||||
|
||||
it('leaves the trace location undefined when not configured', () => {
|
||||
writeUserConfig({ trackingUri: 'http://localhost:5000', experimentId: '0' });
|
||||
|
||||
expect(resolveTracingConfig({ home: tmpHome, cwd: tmpCwd }).traceLocation).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('ensureInitialized', () => {
|
||||
let tmpHome: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpHome = mkdtempSync(join(tmpdir(), 'codex-init-home-'));
|
||||
delete process.env.MLFLOW_TRACKING_URI;
|
||||
delete process.env.MLFLOW_EXPERIMENT_ID;
|
||||
delete process.env.MLFLOW_TRACE_LOCATION;
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpHome, { recursive: true, force: true });
|
||||
delete process.env.MLFLOW_TRACKING_URI;
|
||||
delete process.env.MLFLOW_EXPERIMENT_ID;
|
||||
delete process.env.MLFLOW_TRACE_LOCATION;
|
||||
});
|
||||
|
||||
it('passes a parsed UC trace location to init', () => {
|
||||
process.env.MLFLOW_TRACKING_URI = 'databricks';
|
||||
process.env.MLFLOW_EXPERIMENT_ID = '42';
|
||||
process.env.MLFLOW_TRACE_LOCATION = 'my_catalog.my_schema.my_prefix';
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { init } = require('@mlflow/core') as { init: jest.Mock };
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { ensureInitialized } = require('../src/config') as {
|
||||
ensureInitialized: () => boolean;
|
||||
};
|
||||
|
||||
expect(ensureInitialized()).toBe(true);
|
||||
expect(init).toHaveBeenCalledWith({
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '42',
|
||||
traceLocation: {
|
||||
catalogName: 'my_catalog',
|
||||
schemaName: 'my_schema',
|
||||
tablePrefix: 'my_prefix',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('omits traceLocation when not configured', () => {
|
||||
process.env.MLFLOW_TRACKING_URI = 'http://localhost:5000';
|
||||
process.env.MLFLOW_EXPERIMENT_ID = '0';
|
||||
// Set to empty (not deleted) so a stray ~/.codex/mlflow-tracing.json on the
|
||||
// developer machine can't leak a trace location into this assertion.
|
||||
process.env.MLFLOW_TRACE_LOCATION = '';
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { init } = require('@mlflow/core') as { init: jest.Mock };
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { ensureInitialized } = require('../src/config') as {
|
||||
ensureInitialized: () => boolean;
|
||||
};
|
||||
|
||||
expect(ensureInitialized()).toBe(true);
|
||||
expect(init).toHaveBeenCalledWith({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '0',
|
||||
});
|
||||
});
|
||||
|
||||
it('refuses to initialize when the trace location is malformed', () => {
|
||||
process.env.MLFLOW_TRACKING_URI = 'databricks';
|
||||
process.env.MLFLOW_EXPERIMENT_ID = '42';
|
||||
process.env.MLFLOW_TRACE_LOCATION = 'not-a-valid-location';
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { init } = require('@mlflow/core') as { init: jest.Mock };
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { ensureInitialized } = require('../src/config') as {
|
||||
ensureInitialized: () => boolean;
|
||||
};
|
||||
|
||||
expect(ensureInitialized()).toBe(false);
|
||||
expect(init).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,6 @@
|
||||
{"timestamp":"2026-04-05T10:00:00Z","type":"session_meta","payload":{"id":"test-session-001","timestamp":"2026-04-05T10:00:00Z","cwd":"/tmp/test","originator":"codex-tui","cli_version":"0.118.0","source":"cli"}}
|
||||
{"timestamp":"2026-04-05T10:00:00Z","type":"event_msg","payload":{"type":"task_started"}}
|
||||
{"timestamp":"2026-04-05T10:00:00Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"what is 2+2"}]}}
|
||||
{"timestamp":"2026-04-05T10:00:01Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"4"}]}}
|
||||
{"timestamp":"2026-04-05T10:00:02Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":100,"output_tokens":10,"total_tokens":110}}}}
|
||||
{"timestamp":"2026-04-05T10:00:02Z","type":"event_msg","payload":{"type":"task_complete"}}
|
||||
@@ -0,0 +1,11 @@
|
||||
{"timestamp":"2026-04-20T10:00:00Z","type":"session_meta","payload":{"id":"test-session-fail","timestamp":"2026-04-20T10:00:00Z","cwd":"/tmp/test","originator":"codex-tui","cli_version":"0.118.0","source":"cli"}}
|
||||
{"timestamp":"2026-04-20T10:00:00Z","type":"event_msg","payload":{"type":"task_started"}}
|
||||
{"timestamp":"2026-04-20T10:00:00Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"run nonexistent-cmd"}]}}
|
||||
{"timestamp":"2026-04-20T10:00:01Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"call_fail_1","arguments":"{\"cmd\":\"nonexistent-cmd\"}"}}
|
||||
{"timestamp":"2026-04-20T10:00:02Z","type":"event_msg","payload":{"type":"exec_command_end","call_id":"call_fail_1","exit_code":127,"status":"failed"}}
|
||||
{"timestamp":"2026-04-20T10:00:02Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call_fail_1","output":"zsh: command not found: nonexistent-cmd\nProcess exited with code 127"}}
|
||||
{"timestamp":"2026-04-20T10:00:03Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"call_ok_1","arguments":"{\"cmd\":\"echo ok\"}"}}
|
||||
{"timestamp":"2026-04-20T10:00:04Z","type":"event_msg","payload":{"type":"exec_command_end","call_id":"call_ok_1","exit_code":0,"status":"completed"}}
|
||||
{"timestamp":"2026-04-20T10:00:04Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call_ok_1","output":"ok\nProcess exited with code 0"}}
|
||||
{"timestamp":"2026-04-20T10:00:05Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"The first command failed but the second succeeded."}]}}
|
||||
{"timestamp":"2026-04-20T10:00:06Z","type":"event_msg","payload":{"type":"task_complete"}}
|
||||
@@ -0,0 +1,9 @@
|
||||
{"timestamp":"2026-04-05T10:00:00Z","type":"session_meta","payload":{"id":"test-session-002","timestamp":"2026-04-05T10:00:00Z","cwd":"/tmp/test","originator":"codex-tui","cli_version":"0.118.0","source":"cli"}}
|
||||
{"timestamp":"2026-04-05T10:00:00Z","type":"event_msg","payload":{"type":"task_started"}}
|
||||
{"timestamp":"2026-04-05T10:00:00Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"list files in current directory"}]}}
|
||||
{"timestamp":"2026-04-05T10:00:01Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"I'll list the files for you."}]}}
|
||||
{"timestamp":"2026-04-05T10:00:02Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"call_abc123","arguments":"{\"cmd\":\"ls\"}"}}
|
||||
{"timestamp":"2026-04-05T10:00:03Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call_abc123","output":"file1.txt\nfile2.txt\nfile3.txt"}}
|
||||
{"timestamp":"2026-04-05T10:00:04Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"There are 3 files: file1.txt, file2.txt, file3.txt"}]}}
|
||||
{"timestamp":"2026-04-05T10:00:05Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":200,"output_tokens":50,"total_tokens":250}}}}
|
||||
{"timestamp":"2026-04-05T10:00:05Z","type":"event_msg","payload":{"type":"task_complete"}}
|
||||
@@ -0,0 +1,270 @@
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { createInterface } from 'node:readline';
|
||||
|
||||
import { runSetup } from '../src/commands/setup';
|
||||
import { selectPrompt } from '../src/ui-select';
|
||||
|
||||
jest.mock('node:readline', () => ({ createInterface: jest.fn() }));
|
||||
jest.mock('../src/ui-select', () => ({ selectPrompt: jest.fn() }));
|
||||
const createInterfaceMock = jest.mocked(createInterface);
|
||||
const selectPromptMock = jest.mocked(selectPrompt);
|
||||
|
||||
function mockTextPrompts(answers: string[]): void {
|
||||
const queue = [...answers];
|
||||
createInterfaceMock.mockImplementation(
|
||||
() =>
|
||||
({
|
||||
question: (_prompt: string, cb: (answer: string) => void) => {
|
||||
const next = queue.shift();
|
||||
if (next === undefined) {
|
||||
throw new Error('mockTextPrompts: ran out of answers');
|
||||
}
|
||||
cb(next);
|
||||
},
|
||||
close: () => {},
|
||||
}) as unknown as ReturnType<typeof createInterface>,
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
selectPromptMock.mockReset();
|
||||
createInterfaceMock.mockReset();
|
||||
});
|
||||
|
||||
const HOOK_LINE = 'notify = ["mlflow-codex", "notify-hook"]';
|
||||
const NON_INTERACTIVE = ['--non-interactive'];
|
||||
|
||||
describe('runSetup', () => {
|
||||
let tmpHome: string;
|
||||
let originalExitCode: number | string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpHome = mkdtempSync(join(tmpdir(), 'codex-setup-test-'));
|
||||
originalExitCode = process.exitCode;
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpHome, { recursive: true, force: true });
|
||||
process.exitCode = originalExitCode;
|
||||
});
|
||||
|
||||
function readConfig(path: string): string {
|
||||
return readFileSync(path, 'utf-8');
|
||||
}
|
||||
|
||||
function readJson(path: string): Record<string, unknown> {
|
||||
return JSON.parse(readFileSync(path, 'utf-8')) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
it('creates config.toml and mlflow-tracing.json with defaults when absent', async () => {
|
||||
await runSetup(NON_INTERACTIVE, { home: tmpHome, cwd: tmpHome });
|
||||
|
||||
const configPath = join(tmpHome, '.codex', 'config.toml');
|
||||
const tracingPath = join(tmpHome, '.codex', 'mlflow-tracing.json');
|
||||
expect(existsSync(configPath)).toBe(true);
|
||||
expect(existsSync(tracingPath)).toBe(true);
|
||||
expect(readConfig(configPath)).toContain(HOOK_LINE);
|
||||
expect(readJson(tracingPath)).toEqual({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '0',
|
||||
});
|
||||
});
|
||||
|
||||
it('writes the supplied --tracking-uri and --experiment-id to mlflow-tracing.json', async () => {
|
||||
await runSetup(
|
||||
['--non-interactive', '--tracking-uri', 'http://mlflow.example/', '--experiment-id', '42'],
|
||||
{ home: tmpHome, cwd: tmpHome },
|
||||
);
|
||||
|
||||
expect(readJson(join(tmpHome, '.codex', 'mlflow-tracing.json'))).toEqual({
|
||||
trackingUri: 'http://mlflow.example/',
|
||||
experimentId: '42',
|
||||
});
|
||||
});
|
||||
|
||||
it('prepends the notify hook to an existing config that has no notify entry', async () => {
|
||||
const configPath = join(tmpHome, '.codex', 'config.toml');
|
||||
mkdirSync(join(tmpHome, '.codex'), { recursive: true });
|
||||
const existing = '[some.section]\nkey = "value"\n';
|
||||
writeFileSync(configPath, existing, 'utf-8');
|
||||
|
||||
await runSetup(NON_INTERACTIVE, { home: tmpHome, cwd: tmpHome });
|
||||
|
||||
const content = readConfig(configPath);
|
||||
expect(content).toContain(HOOK_LINE);
|
||||
expect(content).toContain('[some.section]');
|
||||
expect(content).toContain('key = "value"');
|
||||
// notify line must come before the section header so TOML parses correctly
|
||||
expect(content.indexOf('notify = ')).toBeLessThan(content.indexOf('[some.section]'));
|
||||
});
|
||||
|
||||
it('is idempotent when the hook is already registered', async () => {
|
||||
await runSetup(NON_INTERACTIVE, { home: tmpHome, cwd: tmpHome });
|
||||
const configPath = join(tmpHome, '.codex', 'config.toml');
|
||||
const first = readConfig(configPath);
|
||||
|
||||
await runSetup(NON_INTERACTIVE, { home: tmpHome, cwd: tmpHome });
|
||||
const second = readConfig(configPath);
|
||||
|
||||
expect(second).toBe(first);
|
||||
expect(process.exitCode).toBeFalsy();
|
||||
});
|
||||
|
||||
it('refuses to overwrite a different notify entry and signals exit 1', async () => {
|
||||
const configPath = join(tmpHome, '.codex', 'config.toml');
|
||||
mkdirSync(join(tmpHome, '.codex'), { recursive: true });
|
||||
writeFileSync(configPath, 'notify = ["some-other-tool"]\n', 'utf-8');
|
||||
|
||||
await runSetup(NON_INTERACTIVE, { home: tmpHome, cwd: tmpHome });
|
||||
|
||||
const content = readConfig(configPath);
|
||||
expect(content).toBe('notify = ["some-other-tool"]\n');
|
||||
expect(process.exitCode).toBe(1);
|
||||
// mlflow-tracing.json must not be written when we refuse to touch config.toml.
|
||||
expect(existsSync(join(tmpHome, '.codex', 'mlflow-tracing.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('does not treat a comment mentioning "mlflow-codex" as an existing registration', async () => {
|
||||
const configPath = join(tmpHome, '.codex', 'config.toml');
|
||||
mkdirSync(join(tmpHome, '.codex'), { recursive: true });
|
||||
writeFileSync(
|
||||
configPath,
|
||||
'# TODO: switch to mlflow-codex\nnotify = ["some-other-tool"]\n',
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
await runSetup(NON_INTERACTIVE, { home: tmpHome, cwd: tmpHome });
|
||||
|
||||
expect(process.exitCode).toBe(1);
|
||||
const content = readConfig(configPath);
|
||||
expect(content).toContain('notify = ["some-other-tool"]');
|
||||
expect(content).not.toContain(HOOK_LINE);
|
||||
});
|
||||
|
||||
it('rejects a tracking URI that is missing an http(s) scheme and exits 1', async () => {
|
||||
await runSetup(
|
||||
['--non-interactive', '--tracking-uri', 'localhost:5678', '--experiment-id', '48'],
|
||||
{ home: tmpHome, cwd: tmpHome },
|
||||
);
|
||||
|
||||
expect(process.exitCode).toBe(1);
|
||||
// The notify hook is still registered (that step happens before URI
|
||||
// validation and is idempotent), but the tracing config must not be
|
||||
// written with an invalid URI.
|
||||
expect(existsSync(join(tmpHome, '.codex', 'mlflow-tracing.json'))).toBe(false);
|
||||
});
|
||||
|
||||
it('writes to ./.codex/ when --project is passed', async () => {
|
||||
const cwd = mkdtempSync(join(tmpdir(), 'codex-project-test-'));
|
||||
try {
|
||||
await runSetup([...NON_INTERACTIVE, '--project'], { home: tmpHome, cwd });
|
||||
expect(existsSync(join(cwd, '.codex', 'config.toml'))).toBe(true);
|
||||
expect(existsSync(join(cwd, '.codex', 'mlflow-tracing.json'))).toBe(true);
|
||||
expect(existsSync(join(tmpHome, '.codex', 'config.toml'))).toBe(false);
|
||||
expect(existsSync(join(tmpHome, '.codex', 'mlflow-tracing.json'))).toBe(false);
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('interactive scope prompt picks project-local when the user selects Project', async () => {
|
||||
const cwd = mkdtempSync(join(tmpdir(), 'codex-interactive-project-'));
|
||||
try {
|
||||
selectPromptMock.mockResolvedValueOnce(true);
|
||||
mockTextPrompts(['', '']);
|
||||
await runSetup([], { home: tmpHome, cwd });
|
||||
|
||||
expect(selectPromptMock).toHaveBeenCalledTimes(1);
|
||||
expect(existsSync(join(cwd, '.codex', 'config.toml'))).toBe(true);
|
||||
expect(existsSync(join(cwd, '.codex', 'mlflow-tracing.json'))).toBe(true);
|
||||
expect(existsSync(join(tmpHome, '.codex', 'config.toml'))).toBe(false);
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('interactive scope prompt writes user-level when the user selects User', async () => {
|
||||
const cwd = mkdtempSync(join(tmpdir(), 'codex-interactive-user-'));
|
||||
try {
|
||||
selectPromptMock.mockResolvedValueOnce(false);
|
||||
mockTextPrompts(['', '']);
|
||||
await runSetup([], { home: tmpHome, cwd });
|
||||
|
||||
expect(existsSync(join(tmpHome, '.codex', 'config.toml'))).toBe(true);
|
||||
expect(existsSync(join(tmpHome, '.codex', 'mlflow-tracing.json'))).toBe(true);
|
||||
expect(existsSync(join(cwd, '.codex', 'config.toml'))).toBe(false);
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('skips the scope prompt when --project is explicitly passed', async () => {
|
||||
const cwd = mkdtempSync(join(tmpdir(), 'codex-interactive-flag-'));
|
||||
try {
|
||||
mockTextPrompts(['', '']);
|
||||
await runSetup(['--project'], { home: tmpHome, cwd });
|
||||
|
||||
expect(selectPromptMock).not.toHaveBeenCalled();
|
||||
expect(existsSync(join(cwd, '.codex', 'config.toml'))).toBe(true);
|
||||
expect(existsSync(join(tmpHome, '.codex', 'config.toml'))).toBe(false);
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('skips the scope prompt in --non-interactive mode and defaults to project-local', async () => {
|
||||
const cwd = mkdtempSync(join(tmpdir(), 'codex-non-interactive-scope-'));
|
||||
try {
|
||||
await runSetup(NON_INTERACTIVE, { home: tmpHome, cwd });
|
||||
|
||||
expect(selectPromptMock).not.toHaveBeenCalled();
|
||||
expect(existsSync(join(cwd, '.codex', 'config.toml'))).toBe(true);
|
||||
expect(existsSync(join(tmpHome, '.codex', 'config.toml'))).toBe(false);
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('accepts a databricks tracking URI and persists the trace location', async () => {
|
||||
await runSetup(
|
||||
[
|
||||
'--non-interactive',
|
||||
'--tracking-uri',
|
||||
'databricks',
|
||||
'--experiment-id',
|
||||
'42',
|
||||
'--trace-location',
|
||||
'my_catalog.my_schema.my_prefix',
|
||||
],
|
||||
{ home: tmpHome, cwd: tmpHome },
|
||||
);
|
||||
|
||||
expect(process.exitCode).toBeFalsy();
|
||||
expect(readJson(join(tmpHome, '.codex', 'mlflow-tracing.json'))).toEqual({
|
||||
trackingUri: 'databricks',
|
||||
experimentId: '42',
|
||||
traceLocation: 'my_catalog.my_schema.my_prefix',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects an invalid --trace-location and exits 1 without writing config', async () => {
|
||||
await runSetup(
|
||||
[
|
||||
'--non-interactive',
|
||||
'--tracking-uri',
|
||||
'databricks',
|
||||
'--experiment-id',
|
||||
'42',
|
||||
'--trace-location',
|
||||
'not-valid',
|
||||
],
|
||||
{ home: tmpHome, cwd: tmpHome },
|
||||
);
|
||||
|
||||
expect(process.exitCode).toBe(1);
|
||||
expect(existsSync(join(tmpHome, '.codex', 'mlflow-tracing.json'))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,582 @@
|
||||
// Track mock spans
|
||||
let spanCounter = 0;
|
||||
const mockSpans: Record<string, any> = {};
|
||||
const mockTraceInfo: {
|
||||
traceMetadata: Record<string, string>;
|
||||
} = {
|
||||
traceMetadata: {},
|
||||
};
|
||||
|
||||
jest.mock('@mlflow/core', () => {
|
||||
return {
|
||||
init: jest.fn(),
|
||||
startSpan: jest.fn((options: any) => {
|
||||
const id = `span-${++spanCounter}`;
|
||||
const parentId = options.parent ? options.parent.spanId : null;
|
||||
const span = {
|
||||
name: options.name,
|
||||
traceId: 'mock-trace-id',
|
||||
spanId: id,
|
||||
parentId,
|
||||
spanType: options.spanType ?? 'UNKNOWN',
|
||||
inputs: options.inputs ?? {},
|
||||
outputs: {},
|
||||
attributes: { ...(options.attributes ?? {}) },
|
||||
startTimeNs: options.startTimeNs ?? null,
|
||||
endTimeNs: null,
|
||||
statusCode: null as string | null,
|
||||
statusMessage: null as string | null,
|
||||
setAttribute: jest.fn((key: string, value: any) => {
|
||||
span.attributes[key] = value;
|
||||
}),
|
||||
setStatus: jest.fn((code: string, message?: string) => {
|
||||
span.statusCode = code;
|
||||
span.statusMessage = message ?? null;
|
||||
}),
|
||||
end: jest.fn((opts?: any) => {
|
||||
if (opts?.outputs) {
|
||||
span.outputs = opts.outputs;
|
||||
}
|
||||
if (opts?.endTimeNs != null) {
|
||||
span.endTimeNs = opts.endTimeNs;
|
||||
}
|
||||
}),
|
||||
};
|
||||
mockSpans[id] = span;
|
||||
return span;
|
||||
}),
|
||||
flushTraces: jest.fn().mockResolvedValue(undefined),
|
||||
SpanStatusCode: {
|
||||
OK: 'STATUS_CODE_OK',
|
||||
ERROR: 'STATUS_CODE_ERROR',
|
||||
UNSET: 'STATUS_CODE_UNSET',
|
||||
},
|
||||
SpanType: {
|
||||
LLM: 'LLM',
|
||||
AGENT: 'AGENT',
|
||||
TOOL: 'TOOL',
|
||||
UNKNOWN: 'UNKNOWN',
|
||||
},
|
||||
SpanAttributeKey: {
|
||||
TOKEN_USAGE: 'mlflow.chat.tokenUsage',
|
||||
MESSAGE_FORMAT: 'mlflow.message.format',
|
||||
},
|
||||
TraceMetadataKey: {
|
||||
TRACE_SESSION: 'mlflow.trace.session',
|
||||
TRACE_USER: 'mlflow.trace.user',
|
||||
},
|
||||
TokenUsageKey: {
|
||||
INPUT_TOKENS: 'input_tokens',
|
||||
OUTPUT_TOKENS: 'output_tokens',
|
||||
TOTAL_TOKENS: 'total_tokens',
|
||||
},
|
||||
InMemoryTraceManager: {
|
||||
getInstance: jest.fn(() => ({
|
||||
getTrace: jest.fn(() => ({
|
||||
info: mockTraceInfo,
|
||||
})),
|
||||
})),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { resolve } from 'path';
|
||||
|
||||
import {
|
||||
buildToolStatuses,
|
||||
createChildSpans,
|
||||
findTaskCompleteNs,
|
||||
findTaskStartedNs,
|
||||
processNotify,
|
||||
reconstructMessages,
|
||||
} from '../src/tracing';
|
||||
import { readTranscript, getLastTurnRecords } from '../src/transcript';
|
||||
import { flushTraces } from '@mlflow/core';
|
||||
import type { NotifyPayload, RolloutLine } from '../src/types';
|
||||
|
||||
const FIXTURES_DIR = resolve(__dirname, 'fixtures');
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
function getSpans(): any[] {
|
||||
return Object.values(mockSpans);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
function getSpansByType(type: string): any[] {
|
||||
return getSpans().filter((s) => s.spanType === type);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
function getRootSpan(): any {
|
||||
return getSpans().find((s) => s.parentId == null);
|
||||
}
|
||||
|
||||
function makeNotifyPayload(overrides?: Partial<NotifyPayload>): NotifyPayload {
|
||||
return {
|
||||
type: 'agent-turn-complete',
|
||||
'thread-id': 'test-thread-001',
|
||||
'turn-id': 'test-turn-001',
|
||||
cwd: '/tmp/test',
|
||||
client: 'codex-tui',
|
||||
'input-messages': ['what is 2+2'],
|
||||
'last-assistant-message': '4',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('processNotify', () => {
|
||||
beforeEach(() => {
|
||||
spanCounter = 0;
|
||||
Object.keys(mockSpans).forEach((key) => delete mockSpans[key]);
|
||||
mockTraceInfo.traceMetadata = {};
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('creates an AGENT root span with LLM child', async () => {
|
||||
await processNotify(makeNotifyPayload());
|
||||
|
||||
const root = getRootSpan();
|
||||
expect(root).toBeDefined();
|
||||
expect(root.name).toBe('codex_conversation');
|
||||
expect(root.spanType).toBe('AGENT');
|
||||
|
||||
const llmSpans = getSpansByType('LLM');
|
||||
expect(llmSpans.length).toBe(1);
|
||||
expect(llmSpans[0].parentId).toBe(root.spanId);
|
||||
});
|
||||
|
||||
it('passes the user prompt as a raw string on the root span', async () => {
|
||||
await processNotify(makeNotifyPayload());
|
||||
|
||||
const root = getRootSpan();
|
||||
expect(root.inputs).toBe('what is 2+2');
|
||||
});
|
||||
|
||||
it('sets session metadata', async () => {
|
||||
await processNotify(makeNotifyPayload());
|
||||
|
||||
expect(mockTraceInfo.traceMetadata['mlflow.trace.session']).toBe('test-thread-001');
|
||||
expect(mockTraceInfo.traceMetadata['mlflow.trace.user']).toBeDefined();
|
||||
});
|
||||
|
||||
it('uses last input message only', async () => {
|
||||
await processNotify(
|
||||
makeNotifyPayload({
|
||||
'input-messages': ['first prompt', 'second prompt', 'third prompt'],
|
||||
'last-assistant-message': 'response to third',
|
||||
}),
|
||||
);
|
||||
|
||||
const root = getRootSpan();
|
||||
expect(root.inputs).toBe('third prompt');
|
||||
const endCall = (root.end as jest.Mock).mock.calls[0][0];
|
||||
expect(endCall.outputs).toBe('response to third');
|
||||
});
|
||||
|
||||
it('skips when no user prompt', async () => {
|
||||
await processNotify(makeNotifyPayload({ 'input-messages': [] }));
|
||||
|
||||
expect(getSpans().length).toBe(0);
|
||||
expect(flushTraces).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls flushTraces after processing', async () => {
|
||||
await processNotify(makeNotifyPayload());
|
||||
expect(flushTraces).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('sets the assistant response as the raw root span output', async () => {
|
||||
await processNotify(makeNotifyPayload());
|
||||
|
||||
const root = getRootSpan();
|
||||
expect(root.end).toHaveBeenCalled();
|
||||
const endCall = (root.end as jest.Mock).mock.calls[0][0];
|
||||
expect(endCall.outputs).toBe('4');
|
||||
});
|
||||
|
||||
it('uses OpenAI chat format for the fallback LLM span', async () => {
|
||||
await processNotify(makeNotifyPayload());
|
||||
|
||||
const [llm] = getSpansByType('LLM');
|
||||
expect(llm.name).toBe('llm_call');
|
||||
expect(llm.inputs).toEqual({
|
||||
model: 'unknown',
|
||||
messages: [{ role: 'user', content: 'what is 2+2' }],
|
||||
});
|
||||
|
||||
const endCall = (llm.end as jest.Mock).mock.calls[0][0];
|
||||
expect(endCall.outputs).toEqual({
|
||||
choices: [{ message: { role: 'assistant', content: '4' } }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('reconstructMessages', () => {
|
||||
function responseItem(payload: Record<string, unknown>): RolloutLine {
|
||||
return {
|
||||
timestamp: '2026-04-05T10:00:00Z',
|
||||
type: 'response_item',
|
||||
payload: payload as RolloutLine['payload'],
|
||||
};
|
||||
}
|
||||
|
||||
it('maps user and assistant messages to chat format', () => {
|
||||
const items = [
|
||||
responseItem({
|
||||
type: 'message',
|
||||
role: 'user',
|
||||
content: [{ type: 'input_text', text: 'hi' }],
|
||||
}),
|
||||
responseItem({
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
content: [{ type: 'output_text', text: 'hello' }],
|
||||
}),
|
||||
];
|
||||
expect(reconstructMessages(items, items.length)).toEqual([
|
||||
{ role: 'user', content: 'hi' },
|
||||
{ role: 'assistant', content: 'hello' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('maps developer messages to system role', () => {
|
||||
const items = [
|
||||
responseItem({
|
||||
type: 'message',
|
||||
role: 'developer',
|
||||
content: [{ type: 'input_text', text: 'you are a helpful assistant' }],
|
||||
}),
|
||||
];
|
||||
expect(reconstructMessages(items, items.length)).toEqual([
|
||||
{ role: 'system', content: 'you are a helpful assistant' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('maps function_call to assistant message with tool_calls', () => {
|
||||
const items = [
|
||||
responseItem({
|
||||
type: 'function_call',
|
||||
name: 'exec_command',
|
||||
call_id: 'call_1',
|
||||
arguments: '{"cmd":"ls"}',
|
||||
}),
|
||||
];
|
||||
expect(reconstructMessages(items, items.length)).toEqual([
|
||||
{
|
||||
role: 'assistant',
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: 'call_1',
|
||||
type: 'function',
|
||||
function: { name: 'exec_command', arguments: '{"cmd":"ls"}' },
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('maps function_call_output to tool message', () => {
|
||||
const items = [
|
||||
responseItem({
|
||||
type: 'function_call_output',
|
||||
call_id: 'call_1',
|
||||
output: 'file1.txt\nfile2.txt',
|
||||
}),
|
||||
];
|
||||
expect(reconstructMessages(items, items.length)).toEqual([
|
||||
{ role: 'tool', tool_call_id: 'call_1', content: 'file1.txt\nfile2.txt' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('stops at uptoIndex and preserves order across a tool-use turn', () => {
|
||||
const items = [
|
||||
responseItem({
|
||||
type: 'message',
|
||||
role: 'user',
|
||||
content: [{ type: 'input_text', text: 'list files' }],
|
||||
}),
|
||||
responseItem({
|
||||
type: 'function_call',
|
||||
name: 'exec',
|
||||
call_id: 'c1',
|
||||
arguments: '{"cmd":"ls"}',
|
||||
}),
|
||||
responseItem({
|
||||
type: 'function_call_output',
|
||||
call_id: 'c1',
|
||||
output: 'a.txt',
|
||||
}),
|
||||
responseItem({
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
content: [{ type: 'output_text', text: 'there is one file' }],
|
||||
}),
|
||||
];
|
||||
// uptoIndex=3 stops before the final assistant message
|
||||
expect(reconstructMessages(items, 3)).toEqual([
|
||||
{ role: 'user', content: 'list files' },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: 'c1',
|
||||
type: 'function',
|
||||
function: { name: 'exec', arguments: '{"cmd":"ls"}' },
|
||||
},
|
||||
],
|
||||
},
|
||||
{ role: 'tool', tool_call_id: 'c1', content: 'a.txt' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('skips empty-text messages', () => {
|
||||
const items = [
|
||||
responseItem({
|
||||
type: 'message',
|
||||
role: 'user',
|
||||
content: [{ type: 'input_text', text: ' ' }],
|
||||
}),
|
||||
];
|
||||
expect(reconstructMessages(items, items.length)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createChildSpans (integration with real transcript fixture)', () => {
|
||||
// Reset the mock span tracking between tests to avoid pollution from
|
||||
// processNotify tests in the same file.
|
||||
beforeEach(() => {
|
||||
spanCounter = 0;
|
||||
Object.keys(mockSpans).forEach((key) => delete mockSpans[key]);
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('creates the expected span tree from the tool-call fixture', () => {
|
||||
// Fixture conversation:
|
||||
// user "list files in current directory"
|
||||
// assistant "I'll list the files for you."
|
||||
// function_call exec_command({"cmd":"ls"}) -> call_abc123
|
||||
// function_call_output "file1.txt\nfile2.txt\nfile3.txt"
|
||||
// assistant "There are 3 files: ..."
|
||||
const records = readTranscript(resolve(FIXTURES_DIR, 'with-tool-call.jsonl'));
|
||||
const turn = getLastTurnRecords(records);
|
||||
|
||||
// Build a fake parent span that startSpan() can parent children to
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const parent = { spanId: 'root' } as any;
|
||||
|
||||
createChildSpans(parent, turn, 'gpt-4');
|
||||
|
||||
const llmSpans = getSpansByType('LLM');
|
||||
const toolSpans = getSpansByType('TOOL');
|
||||
|
||||
// 2 assistant messages -> 2 LLM spans; 1 function_call -> 1 TOOL span
|
||||
expect(llmSpans.length).toBe(2);
|
||||
expect(toolSpans.length).toBe(1);
|
||||
|
||||
// All children parented to the fake root
|
||||
for (const span of [...llmSpans, ...toolSpans]) {
|
||||
expect(span.parentId).toBe('root');
|
||||
}
|
||||
|
||||
// First LLM span: only the user message is in scope, no tool_calls yet
|
||||
expect(llmSpans[0].name).toBe('llm_call');
|
||||
expect(llmSpans[0].inputs).toEqual({
|
||||
model: 'gpt-4',
|
||||
messages: [{ role: 'user', content: 'list files in current directory' }],
|
||||
});
|
||||
const firstLlmEnd = (llmSpans[0].end as jest.Mock).mock.calls[0][0];
|
||||
expect(firstLlmEnd.outputs).toEqual({
|
||||
choices: [{ message: { role: 'assistant', content: "I'll list the files for you." } }],
|
||||
});
|
||||
|
||||
// TOOL span
|
||||
expect(toolSpans[0].name).toBe('tool_exec_command');
|
||||
expect(toolSpans[0].inputs).toEqual({ cmd: 'ls' });
|
||||
expect(toolSpans[0].attributes).toEqual({
|
||||
tool_name: 'exec_command',
|
||||
tool_id: 'call_abc123',
|
||||
});
|
||||
const toolEnd = (toolSpans[0].end as jest.Mock).mock.calls[0][0];
|
||||
expect(toolEnd.outputs).toEqual({
|
||||
result: 'file1.txt\nfile2.txt\nfile3.txt',
|
||||
});
|
||||
|
||||
// Second LLM span: full conversation history including the tool-call and
|
||||
// tool-result should be reconstructed in inputs.messages
|
||||
expect(llmSpans[1].inputs).toEqual({
|
||||
model: 'gpt-4',
|
||||
messages: [
|
||||
{ role: 'user', content: 'list files in current directory' },
|
||||
{ role: 'assistant', content: "I'll list the files for you." },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: 'call_abc123',
|
||||
type: 'function',
|
||||
function: { name: 'exec_command', arguments: '{"cmd":"ls"}' },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: 'tool',
|
||||
tool_call_id: 'call_abc123',
|
||||
content: 'file1.txt\nfile2.txt\nfile3.txt',
|
||||
},
|
||||
],
|
||||
});
|
||||
const secondLlmEnd = (llmSpans[1].end as jest.Mock).mock.calls[0][0];
|
||||
expect(secondLlmEnd.outputs).toEqual({
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: 'There are 3 files: file1.txt, file2.txt, file3.txt',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('derives span timestamps from turn boundaries, not next-record chaining', () => {
|
||||
// Fixture timestamps are spaced 1 second apart:
|
||||
// 10:00:00Z task_started
|
||||
// 10:00:00Z user "list files in current directory"
|
||||
// 10:00:01Z assistant "I'll list the files for you."
|
||||
// 10:00:02Z function_call exec_command
|
||||
// 10:00:03Z function_call_output
|
||||
// 10:00:04Z assistant "There are 3 files: ..."
|
||||
// 10:00:05Z task_complete
|
||||
//
|
||||
// Expected:
|
||||
// LLM #1: task_started (00) -> assistant #1 (01) = 1s
|
||||
// TOOL: function_call (02) -> function_call_output (03) = 1s
|
||||
// LLM #2: function_call_output (03) -> assistant #2 (04) = 1s
|
||||
const records = readTranscript(resolve(FIXTURES_DIR, 'with-tool-call.jsonl'));
|
||||
const turn = getLastTurnRecords(records);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const parent = { spanId: 'root' } as any;
|
||||
createChildSpans(parent, turn, 'gpt-4');
|
||||
|
||||
const NS_PER_SEC = 1_000_000_000;
|
||||
const parseNs = (iso: string) => new Date(iso).getTime() * 1_000_000;
|
||||
|
||||
const llmSpans = getSpansByType('LLM');
|
||||
const toolSpans = getSpansByType('TOOL');
|
||||
|
||||
// LLM #1: task_started -> first assistant message
|
||||
expect(llmSpans[0].startTimeNs).toBe(parseNs('2026-04-05T10:00:00Z'));
|
||||
expect(llmSpans[0].endTimeNs).toBe(parseNs('2026-04-05T10:00:01Z'));
|
||||
expect(llmSpans[0].endTimeNs - llmSpans[0].startTimeNs).toBe(NS_PER_SEC);
|
||||
|
||||
// TOOL: function_call -> matching function_call_output (by call_id)
|
||||
expect(toolSpans[0].startTimeNs).toBe(parseNs('2026-04-05T10:00:02Z'));
|
||||
expect(toolSpans[0].endTimeNs).toBe(parseNs('2026-04-05T10:00:03Z'));
|
||||
expect(toolSpans[0].endTimeNs - toolSpans[0].startTimeNs).toBe(NS_PER_SEC);
|
||||
|
||||
// LLM #2: previous function_call_output -> second assistant message
|
||||
// (NOT from the previous assistant message — the LLM was waiting on the
|
||||
// tool during that time)
|
||||
expect(llmSpans[1].startTimeNs).toBe(parseNs('2026-04-05T10:00:03Z'));
|
||||
expect(llmSpans[1].endTimeNs).toBe(parseNs('2026-04-05T10:00:04Z'));
|
||||
expect(llmSpans[1].endTimeNs - llmSpans[1].startTimeNs).toBe(NS_PER_SEC);
|
||||
|
||||
// Sanity: spans don't overlap
|
||||
expect(llmSpans[0].endTimeNs).toBeLessThanOrEqual(toolSpans[0].startTimeNs);
|
||||
expect(toolSpans[0].endTimeNs).toBeLessThanOrEqual(llmSpans[1].startTimeNs);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findTaskStartedNs / findTaskCompleteNs', () => {
|
||||
it('extract the turn boundary timestamps from the fixture', () => {
|
||||
const records = readTranscript(resolve(FIXTURES_DIR, 'with-tool-call.jsonl'));
|
||||
const turn = getLastTurnRecords(records);
|
||||
|
||||
const parseNs = (iso: string) => new Date(iso).getTime() * 1_000_000;
|
||||
expect(findTaskStartedNs(turn)).toBe(parseNs('2026-04-05T10:00:00Z'));
|
||||
expect(findTaskCompleteNs(turn)).toBe(parseNs('2026-04-05T10:00:05Z'));
|
||||
});
|
||||
|
||||
it('returns null when the turn lacks a task_started or task_complete event', () => {
|
||||
// A turn that only contains a response_item (no event_msg markers)
|
||||
const records: RolloutLine[] = [
|
||||
{
|
||||
timestamp: '2026-04-05T10:00:00Z',
|
||||
type: 'response_item',
|
||||
payload: { type: 'message', role: 'user', content: [] },
|
||||
},
|
||||
];
|
||||
expect(findTaskStartedNs(records)).toBeNull();
|
||||
expect(findTaskCompleteNs(records)).toBeNull();
|
||||
});
|
||||
|
||||
it('returned values bracket all child spans from createChildSpans', () => {
|
||||
// Invariant: if processNotify passes these timestamps as the root span's
|
||||
// startTimeNs/endTimeNs, the root span will correctly enclose every
|
||||
// child created by createChildSpans.
|
||||
const records = readTranscript(resolve(FIXTURES_DIR, 'with-tool-call.jsonl'));
|
||||
const turn = getLastTurnRecords(records);
|
||||
const startNs = findTaskStartedNs(turn)!;
|
||||
const completeNs = findTaskCompleteNs(turn)!;
|
||||
|
||||
// Reset the span tracker populated by earlier describe blocks
|
||||
spanCounter = 0;
|
||||
Object.keys(mockSpans).forEach((key) => delete mockSpans[key]);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const parent = { spanId: 'root' } as any;
|
||||
createChildSpans(parent, turn, 'gpt-4');
|
||||
|
||||
for (const span of Object.values(mockSpans)) {
|
||||
expect(span.startTimeNs).toBeGreaterThanOrEqual(startNs);
|
||||
const endNs = (span.end as jest.Mock).mock.calls[0]?.[0]?.endTimeNs;
|
||||
expect(endNs).toBeLessThanOrEqual(completeNs);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('tool span failure status', () => {
|
||||
beforeEach(() => {
|
||||
spanCounter = 0;
|
||||
Object.keys(mockSpans).forEach((key) => delete mockSpans[key]);
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('buildToolStatuses flags failed exec_command_end by status and exit_code', () => {
|
||||
const records = readTranscript(resolve(FIXTURES_DIR, 'with-failed-tool.jsonl'));
|
||||
const turn = getLastTurnRecords(records);
|
||||
|
||||
const statuses = buildToolStatuses(turn);
|
||||
expect(statuses).toEqual({
|
||||
call_fail_1: { failed: true, exitCode: 127 },
|
||||
call_ok_1: { failed: false, exitCode: 0 },
|
||||
});
|
||||
});
|
||||
|
||||
it('sets ERROR status on TOOL spans whose exec_command_end reports failure', () => {
|
||||
const records = readTranscript(resolve(FIXTURES_DIR, 'with-failed-tool.jsonl'));
|
||||
const turn = getLastTurnRecords(records);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const parent = { spanId: 'root' } as any;
|
||||
createChildSpans(parent, turn, 'gpt-4');
|
||||
|
||||
const toolSpans = getSpansByType('TOOL');
|
||||
expect(toolSpans.length).toBe(2);
|
||||
|
||||
const failed = toolSpans.find((s) => s.attributes.tool_id === 'call_fail_1');
|
||||
const ok = toolSpans.find((s) => s.attributes.tool_id === 'call_ok_1');
|
||||
|
||||
expect(failed.statusCode).toBe('STATUS_CODE_ERROR');
|
||||
expect(failed.statusMessage).toContain('127');
|
||||
expect(failed.setStatus).toHaveBeenCalled();
|
||||
|
||||
// OK tool: setStatus should NOT have been called
|
||||
expect(ok.statusCode).toBeNull();
|
||||
expect(ok.setStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import { resolve } from 'path';
|
||||
|
||||
import {
|
||||
readTranscript,
|
||||
parseTimestampToNs,
|
||||
extractTextFromContent,
|
||||
findLastUserPrompt,
|
||||
getLastTurnRecords,
|
||||
getTokenUsage,
|
||||
getModel,
|
||||
getSessionId,
|
||||
buildToolResultMap,
|
||||
} from '../src/transcript';
|
||||
|
||||
const FIXTURES_DIR = resolve(__dirname, 'fixtures');
|
||||
|
||||
describe('parseTimestampToNs', () => {
|
||||
it('parses ISO timestamp', () => {
|
||||
const result = parseTimestampToNs('2026-04-05T10:00:00Z');
|
||||
expect(result).toBeGreaterThan(0);
|
||||
expect(typeof result).toBe('number');
|
||||
});
|
||||
|
||||
it('returns null for empty input', () => {
|
||||
expect(parseTimestampToNs(null)).toBeNull();
|
||||
expect(parseTimestampToNs(undefined)).toBeNull();
|
||||
expect(parseTimestampToNs('')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractTextFromContent', () => {
|
||||
it('extracts text from content blocks', () => {
|
||||
const content = [
|
||||
{ type: 'output_text' as const, text: 'hello' },
|
||||
{ type: 'output_text' as const, text: 'world' },
|
||||
];
|
||||
expect(extractTextFromContent(content)).toBe('hello\nworld');
|
||||
});
|
||||
|
||||
it('returns string content as-is', () => {
|
||||
expect(extractTextFromContent('plain text')).toBe('plain text');
|
||||
});
|
||||
|
||||
it('returns empty string for undefined', () => {
|
||||
expect(extractTextFromContent(undefined)).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('readTranscript + parsing', () => {
|
||||
const basicRecords = readTranscript(resolve(FIXTURES_DIR, 'basic.jsonl'));
|
||||
const toolRecords = readTranscript(resolve(FIXTURES_DIR, 'with-tool-call.jsonl'));
|
||||
|
||||
it('reads basic transcript', () => {
|
||||
expect(basicRecords.length).toBe(6);
|
||||
});
|
||||
|
||||
it('reads tool call transcript', () => {
|
||||
expect(toolRecords.length).toBe(9);
|
||||
});
|
||||
|
||||
it('finds last user prompt', () => {
|
||||
const result = findLastUserPrompt(basicRecords);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.text).toBe('what is 2+2');
|
||||
});
|
||||
|
||||
it('gets last turn records', () => {
|
||||
const turn = getLastTurnRecords(basicRecords);
|
||||
expect(turn.length).toBeGreaterThan(0);
|
||||
// Should include task_started through task_complete
|
||||
expect(turn[0].type).toBe('event_msg');
|
||||
});
|
||||
|
||||
it('gets token usage', () => {
|
||||
const usage = getTokenUsage(basicRecords);
|
||||
expect(usage).not.toBeNull();
|
||||
expect(usage!.input_tokens).toBe(100);
|
||||
expect(usage!.output_tokens).toBe(10);
|
||||
expect(usage!.total_tokens).toBe(110);
|
||||
});
|
||||
|
||||
it('gets model from session meta', () => {
|
||||
// No model in our fixture, should return unknown
|
||||
expect(getModel(basicRecords)).toBe('unknown');
|
||||
});
|
||||
|
||||
it('gets session ID', () => {
|
||||
expect(getSessionId(basicRecords)).toBe('test-session-001');
|
||||
});
|
||||
|
||||
it('builds tool result map', () => {
|
||||
const results = buildToolResultMap(toolRecords);
|
||||
expect(results['call_abc123']).toBe('file1.txt\nfile2.txt\nfile3.txt');
|
||||
});
|
||||
|
||||
it('returns empty tool result map for basic transcript', () => {
|
||||
const results = buildToolResultMap(basicRecords);
|
||||
expect(Object.keys(results).length).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"rootDir": "src"
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
# MLflow Typescript SDK - Gemini
|
||||
|
||||
Seamlessly integrate [MLflow Tracing](https://github.com/mlflow/mlflow/tree/main/libs/typescript) with Gemini to automatically trace your Claude API calls.
|
||||
|
||||
| Package | NPM | Description |
|
||||
| -------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- |
|
||||
| [@mlflow/gemini](./) | [](https://www.npmjs.com/package/@mlflow/gemini) | Auto-instrumentation integration for Gemini. |
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @mlflow/gemini
|
||||
```
|
||||
|
||||
The package includes the [`@mlflow/core`](https://github.com/mlflow/mlflow/tree/main/libs/typescript) package and `@google/genai` package as peer dependencies. Depending on your package manager, you may need to install these two packages separately.
|
||||
|
||||
## Quickstart
|
||||
|
||||
Start MLflow Tracking Server if you don't have one already:
|
||||
|
||||
```bash
|
||||
pip install mlflow
|
||||
mlflow server --backend-store-uri sqlite:///mlruns.db --port 5000
|
||||
```
|
||||
|
||||
Self-hosting MLflow server requires Python 3.10 or higher. If you don't have one, you can also use [managed MLflow service](https://mlflow.org/#get-started) for free to get started quickly.
|
||||
|
||||
Instantiate MLflow SDK in your application:
|
||||
|
||||
```typescript
|
||||
import * as mlflow from '@mlflow/core';
|
||||
|
||||
mlflow.init({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '<experiment-id>',
|
||||
});
|
||||
```
|
||||
|
||||
Create a trace for Gemini:
|
||||
|
||||
```typescript
|
||||
import { tracedGemini } from '@mlflow/gemini';
|
||||
import { GoogleGenAI } from '@google/genai';
|
||||
|
||||
const gemini = new GoogleGenAI({ apiKey: process.env.GEMINI_API_KEY });
|
||||
const client = tracedGemini(gemini);
|
||||
|
||||
const response = await client.models.generateContent({
|
||||
model: 'gemini-2.0-flash-001',
|
||||
contents: 'Hello Gemini',
|
||||
});
|
||||
```
|
||||
|
||||
View traces in MLflow UI:
|
||||
|
||||

|
||||
|
||||
## Documentation 📘
|
||||
|
||||
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart).
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
|
||||
@@ -0,0 +1,16 @@
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
roots: ['<rootDir>/tests', '<rootDir>/src'],
|
||||
testMatch: ['**/*.test.ts'],
|
||||
moduleFileExtensions: ['ts', 'js', 'json', 'node'],
|
||||
transform: {
|
||||
'^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
|
||||
},
|
||||
globalSetup: '<rootDir>/../../jest.global-server-setup.ts',
|
||||
globalTeardown: '<rootDir>/../../jest.global-server-teardown.ts',
|
||||
testTimeout: 30000,
|
||||
forceExit: true,
|
||||
detectOpenHandles: true,
|
||||
};
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"name": "@mlflow/gemini",
|
||||
"version": "0.3.0",
|
||||
"description": "Gemini integration package for MLflow Tracing",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mlflow/mlflow.git"
|
||||
},
|
||||
"homepage": "https://mlflow.org/",
|
||||
"author": {
|
||||
"name": "MLflow",
|
||||
"url": "https://mlflow.org/"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/mlflow/mlflow/issues"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"keywords": [
|
||||
"mlflow",
|
||||
"tracing",
|
||||
"observability",
|
||||
"opentelemetry",
|
||||
"llm",
|
||||
"gemini",
|
||||
"google",
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"prepublishOnly": "npm run build",
|
||||
"test": "jest",
|
||||
"lint": "eslint . --ext .ts --max-warnings 0",
|
||||
"lint:fix": "eslint . --ext .ts --fix",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@mlflow/core": "^0.3.0",
|
||||
"@google/genai": "^1.22.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"jest": "^29.6.2",
|
||||
"json-bigint": "^1.0.0",
|
||||
"typescript": "^5.8.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"files": [
|
||||
"dist/"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* MLflow Tracing wrapper for the @google/genai Gemini SDK.
|
||||
*/
|
||||
|
||||
import { withSpan, SpanAttributeKey, SpanType, type TokenUsage, type LiveSpan } from '@mlflow/core';
|
||||
|
||||
const SUPPORTED_MODULES = ['models'];
|
||||
const SUPPORTED_METHODS = ['generateContent'];
|
||||
|
||||
/**
|
||||
* Create a traced version of Gemini client with MLflow tracing
|
||||
* @param geminiClient - The Gemini client instance to trace
|
||||
* @returns Traced Gemini client with tracing capabilities
|
||||
*/
|
||||
export function tracedGemini<T = any>(geminiClient: T): T {
|
||||
const tracedClient = new Proxy(geminiClient as any, {
|
||||
get(target, prop, receiver) {
|
||||
const original = Reflect.get(target, prop, receiver);
|
||||
const moduleName = (target as object).constructor?.name;
|
||||
|
||||
if (typeof original === 'function') {
|
||||
if (shouldTraceMethod(moduleName, String(prop))) {
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return wrapWithTracing(original as Function, String(prop));
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return (original as Function).bind(target) as T;
|
||||
}
|
||||
|
||||
if (
|
||||
original &&
|
||||
!Array.isArray(original) &&
|
||||
!(original instanceof Date) &&
|
||||
typeof original === 'object'
|
||||
) {
|
||||
return tracedGemini(original) as T;
|
||||
}
|
||||
|
||||
return original as T;
|
||||
},
|
||||
});
|
||||
|
||||
return tracedClient as T;
|
||||
}
|
||||
|
||||
function shouldTraceMethod(moduleName: string | undefined, methodName: string): boolean {
|
||||
if (!moduleName) {
|
||||
return false;
|
||||
}
|
||||
const lowerModuleName = moduleName.toLowerCase();
|
||||
return SUPPORTED_MODULES.includes(lowerModuleName) && SUPPORTED_METHODS.includes(methodName);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
function wrapWithTracing(fn: Function, methodName: string): Function {
|
||||
const spanType = getSpanType(methodName);
|
||||
|
||||
return function (this: any, ...args: any[]) {
|
||||
if (!spanType) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return fn.apply(this, args);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return withSpan(
|
||||
async (span: LiveSpan) => {
|
||||
span.setInputs(args[0]);
|
||||
|
||||
const result = await fn.apply(this, args);
|
||||
|
||||
span.setOutputs(result);
|
||||
|
||||
try {
|
||||
const usage = extractTokenUsage(result);
|
||||
if (usage) {
|
||||
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
|
||||
}
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('Error extracting token usage', error);
|
||||
}
|
||||
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'gemini');
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return result;
|
||||
},
|
||||
{ name: methodName, spanType },
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
function getSpanType(methodName: string): SpanType | undefined {
|
||||
switch (methodName) {
|
||||
case 'generateContent':
|
||||
return SpanType.LLM;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
function extractTokenUsage(response: any): TokenUsage | undefined {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const usage = response?.usageMetadata ?? response?.usage;
|
||||
if (!usage) {
|
||||
return undefined;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const input = usage.promptTokenCount;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const output = usage.candidatesTokenCount;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const total = usage.totalTokenCount;
|
||||
|
||||
if (input !== undefined && output !== undefined && total !== undefined) {
|
||||
return {
|
||||
input_tokens: input,
|
||||
output_tokens: output,
|
||||
total_tokens: total,
|
||||
};
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
/**
|
||||
* Tests for MLflow Gemini integration with MSW mock server
|
||||
*/
|
||||
|
||||
import * as mlflow from '@mlflow/core';
|
||||
import { tracedGemini } from '../src';
|
||||
import { GoogleGenAI } from '@google/genai';
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { geminiMockHandlers } from './mockGeminiServer';
|
||||
import { createAuthProvider } from '@mlflow/core/src/auth';
|
||||
|
||||
const TEST_TRACKING_URI = 'http://localhost:5000';
|
||||
|
||||
describe('tracedGemini', () => {
|
||||
let experimentId: string;
|
||||
let client: mlflow.MlflowClient;
|
||||
let server: ReturnType<typeof setupServer>;
|
||||
|
||||
beforeAll(async () => {
|
||||
const authProvider = createAuthProvider({ trackingUri: TEST_TRACKING_URI });
|
||||
client = new mlflow.MlflowClient({ trackingUri: TEST_TRACKING_URI, authProvider });
|
||||
|
||||
const experimentName = `test-experiment-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
|
||||
experimentId = await client.createExperiment(experimentName);
|
||||
mlflow.init({
|
||||
trackingUri: TEST_TRACKING_URI,
|
||||
experimentId: experimentId,
|
||||
});
|
||||
|
||||
server = setupServer(...geminiMockHandlers);
|
||||
server.listen();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
server.close();
|
||||
await client.deleteExperiment(experimentId);
|
||||
});
|
||||
|
||||
const getLastActiveTrace = async (): Promise<mlflow.Trace> => {
|
||||
await mlflow.flushTraces();
|
||||
const traceId = mlflow.getLastActiveTraceId();
|
||||
const trace = await client.getTrace(traceId!);
|
||||
return trace;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
server.resetHandlers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('Generate Content', () => {
|
||||
it('should trace models.generateContent() without parent span', async () => {
|
||||
const gemini = new GoogleGenAI({ apiKey: 'test-key' });
|
||||
const wrappedGemini = tracedGemini(gemini);
|
||||
|
||||
const result = await wrappedGemini.models.generateContent({
|
||||
model: 'gemini-2.0-flash-001',
|
||||
contents: 'Hello Gemini',
|
||||
});
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result.usageMetadata).toBeDefined();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
expect(trace.data.spans.length).toBe(1);
|
||||
|
||||
const llmSpan = trace.data.spans[0];
|
||||
expect(llmSpan).toBeDefined();
|
||||
expect(llmSpan.name).toBe('generateContent');
|
||||
expect(llmSpan.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(llmSpan.logLevel).toBe(mlflow.SpanLogLevel.INFO);
|
||||
expect(llmSpan.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(llmSpan.inputs).toEqual({
|
||||
model: 'gemini-2.0-flash-001',
|
||||
contents: 'Hello Gemini',
|
||||
});
|
||||
expect(llmSpan.outputs).toEqual(result);
|
||||
expect(llmSpan.startTime).toBeDefined();
|
||||
expect(llmSpan.endTime).toBeDefined();
|
||||
|
||||
const spanTokenUsage = llmSpan.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
|
||||
expect(spanTokenUsage).toBeDefined();
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.TOTAL_TOKENS]).toBe('number');
|
||||
expect(spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe(10);
|
||||
expect(spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe(5);
|
||||
expect(spanTokenUsage[mlflow.TokenUsageKey.TOTAL_TOKENS]).toBe(15);
|
||||
|
||||
const messageFormat = llmSpan.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('gemini');
|
||||
});
|
||||
|
||||
it('should trace models.generateContent()', async () => {
|
||||
const gemini = new GoogleGenAI({ apiKey: 'test-key' });
|
||||
const wrappedGemini = tracedGemini(gemini);
|
||||
|
||||
const result = await mlflow.withSpan(
|
||||
async () => {
|
||||
return await wrappedGemini.models.generateContent({
|
||||
model: 'gemini-2.0-flash-001',
|
||||
contents: 'Hello Gemini',
|
||||
});
|
||||
},
|
||||
{
|
||||
name: 'test-trace',
|
||||
spanType: mlflow.SpanType.CHAIN,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result.usageMetadata).toBeDefined();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
expect(trace.data.spans.length).toBe(2);
|
||||
|
||||
const llmSpan = trace.data.spans.find((s) => s.spanType === mlflow.SpanType.LLM)!;
|
||||
expect(llmSpan).toBeDefined();
|
||||
expect(llmSpan.name).toBe('generateContent');
|
||||
expect(llmSpan.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(llmSpan.inputs).toEqual({
|
||||
model: 'gemini-2.0-flash-001',
|
||||
contents: 'Hello Gemini',
|
||||
});
|
||||
expect(llmSpan.outputs).toEqual(result);
|
||||
expect(llmSpan.startTime).toBeDefined();
|
||||
expect(llmSpan.endTime).toBeDefined();
|
||||
|
||||
const spanTokenUsage = llmSpan.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
|
||||
expect(spanTokenUsage).toBeDefined();
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.TOTAL_TOKENS]).toBe('number');
|
||||
expect(spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe(10);
|
||||
expect(spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe(5);
|
||||
expect(spanTokenUsage[mlflow.TokenUsageKey.TOTAL_TOKENS]).toBe(15);
|
||||
|
||||
const messageFormat = llmSpan.attributes[mlflow.SpanAttributeKey.MESSAGE_FORMAT];
|
||||
expect(messageFormat).toBe('gemini');
|
||||
});
|
||||
|
||||
it('should handle generateContent errors properly', async () => {
|
||||
server.use(
|
||||
http.post(
|
||||
'https://generativelanguage.googleapis.com/v1beta/models/*\\:generateContent',
|
||||
() => {
|
||||
return HttpResponse.json(
|
||||
{
|
||||
error: {
|
||||
code: 429,
|
||||
message: 'Resource has been exhausted',
|
||||
status: 'RESOURCE_EXHAUSTED',
|
||||
},
|
||||
},
|
||||
{ status: 429 },
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const gemini = new GoogleGenAI({ apiKey: 'test-key' });
|
||||
const wrappedGemini = tracedGemini(gemini);
|
||||
|
||||
await expect(
|
||||
mlflow.withSpan(
|
||||
async () => {
|
||||
return await wrappedGemini.models.generateContent({
|
||||
model: 'gemini-2.0-flash-001',
|
||||
contents: 'This should fail',
|
||||
});
|
||||
},
|
||||
{
|
||||
name: 'error-test-trace',
|
||||
spanType: mlflow.SpanType.CHAIN,
|
||||
},
|
||||
),
|
||||
).rejects.toThrow();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('ERROR');
|
||||
|
||||
const llmSpan = trace.data.spans.find((s) => s.spanType === mlflow.SpanType.LLM);
|
||||
expect(llmSpan).toBeDefined();
|
||||
expect(llmSpan!.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
|
||||
expect(llmSpan!.inputs).toEqual({
|
||||
model: 'gemini-2.0-flash-001',
|
||||
contents: 'This should fail',
|
||||
});
|
||||
expect(llmSpan!.outputs).toBeUndefined();
|
||||
expect(llmSpan!.startTime).toBeDefined();
|
||||
expect(llmSpan!.endTime).toBeDefined();
|
||||
});
|
||||
|
||||
it('should trace Gemini request wrapped in a parent span', async () => {
|
||||
const gemini = new GoogleGenAI({ apiKey: 'test-key' });
|
||||
const wrappedGemini = tracedGemini(gemini);
|
||||
|
||||
const result = await mlflow.withSpan(
|
||||
async (_span) => {
|
||||
const response = await wrappedGemini.models.generateContent({
|
||||
model: 'gemini-2.0-flash-001',
|
||||
contents: 'Hello from parent span',
|
||||
});
|
||||
return response;
|
||||
},
|
||||
{
|
||||
name: 'predict',
|
||||
spanType: mlflow.SpanType.CHAIN,
|
||||
inputs: 'Hello from parent span',
|
||||
},
|
||||
);
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
expect(trace.data.spans.length).toBe(2);
|
||||
|
||||
const parentSpan = trace.data.spans[0];
|
||||
expect(parentSpan.name).toBe('predict');
|
||||
expect(parentSpan.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(parentSpan.spanType).toBe(mlflow.SpanType.CHAIN);
|
||||
expect(parentSpan.inputs).toEqual('Hello from parent span');
|
||||
expect(parentSpan.outputs).toEqual(result);
|
||||
expect(parentSpan.startTime).toBeDefined();
|
||||
expect(parentSpan.endTime).toBeDefined();
|
||||
|
||||
const childSpan = trace.data.spans[1];
|
||||
expect(childSpan.name).toBe('generateContent');
|
||||
expect(childSpan.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(childSpan.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(childSpan.inputs).toEqual({
|
||||
model: 'gemini-2.0-flash-001',
|
||||
contents: 'Hello from parent span',
|
||||
});
|
||||
expect(childSpan.outputs).toBeDefined();
|
||||
expect(childSpan.startTime).toBeDefined();
|
||||
expect(childSpan.endTime).toBeDefined();
|
||||
});
|
||||
|
||||
it('should not trace methods outside SUPPORTED_METHODS', () => {
|
||||
const gemini = new GoogleGenAI({ apiKey: 'test-key' });
|
||||
const wrappedGemini = tracedGemini(gemini);
|
||||
|
||||
expect(wrappedGemini.models).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* MSW-based Gemini API mock server for testing
|
||||
* Provides realistic Gemini API responses for comprehensive testing
|
||||
*/
|
||||
|
||||
import { http, HttpResponse } from 'msw';
|
||||
|
||||
interface GeminiGenerateContentRequest {
|
||||
model?: string;
|
||||
contents: any;
|
||||
config?: any;
|
||||
}
|
||||
|
||||
interface GeminiGenerateContentResponse {
|
||||
candidates?: Array<{
|
||||
content: {
|
||||
parts: Array<{
|
||||
text: string;
|
||||
}>;
|
||||
role: string;
|
||||
};
|
||||
finishReason: string;
|
||||
index: number;
|
||||
}>;
|
||||
usageMetadata: {
|
||||
promptTokenCount: number;
|
||||
candidatesTokenCount: number;
|
||||
totalTokenCount: number;
|
||||
};
|
||||
text?: () => string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a realistic Gemini generateContent response
|
||||
*/
|
||||
function createGenerateContentResponse(
|
||||
_request: GeminiGenerateContentRequest,
|
||||
): GeminiGenerateContentResponse {
|
||||
const responseText = 'Test response from Gemini';
|
||||
|
||||
return {
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{
|
||||
text: responseText,
|
||||
},
|
||||
],
|
||||
role: 'model',
|
||||
},
|
||||
finishReason: 'STOP',
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
usageMetadata: {
|
||||
promptTokenCount: 10,
|
||||
candidatesTokenCount: 5,
|
||||
totalTokenCount: 15,
|
||||
},
|
||||
text: () => responseText,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Main MSW handlers for Gemini API endpoints
|
||||
*/
|
||||
export const geminiMockHandlers = [
|
||||
http.post(
|
||||
'https://generativelanguage.googleapis.com/v1beta/models/*\\:generateContent',
|
||||
async ({ request }) => {
|
||||
const body = (await request.json()) as GeminiGenerateContentRequest;
|
||||
return HttpResponse.json(createGenerateContentResponse(body));
|
||||
},
|
||||
),
|
||||
|
||||
http.post(
|
||||
'https://generativelanguage.googleapis.com/v1/models/*\\:generateContent',
|
||||
async ({ request }) => {
|
||||
const body = (await request.json()) as GeminiGenerateContentRequest;
|
||||
return HttpResponse.json(createGenerateContentResponse(body));
|
||||
},
|
||||
),
|
||||
];
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"include": ["src/**/*"],
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* MSW-based OpenAI API mock server for testing
|
||||
* Provides realistic OpenAI API responses for comprehensive testing
|
||||
*/
|
||||
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import {
|
||||
ChatCompletion,
|
||||
ChatCompletionCreateParams,
|
||||
CreateEmbeddingResponse,
|
||||
EmbeddingCreateParams,
|
||||
} from 'openai/resources/index';
|
||||
import { ResponseCreateParams, Response } from 'openai/resources/responses/responses';
|
||||
import { setupServer } from 'msw/node';
|
||||
|
||||
/**
|
||||
* Create a realistic chat completion response
|
||||
*/
|
||||
function createChatCompletionResponse(request: ChatCompletionCreateParams): ChatCompletion {
|
||||
const timestamp = Math.floor(Date.now() / 1000);
|
||||
const requestId = `chatcmpl-${Math.random().toString(36).substring(2, 15)}`;
|
||||
|
||||
return {
|
||||
id: requestId,
|
||||
object: 'chat.completion',
|
||||
created: timestamp,
|
||||
model: request.model,
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: 'Test response content',
|
||||
refusal: null,
|
||||
},
|
||||
finish_reason: 'stop',
|
||||
logprobs: null,
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 200,
|
||||
total_tokens: 300,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a mock response for Responses API
|
||||
*/
|
||||
function createResponsesResponse(request: ResponseCreateParams): Response {
|
||||
return {
|
||||
id: 'responses-123',
|
||||
object: 'response',
|
||||
model: request.model || '',
|
||||
output: [
|
||||
{
|
||||
id: 'response-123',
|
||||
content: [
|
||||
{
|
||||
type: 'output_text',
|
||||
text: 'Dummy output',
|
||||
annotations: [],
|
||||
},
|
||||
],
|
||||
role: 'assistant',
|
||||
status: 'completed',
|
||||
type: 'message',
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
input_tokens: 36,
|
||||
output_tokens: 87,
|
||||
total_tokens: 123,
|
||||
input_tokens_details: {
|
||||
cached_tokens: 0,
|
||||
},
|
||||
output_tokens_details: {
|
||||
reasoning_tokens: 0,
|
||||
},
|
||||
},
|
||||
created_at: 123,
|
||||
output_text: 'Dummy output',
|
||||
error: null,
|
||||
incomplete_details: null,
|
||||
instructions: null,
|
||||
metadata: null,
|
||||
parallel_tool_calls: false,
|
||||
temperature: 0.5,
|
||||
tools: [],
|
||||
top_p: 1,
|
||||
tool_choice: 'auto',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a mock response for Embeddings API
|
||||
*/
|
||||
function createEmbeddingResponse(request: EmbeddingCreateParams): CreateEmbeddingResponse {
|
||||
const inputs = Array.isArray(request.input) ? request.input : [request.input];
|
||||
|
||||
return {
|
||||
object: 'list',
|
||||
data: inputs.map((_, index) => ({
|
||||
object: 'embedding',
|
||||
index,
|
||||
embedding: Array(1536)
|
||||
.fill(0)
|
||||
.map(() => Math.random() * 0.1 - 0.05),
|
||||
})),
|
||||
model: request.model,
|
||||
usage: {
|
||||
prompt_tokens: inputs.length * 10,
|
||||
total_tokens: inputs.length * 10,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Main MSW handlers for OpenAI API endpoints
|
||||
*/
|
||||
export const openAIMockHandlers = [
|
||||
http.post('https://api.openai.com/v1/chat/completions', async ({ request }) => {
|
||||
const body = (await request.json()) as ChatCompletionCreateParams;
|
||||
return HttpResponse.json(createChatCompletionResponse(body));
|
||||
}),
|
||||
http.post('https://api.openai.com/v1/responses', async ({ request }) => {
|
||||
const body = (await request.json()) as ResponseCreateParams;
|
||||
return HttpResponse.json(createResponsesResponse(body));
|
||||
}),
|
||||
http.post('https://api.openai.com/v1/embeddings', async ({ request }) => {
|
||||
const body = (await request.json()) as EmbeddingCreateParams;
|
||||
return HttpResponse.json(createEmbeddingResponse(body));
|
||||
}),
|
||||
];
|
||||
|
||||
export const openAIMswServer = setupServer(...openAIMockHandlers);
|
||||
|
||||
export function useMockOpenAIServer(): void {
|
||||
beforeAll(() => openAIMswServer.listen());
|
||||
afterEach(() => openAIMswServer.resetHandlers());
|
||||
afterAll(() => openAIMswServer.close());
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
# MLflow Typescript SDK - OpenAI
|
||||
|
||||
Seamlessly integrate [MLflow Tracing](https://github.com/mlflow/mlflow/tree/main/libs/typescript) with OpenAI to automatically trace your OpenAI API calls.
|
||||
|
||||
| Package | NPM | Description |
|
||||
| -------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- |
|
||||
| [@mlflow/openai](./) | [](https://www.npmjs.com/package/@mlflow/openai) | Auto-instrumentation integration for OpenAI. |
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @mlflow/openai
|
||||
```
|
||||
|
||||
The package includes the [`@mlflow/core`](https://github.com/mlflow/mlflow/tree/main/libs/typescript) package and `openai` package as peer dependencies. Depending on your package manager, you may need to install these two packages separately.
|
||||
|
||||
## Quickstart
|
||||
|
||||
Start MLflow Tracking Server. If you have a local Python environment, you can run the following command:
|
||||
|
||||
```bash
|
||||
pip install mlflow
|
||||
mlflow server --backend-store-uri sqlite:///mlruns.db --port 5000
|
||||
```
|
||||
|
||||
If you don't have Python environment locally, MLflow also supports Docker deployment or managed services. See [Self-Hosting Guide](https://mlflow.org/docs/latest/self-hosting/index.html) for getting started.
|
||||
|
||||
Instantiate MLflow SDK in your application:
|
||||
|
||||
```typescript
|
||||
import * as mlflow from '@mlflow/core';
|
||||
|
||||
mlflow.init({
|
||||
trackingUri: 'http://localhost:5000',
|
||||
experimentId: '<experiment-id>',
|
||||
});
|
||||
```
|
||||
|
||||
Create a trace:
|
||||
|
||||
```typescript
|
||||
import { OpenAI } from 'openai';
|
||||
import { tracedOpenAI } from '@mlflow/openai';
|
||||
|
||||
// Wrap the OpenAI client with the tracedOpenAI function
|
||||
const client = tracedOpenAI(new OpenAI());
|
||||
|
||||
// Invoke the client as usual
|
||||
const response = await client.chat.completions.create({
|
||||
model: 'o4-mini',
|
||||
messages: [
|
||||
{ role: 'system', content: 'You are a helpful weather assistant.' },
|
||||
{ role: 'user', content: "What's the weather like in Seattle?" },
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
View traces in MLflow UI:
|
||||
|
||||

|
||||
|
||||
## Documentation 📘
|
||||
|
||||
Official documentation for MLflow Typescript SDK can be found [here](https://mlflow.org/docs/latest/genai/tracing/quickstart).
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [Apache License 2.0](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt).
|
||||
@@ -0,0 +1 @@
|
||||
TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.
|
||||
@@ -0,0 +1 @@
|
||||
window.hierarchyData = 'eJyrVirKzy8pVrKKjtVRKkpNy0lNLsnMzytWsqqurQUAmx4Kpg==';
|
||||
@@ -0,0 +1,110 @@
|
||||
:root {
|
||||
--light-hl-0: #795e26;
|
||||
--dark-hl-0: #dcdcaa;
|
||||
--light-hl-1: #000000;
|
||||
--dark-hl-1: #d4d4d4;
|
||||
--light-hl-2: #a31515;
|
||||
--dark-hl-2: #ce9178;
|
||||
--light-hl-3: #0000ff;
|
||||
--dark-hl-3: #569cd6;
|
||||
--light-hl-4: #098658;
|
||||
--dark-hl-4: #b5cea8;
|
||||
--light-hl-5: #af00db;
|
||||
--dark-hl-5: #c586c0;
|
||||
--light-hl-6: #001080;
|
||||
--dark-hl-6: #9cdcfe;
|
||||
--light-hl-7: #008000;
|
||||
--dark-hl-7: #6a9955;
|
||||
--light-hl-8: #0070c1;
|
||||
--dark-hl-8: #4fc1ff;
|
||||
--light-code-background: #ffffff;
|
||||
--dark-code-background: #1e1e1e;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: light) {
|
||||
:root {
|
||||
--hl-0: var(--light-hl-0);
|
||||
--hl-1: var(--light-hl-1);
|
||||
--hl-2: var(--light-hl-2);
|
||||
--hl-3: var(--light-hl-3);
|
||||
--hl-4: var(--light-hl-4);
|
||||
--hl-5: var(--light-hl-5);
|
||||
--hl-6: var(--light-hl-6);
|
||||
--hl-7: var(--light-hl-7);
|
||||
--hl-8: var(--light-hl-8);
|
||||
--code-background: var(--light-code-background);
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--hl-0: var(--dark-hl-0);
|
||||
--hl-1: var(--dark-hl-1);
|
||||
--hl-2: var(--dark-hl-2);
|
||||
--hl-3: var(--dark-hl-3);
|
||||
--hl-4: var(--dark-hl-4);
|
||||
--hl-5: var(--dark-hl-5);
|
||||
--hl-6: var(--dark-hl-6);
|
||||
--hl-7: var(--dark-hl-7);
|
||||
--hl-8: var(--dark-hl-8);
|
||||
--code-background: var(--dark-code-background);
|
||||
}
|
||||
}
|
||||
|
||||
:root[data-theme='light'] {
|
||||
--hl-0: var(--light-hl-0);
|
||||
--hl-1: var(--light-hl-1);
|
||||
--hl-2: var(--light-hl-2);
|
||||
--hl-3: var(--light-hl-3);
|
||||
--hl-4: var(--light-hl-4);
|
||||
--hl-5: var(--light-hl-5);
|
||||
--hl-6: var(--light-hl-6);
|
||||
--hl-7: var(--light-hl-7);
|
||||
--hl-8: var(--light-hl-8);
|
||||
--code-background: var(--light-code-background);
|
||||
}
|
||||
|
||||
:root[data-theme='dark'] {
|
||||
--hl-0: var(--dark-hl-0);
|
||||
--hl-1: var(--dark-hl-1);
|
||||
--hl-2: var(--dark-hl-2);
|
||||
--hl-3: var(--dark-hl-3);
|
||||
--hl-4: var(--dark-hl-4);
|
||||
--hl-5: var(--dark-hl-5);
|
||||
--hl-6: var(--dark-hl-6);
|
||||
--hl-7: var(--dark-hl-7);
|
||||
--hl-8: var(--dark-hl-8);
|
||||
--code-background: var(--dark-code-background);
|
||||
}
|
||||
|
||||
.hl-0 {
|
||||
color: var(--hl-0);
|
||||
}
|
||||
.hl-1 {
|
||||
color: var(--hl-1);
|
||||
}
|
||||
.hl-2 {
|
||||
color: var(--hl-2);
|
||||
}
|
||||
.hl-3 {
|
||||
color: var(--hl-3);
|
||||
}
|
||||
.hl-4 {
|
||||
color: var(--hl-4);
|
||||
}
|
||||
.hl-5 {
|
||||
color: var(--hl-5);
|
||||
}
|
||||
.hl-6 {
|
||||
color: var(--hl-6);
|
||||
}
|
||||
.hl-7 {
|
||||
color: var(--hl-7);
|
||||
}
|
||||
.hl-8 {
|
||||
color: var(--hl-8);
|
||||
}
|
||||
pre,
|
||||
code {
|
||||
background: var(--code-background);
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 12 KiB |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,2 @@
|
||||
window.navigationData =
|
||||
'eJyLrlYqSa0oUbJSKilKTE5N8S9IzXP0VNJRKkgsyVCyUkorzUsuyczPK9ZHltfLKMnNUdJRys7MS1GyMjOpjQUAjN0Ylg==';
|
||||
@@ -0,0 +1,2 @@
|
||||
window.searchData =
|
||||
'eJxNkM1qAzEMhN9lzmKzpKVpfesxp9x6MUtYbIWaeuXF9qYB43cv+0dzE6PR6JMKYvhNULrgx4mFenslSD8wFHLsDdvLyPJ5BmGKHgq3SUx2QdLhud1858GDYHyfEicooHYEJ5YfUAV3jskFgcKxeWk+QLg59nZevG4jmDAMLBkEG8y0lN1m+2KTQ5zNq/vQgnRLx+b9dOo60vvsoi/CHrEpC8qdY2Z7XpG03s4LI0vvQAXXjbbdH1DQQpVa/9lUqU94c2+OHt3I3glD6a7WPwUQb30=';
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,286 @@
|
||||
<!doctype html>
|
||||
<html class="default" lang="en" data-base="../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta http-equiv="x-ua-compatible" content="IE=edge" />
|
||||
<title>tracedOpenAI | mlflow-openai</title>
|
||||
<meta name="description" content="Documentation for mlflow-openai" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<link rel="stylesheet" href="../assets/style.css" />
|
||||
<link rel="stylesheet" href="../assets/highlight.css" />
|
||||
<script defer src="../assets/main.js"></script>
|
||||
<script async src="../assets/icons.js" id="tsd-icons-script"></script>
|
||||
<script async src="../assets/search.js" id="tsd-search-script"></script>
|
||||
<script async src="../assets/navigation.js" id="tsd-nav-script"></script>
|
||||
</head>
|
||||
<body>
|
||||
<script>
|
||||
document.documentElement.dataset.theme = localStorage.getItem('tsd-theme') || 'os';
|
||||
document.body.style.display = 'none';
|
||||
setTimeout(
|
||||
() => (window.app ? app.showPage() : document.body.style.removeProperty('display')),
|
||||
500,
|
||||
);
|
||||
</script>
|
||||
<header class="tsd-page-toolbar">
|
||||
<div class="tsd-toolbar-contents container">
|
||||
<a href="../index.html" class="title">mlflow-openai</a>
|
||||
<div id="tsd-toolbar-links"></div>
|
||||
<button id="tsd-search-trigger" class="tsd-widget" aria-label="Search">
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
||||
<use href="../assets/icons.svg#icon-search"></use>
|
||||
</svg>
|
||||
</button>
|
||||
<dialog id="tsd-search" aria-label="Search">
|
||||
<input
|
||||
role="combobox"
|
||||
id="tsd-search-input"
|
||||
aria-controls="tsd-search-results"
|
||||
aria-autocomplete="list"
|
||||
aria-expanded="true"
|
||||
autocapitalize="off"
|
||||
autocomplete="off"
|
||||
placeholder="Search the docs"
|
||||
maxlength="100"
|
||||
/>
|
||||
<ul role="listbox" id="tsd-search-results"></ul>
|
||||
<div id="tsd-search-status" aria-live="polite" aria-atomic="true">
|
||||
<div>Preparing search index...</div>
|
||||
</div>
|
||||
</dialog>
|
||||
<a
|
||||
href="#"
|
||||
class="tsd-widget menu"
|
||||
id="tsd-toolbar-menu-trigger"
|
||||
data-toggle="menu"
|
||||
aria-label="Menu"
|
||||
><svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
||||
<use href="../assets/icons.svg#icon-menu"></use></svg
|
||||
></a>
|
||||
</div>
|
||||
</header>
|
||||
<div class="container container-main">
|
||||
<div class="col-content">
|
||||
<div class="tsd-page-title">
|
||||
<ul class="tsd-breadcrumb" aria-label="Breadcrumb">
|
||||
<li><a href="" aria-current="page">tracedOpenAI</a></li>
|
||||
</ul>
|
||||
<h1>Function tracedOpenAI</h1>
|
||||
</div>
|
||||
<section class="tsd-panel">
|
||||
<ul class="tsd-signatures">
|
||||
<li class="">
|
||||
<div class="tsd-signature tsd-anchor-link" id="tracedopenai">
|
||||
<span class="tsd-kind-call-signature">tracedOpenAI</span
|
||||
><span class="tsd-signature-symbol"><</span
|
||||
><a class="tsd-signature-type tsd-kind-type-parameter" href="#tracedopenait">T</a>
|
||||
<span class="tsd-signature-symbol">=</span>
|
||||
<span class="tsd-signature-type">any</span
|
||||
><span class="tsd-signature-symbol">></span
|
||||
><span class="tsd-signature-symbol">(</span
|
||||
><span class="tsd-kind-parameter">openaiClient</span
|
||||
><span class="tsd-signature-symbol">:</span>
|
||||
<a class="tsd-signature-type tsd-kind-type-parameter" href="#tracedopenait">T</a
|
||||
><span class="tsd-signature-symbol">)</span
|
||||
><span class="tsd-signature-symbol">:</span>
|
||||
<a class="tsd-signature-type tsd-kind-type-parameter" href="#tracedopenait">T</a
|
||||
><a href="#tracedopenai" aria-label="Permalink" class="tsd-anchor-icon"
|
||||
><svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<use href="../assets/icons.svg#icon-anchor"></use></svg
|
||||
></a>
|
||||
</div>
|
||||
<div class="tsd-description">
|
||||
<div class="tsd-comment tsd-typography">
|
||||
<p>Create a traced version of OpenAI client with MLflow tracing</p>
|
||||
</div>
|
||||
<section class="tsd-panel">
|
||||
<h4>Type Parameters</h4>
|
||||
<ul class="tsd-type-parameter-list">
|
||||
<li>
|
||||
<span id="tracedopenait"
|
||||
><span class="tsd-kind-type-parameter">T</span> =
|
||||
<span class="tsd-signature-type">any</span></span
|
||||
>
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
<div class="tsd-parameters">
|
||||
<h4 class="tsd-parameters-title">Parameters</h4>
|
||||
<ul class="tsd-parameter-list">
|
||||
<li>
|
||||
<span
|
||||
><span class="tsd-kind-parameter">openaiClient</span>:
|
||||
<a class="tsd-signature-type tsd-kind-type-parameter" href="#tracedopenait"
|
||||
>T</a
|
||||
></span
|
||||
>
|
||||
<div class="tsd-comment tsd-typography">
|
||||
<p>The OpenAI client instance to trace</p>
|
||||
</div>
|
||||
<div class="tsd-comment tsd-typography"></div>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<h4 class="tsd-returns-title">
|
||||
Returns
|
||||
<a class="tsd-signature-type tsd-kind-type-parameter" href="#tracedopenait">T</a>
|
||||
</h4>
|
||||
<p>Traced OpenAI client with tracing capabilities</p>
|
||||
<div class="tsd-comment tsd-typography">
|
||||
<div class="tsd-tag-example">
|
||||
<h4 class="tsd-anchor-link" id="example">
|
||||
Example<a href="#example" aria-label="Permalink" class="tsd-anchor-icon"
|
||||
><svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<use href="../assets/icons.svg#icon-anchor"></use></svg
|
||||
></a>
|
||||
</h4>
|
||||
<pre><code class="ts"><span class="hl-3">const</span><span class="hl-1"> </span><span class="hl-8">openai</span><span class="hl-1"> = </span><span class="hl-3">new</span><span class="hl-1"> </span><span class="hl-0">OpenAI</span><span class="hl-1">({ </span><span class="hl-6">apiKey:</span><span class="hl-1"> </span><span class="hl-2">'test-key'</span><span class="hl-1"> });</span><br/><span class="hl-3">const</span><span class="hl-1"> </span><span class="hl-8">wrappedOpenAI</span><span class="hl-1"> = </span><span class="hl-0">tracedOpenAI</span><span class="hl-1">(</span><span class="hl-6">openai</span><span class="hl-1">);</span><br/><br/><span class="hl-3">const</span><span class="hl-1"> </span><span class="hl-8">response</span><span class="hl-1"> = </span><span class="hl-5">await</span><span class="hl-1"> </span><span class="hl-6">wrappedOpenAI</span><span class="hl-1">.</span><span class="hl-6">chat</span><span class="hl-1">.</span><span class="hl-6">completions</span><span class="hl-1">.</span><span class="hl-0">create</span><span class="hl-1">({</span><br/><span class="hl-1"> </span><span class="hl-6">messages:</span><span class="hl-1"> [{ </span><span class="hl-6">role:</span><span class="hl-1"> </span><span class="hl-2">'user'</span><span class="hl-1">, </span><span class="hl-6">content:</span><span class="hl-1"> </span><span class="hl-2">'Hello!'</span><span class="hl-1"> }],</span><br/><span class="hl-1"> </span><span class="hl-6">model:</span><span class="hl-1"> </span><span class="hl-2">'gpt-4o-mini'</span><span class="hl-1">,</span><br/><span class="hl-1"> </span><span class="hl-6">temperature:</span><span class="hl-1"> </span><span class="hl-4">0.5</span><br/><span class="hl-1">});</span><br/><br/><span class="hl-7">// The trace for the LLM call will be logged to MLflow</span>
|
||||
</code><button type="button">Copy</button></pre>
|
||||
</div>
|
||||
</div>
|
||||
<aside class="tsd-sources">
|
||||
<ul>
|
||||
<li>
|
||||
Defined in
|
||||
<a
|
||||
href="https://github.com/B-Step62/mlflow/blob/eb4d233dd0494814558dd6fc289a886ce32ff004/packages/typescript/integrations/openai/src/index.ts#L34"
|
||||
>index.ts:34</a
|
||||
>
|
||||
</li>
|
||||
</ul>
|
||||
</aside>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
</div>
|
||||
<div class="col-sidebar">
|
||||
<div class="page-menu">
|
||||
<div class="tsd-navigation settings">
|
||||
<details class="tsd-accordion">
|
||||
<summary class="tsd-accordion-summary">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" aria-hidden="true">
|
||||
<use href="../assets/icons.svg#icon-chevronDown"></use>
|
||||
</svg>
|
||||
<h3>Settings</h3>
|
||||
</summary>
|
||||
<div class="tsd-accordion-details">
|
||||
<div class="tsd-filter-visibility">
|
||||
<span class="settings-label">Member Visibility</span>
|
||||
<ul id="tsd-filter-options">
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input type="checkbox" id="tsd-filter-protected" name="protected" /><svg
|
||||
width="32"
|
||||
height="32"
|
||||
viewBox="0 0 32 32"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>Protected</span></label
|
||||
>
|
||||
</li>
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input
|
||||
type="checkbox"
|
||||
id="tsd-filter-inherited"
|
||||
name="inherited"
|
||||
checked
|
||||
/><svg width="32" height="32" viewBox="0 0 32 32" aria-hidden="true">
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>Inherited</span></label
|
||||
>
|
||||
</li>
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input type="checkbox" id="tsd-filter-external" name="external" /><svg
|
||||
width="32"
|
||||
height="32"
|
||||
viewBox="0 0 32 32"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>External</span></label
|
||||
>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="tsd-theme-toggle">
|
||||
<label class="settings-label" for="tsd-theme">Theme</label
|
||||
><select id="tsd-theme">
|
||||
<option value="os">OS</option>
|
||||
<option value="light">Light</option>
|
||||
<option value="dark">Dark</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
<div class="site-menu">
|
||||
<nav class="tsd-navigation">
|
||||
<a href="../modules.html">mlflow-openai</a>
|
||||
<ul class="tsd-small-nested-navigation" id="tsd-nav-container">
|
||||
<li>Loading...</li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
<p class="tsd-generator">
|
||||
Generated using <a href="https://typedoc.org/" target="_blank">TypeDoc</a>
|
||||
</p>
|
||||
</footer>
|
||||
<div class="overlay"></div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,196 @@
|
||||
<!doctype html>
|
||||
<html class="default" lang="en" data-base="./">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta http-equiv="x-ua-compatible" content="IE=edge" />
|
||||
<title>mlflow-openai</title>
|
||||
<meta name="description" content="Documentation for mlflow-openai" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<link rel="stylesheet" href="assets/style.css" />
|
||||
<link rel="stylesheet" href="assets/highlight.css" />
|
||||
<script defer src="assets/main.js"></script>
|
||||
<script async src="assets/icons.js" id="tsd-icons-script"></script>
|
||||
<script async src="assets/search.js" id="tsd-search-script"></script>
|
||||
<script async src="assets/navigation.js" id="tsd-nav-script"></script>
|
||||
</head>
|
||||
<body>
|
||||
<script>
|
||||
document.documentElement.dataset.theme = localStorage.getItem('tsd-theme') || 'os';
|
||||
document.body.style.display = 'none';
|
||||
setTimeout(
|
||||
() => (window.app ? app.showPage() : document.body.style.removeProperty('display')),
|
||||
500,
|
||||
);
|
||||
</script>
|
||||
<header class="tsd-page-toolbar">
|
||||
<div class="tsd-toolbar-contents container">
|
||||
<a href="index.html" class="title">mlflow-openai</a>
|
||||
<div id="tsd-toolbar-links"></div>
|
||||
<button id="tsd-search-trigger" class="tsd-widget" aria-label="Search">
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-search"></use>
|
||||
</svg>
|
||||
</button>
|
||||
<dialog id="tsd-search" aria-label="Search">
|
||||
<input
|
||||
role="combobox"
|
||||
id="tsd-search-input"
|
||||
aria-controls="tsd-search-results"
|
||||
aria-autocomplete="list"
|
||||
aria-expanded="true"
|
||||
autocapitalize="off"
|
||||
autocomplete="off"
|
||||
placeholder="Search the docs"
|
||||
maxlength="100"
|
||||
/>
|
||||
<ul role="listbox" id="tsd-search-results"></ul>
|
||||
<div id="tsd-search-status" aria-live="polite" aria-atomic="true">
|
||||
<div>Preparing search index...</div>
|
||||
</div>
|
||||
</dialog>
|
||||
<a
|
||||
href="#"
|
||||
class="tsd-widget menu"
|
||||
id="tsd-toolbar-menu-trigger"
|
||||
data-toggle="menu"
|
||||
aria-label="Menu"
|
||||
><svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-menu"></use></svg
|
||||
></a>
|
||||
</div>
|
||||
</header>
|
||||
<div class="container container-main">
|
||||
<div class="col-content">
|
||||
<div class="tsd-page-title"><h1>mlflow-openai</h1></div>
|
||||
<h2>Hierarchy Summary</h2>
|
||||
</div>
|
||||
<div class="col-sidebar">
|
||||
<div class="page-menu">
|
||||
<div class="tsd-navigation settings">
|
||||
<details class="tsd-accordion">
|
||||
<summary class="tsd-accordion-summary">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-chevronDown"></use>
|
||||
</svg>
|
||||
<h3>Settings</h3>
|
||||
</summary>
|
||||
<div class="tsd-accordion-details">
|
||||
<div class="tsd-filter-visibility">
|
||||
<span class="settings-label">Member Visibility</span>
|
||||
<ul id="tsd-filter-options">
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input type="checkbox" id="tsd-filter-protected" name="protected" /><svg
|
||||
width="32"
|
||||
height="32"
|
||||
viewBox="0 0 32 32"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>Protected</span></label
|
||||
>
|
||||
</li>
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input
|
||||
type="checkbox"
|
||||
id="tsd-filter-inherited"
|
||||
name="inherited"
|
||||
checked
|
||||
/><svg width="32" height="32" viewBox="0 0 32 32" aria-hidden="true">
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>Inherited</span></label
|
||||
>
|
||||
</li>
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input type="checkbox" id="tsd-filter-external" name="external" /><svg
|
||||
width="32"
|
||||
height="32"
|
||||
viewBox="0 0 32 32"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>External</span></label
|
||||
>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="tsd-theme-toggle">
|
||||
<label class="settings-label" for="tsd-theme">Theme</label
|
||||
><select id="tsd-theme">
|
||||
<option value="os">OS</option>
|
||||
<option value="light">Light</option>
|
||||
<option value="dark">Dark</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
<div class="site-menu">
|
||||
<nav class="tsd-navigation">
|
||||
<a href="modules.html">mlflow-openai</a>
|
||||
<ul class="tsd-small-nested-navigation" id="tsd-nav-container">
|
||||
<li>Loading...</li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
<p class="tsd-generator">
|
||||
Generated using <a href="https://typedoc.org/" target="_blank">TypeDoc</a>
|
||||
</p>
|
||||
</footer>
|
||||
<div class="overlay"></div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,333 @@
|
||||
<!doctype html>
|
||||
<html class="default" lang="en" data-base="./">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta http-equiv="x-ua-compatible" content="IE=edge" />
|
||||
<title>mlflow-openai</title>
|
||||
<meta name="description" content="Documentation for mlflow-openai" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<link rel="stylesheet" href="assets/style.css" />
|
||||
<link rel="stylesheet" href="assets/highlight.css" />
|
||||
<script defer src="assets/main.js"></script>
|
||||
<script async src="assets/icons.js" id="tsd-icons-script"></script>
|
||||
<script async src="assets/search.js" id="tsd-search-script"></script>
|
||||
<script async src="assets/navigation.js" id="tsd-nav-script"></script>
|
||||
</head>
|
||||
<body>
|
||||
<script>
|
||||
document.documentElement.dataset.theme = localStorage.getItem('tsd-theme') || 'os';
|
||||
document.body.style.display = 'none';
|
||||
setTimeout(
|
||||
() => (window.app ? app.showPage() : document.body.style.removeProperty('display')),
|
||||
500,
|
||||
);
|
||||
</script>
|
||||
<header class="tsd-page-toolbar">
|
||||
<div class="tsd-toolbar-contents container">
|
||||
<a href="index.html" class="title">mlflow-openai</a>
|
||||
<div id="tsd-toolbar-links"></div>
|
||||
<button id="tsd-search-trigger" class="tsd-widget" aria-label="Search">
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-search"></use>
|
||||
</svg>
|
||||
</button>
|
||||
<dialog id="tsd-search" aria-label="Search">
|
||||
<input
|
||||
role="combobox"
|
||||
id="tsd-search-input"
|
||||
aria-controls="tsd-search-results"
|
||||
aria-autocomplete="list"
|
||||
aria-expanded="true"
|
||||
autocapitalize="off"
|
||||
autocomplete="off"
|
||||
placeholder="Search the docs"
|
||||
maxlength="100"
|
||||
/>
|
||||
<ul role="listbox" id="tsd-search-results"></ul>
|
||||
<div id="tsd-search-status" aria-live="polite" aria-atomic="true">
|
||||
<div>Preparing search index...</div>
|
||||
</div>
|
||||
</dialog>
|
||||
<a
|
||||
href="#"
|
||||
class="tsd-widget menu"
|
||||
id="tsd-toolbar-menu-trigger"
|
||||
data-toggle="menu"
|
||||
aria-label="Menu"
|
||||
><svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-menu"></use></svg
|
||||
></a>
|
||||
</div>
|
||||
</header>
|
||||
<div class="container container-main">
|
||||
<div class="col-content">
|
||||
<div class="tsd-page-title"><h1>mlflow-openai</h1></div>
|
||||
<div class="tsd-panel tsd-typography">
|
||||
<h1 id="mlflow-typescript-sdk---openai" class="tsd-anchor-link">
|
||||
MLflow Typescript SDK - OpenAI<a
|
||||
href="#mlflow-typescript-sdk---openai"
|
||||
aria-label="Permalink"
|
||||
class="tsd-anchor-icon"
|
||||
><svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-anchor"></use></svg
|
||||
></a>
|
||||
</h1>
|
||||
<p>
|
||||
Seamlessly integrate
|
||||
<a href="https://github.com/mlflow/mlflow/tree/main/libs/typescript">MLflow Tracing</a>
|
||||
with OpenAI to automatically trace your OpenAI API calls.
|
||||
</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Package</th>
|
||||
<th>NPM</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><a href="./libs/typescript/integrations/openai">mlflow-openai</a></td>
|
||||
<td>
|
||||
<a href="https://www.npmjs.com/package/mlflow-tracing-openai"
|
||||
><img
|
||||
src="https://img.shields.io/npm/v/mlflow-tracing-openai?style=flat-square"
|
||||
alt="npm package"
|
||||
/></a>
|
||||
</td>
|
||||
<td>Auto-instrumentation integration for OpenAI.</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h2 id="installation" class="tsd-anchor-link">
|
||||
Installation<a href="#installation" aria-label="Permalink" class="tsd-anchor-icon"
|
||||
><svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-anchor"></use></svg
|
||||
></a>
|
||||
</h2>
|
||||
<pre><code class="bash"><span class="hl-0">npm</span><span class="hl-1"> </span><span class="hl-2">install</span><span class="hl-1"> </span><span class="hl-2">mlflow-openai</span>
|
||||
</code><button type="button">Copy</button></pre>
|
||||
|
||||
<p>
|
||||
The package includes the
|
||||
<a href="https://github.com/mlflow/mlflow/tree/main/libs/typescript"
|
||||
><code>mlflow-tracing</code></a
|
||||
>
|
||||
package and <code>openai</code> package as peer dependencies. Depending on your package
|
||||
manager, you may need to install these two packages separately.
|
||||
</p>
|
||||
<h2 id="quickstart" class="tsd-anchor-link">
|
||||
Quickstart<a href="#quickstart" aria-label="Permalink" class="tsd-anchor-icon"
|
||||
><svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-anchor"></use></svg
|
||||
></a>
|
||||
</h2>
|
||||
<p>Start MLflow Tracking Server if you don't have one already:</p>
|
||||
<pre><code class="bash"><span class="hl-0">pip</span><span class="hl-1"> </span><span class="hl-2">install</span><span class="hl-1"> </span><span class="hl-2">mlflow</span><br/><span class="hl-0">mlflow</span><span class="hl-1"> </span><span class="hl-2">server</span><span class="hl-1"> </span><span class="hl-3">--backend-store-uri</span><span class="hl-1"> </span><span class="hl-2">sqlite:///mlruns.db</span><span class="hl-1"> </span><span class="hl-3">--port</span><span class="hl-1"> </span><span class="hl-4">5000</span>
|
||||
</code><button type="button">Copy</button></pre>
|
||||
|
||||
<p>
|
||||
Self-hosting MLflow server requires Python 3.10 or higher. If you don't have one, you
|
||||
can also use <a href="https://mlflow.org/#get-started">managed MLflow service</a> for
|
||||
free to get started quickly.
|
||||
</p>
|
||||
<p>Instantiate MLflow SDK in your application:</p>
|
||||
<pre><code class="typescript"><span class="hl-5">import</span><span class="hl-1"> </span><span class="hl-3">*</span><span class="hl-1"> </span><span class="hl-5">as</span><span class="hl-1"> </span><span class="hl-6">mlflow</span><span class="hl-1"> </span><span class="hl-5">from</span><span class="hl-1"> </span><span class="hl-2">'mlflow-tracing'</span><span class="hl-1">;</span><br/><br/><span class="hl-6">mlflow</span><span class="hl-1">.</span><span class="hl-0">init</span><span class="hl-1">({</span><br/><span class="hl-1"> </span><span class="hl-6">trackingUri:</span><span class="hl-1"> </span><span class="hl-2">'http://localhost:5000'</span><span class="hl-1">,</span><br/><span class="hl-1"> </span><span class="hl-6">experimentId:</span><span class="hl-1"> </span><span class="hl-2">'<experiment-id>'</span><br/><span class="hl-1">});</span>
|
||||
</code><button type="button">Copy</button></pre>
|
||||
|
||||
<p>Create a trace:</p>
|
||||
<pre><code class="typescript"><span class="hl-5">import</span><span class="hl-1"> { </span><span class="hl-6">OpenAI</span><span class="hl-1"> } </span><span class="hl-5">from</span><span class="hl-1"> </span><span class="hl-2">'openai'</span><span class="hl-1">;</span><br/><span class="hl-5">import</span><span class="hl-1"> { </span><span class="hl-6">tracedOpenAI</span><span class="hl-1"> } </span><span class="hl-5">from</span><span class="hl-1"> </span><span class="hl-2">'mlflow-openai'</span><span class="hl-1">;</span><br/><br/><span class="hl-7">// Wrap the OpenAI client with the tracedOpenAI function</span><br/><span class="hl-3">const</span><span class="hl-1"> </span><span class="hl-8">client</span><span class="hl-1"> = </span><span class="hl-0">tracedOpenAI</span><span class="hl-1">(</span><span class="hl-3">new</span><span class="hl-1"> </span><span class="hl-0">OpenAI</span><span class="hl-1">());</span><br/><br/><span class="hl-7">// Invoke the client as usual</span><br/><span class="hl-3">const</span><span class="hl-1"> </span><span class="hl-8">response</span><span class="hl-1"> = </span><span class="hl-5">await</span><span class="hl-1"> </span><span class="hl-6">client</span><span class="hl-1">.</span><span class="hl-6">chat</span><span class="hl-1">.</span><span class="hl-6">completions</span><span class="hl-1">.</span><span class="hl-0">create</span><span class="hl-1">({</span><br/><span class="hl-1"> </span><span class="hl-6">model:</span><span class="hl-1"> </span><span class="hl-2">'o4-mini'</span><span class="hl-1">,</span><br/><span class="hl-1"> </span><span class="hl-6">messages:</span><span class="hl-1"> [</span><br/><span class="hl-1"> { </span><span class="hl-6">role:</span><span class="hl-1"> </span><span class="hl-2">'system'</span><span class="hl-1">, </span><span class="hl-6">content:</span><span class="hl-1"> </span><span class="hl-2">'You are a helpful weather assistant.'</span><span class="hl-1"> },</span><br/><span class="hl-1"> { </span><span class="hl-6">role:</span><span class="hl-1"> </span><span class="hl-2">'user'</span><span class="hl-1">, </span><span class="hl-6">content:</span><span class="hl-1"> </span><span class="hl-2">"What's the weather like in Seattle?"</span><span class="hl-1"> }</span><br/><span class="hl-1"> ]</span><br/><span class="hl-1">});</span>
|
||||
</code><button type="button">Copy</button></pre>
|
||||
|
||||
<p>View traces in MLflow UI:</p>
|
||||
<p>
|
||||
<img
|
||||
src="https://github.com/mlflow/mlflow/blob/891fed9a746477f808dd2b82d3abb2382293c564/docs/static/images/llms/tracing/quickstart/single-openai-trace-detail.png?raw=true"
|
||||
alt="MLflow Tracing UI"
|
||||
/>
|
||||
</p>
|
||||
<h2 id="documentation-📘" class="tsd-anchor-link">
|
||||
Documentation 📘<a
|
||||
href="#documentation-📘"
|
||||
aria-label="Permalink"
|
||||
class="tsd-anchor-icon"
|
||||
><svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-anchor"></use></svg
|
||||
></a>
|
||||
</h2>
|
||||
<p>
|
||||
Official documentation for MLflow Typescript SDK can be found
|
||||
<a href="https://mlflow.org/docs/latest/genai/tracing/quickstart">here</a>.
|
||||
</p>
|
||||
<h2 id="license" class="tsd-anchor-link">
|
||||
License<a href="#license" aria-label="Permalink" class="tsd-anchor-icon"
|
||||
><svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-anchor"></use></svg
|
||||
></a>
|
||||
</h2>
|
||||
<p>
|
||||
This project is licensed under the
|
||||
<a href="https://github.com/mlflow/mlflow/blob/master/LICENSE.txt">Apache License 2.0</a
|
||||
>.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-sidebar">
|
||||
<div class="page-menu">
|
||||
<div class="tsd-navigation settings">
|
||||
<details class="tsd-accordion">
|
||||
<summary class="tsd-accordion-summary">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-chevronDown"></use>
|
||||
</svg>
|
||||
<h3>Settings</h3>
|
||||
</summary>
|
||||
<div class="tsd-accordion-details">
|
||||
<div class="tsd-filter-visibility">
|
||||
<span class="settings-label">Member Visibility</span>
|
||||
<ul id="tsd-filter-options">
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input type="checkbox" id="tsd-filter-protected" name="protected" /><svg
|
||||
width="32"
|
||||
height="32"
|
||||
viewBox="0 0 32 32"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>Protected</span></label
|
||||
>
|
||||
</li>
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input
|
||||
type="checkbox"
|
||||
id="tsd-filter-inherited"
|
||||
name="inherited"
|
||||
checked
|
||||
/><svg width="32" height="32" viewBox="0 0 32 32" aria-hidden="true">
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>Inherited</span></label
|
||||
>
|
||||
</li>
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input type="checkbox" id="tsd-filter-external" name="external" /><svg
|
||||
width="32"
|
||||
height="32"
|
||||
viewBox="0 0 32 32"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>External</span></label
|
||||
>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="tsd-theme-toggle">
|
||||
<label class="settings-label" for="tsd-theme">Theme</label
|
||||
><select id="tsd-theme">
|
||||
<option value="os">OS</option>
|
||||
<option value="light">Light</option>
|
||||
<option value="dark">Dark</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
<details open class="tsd-accordion tsd-page-navigation">
|
||||
<summary class="tsd-accordion-summary">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-chevronDown"></use>
|
||||
</svg>
|
||||
<h3>On This Page</h3>
|
||||
</summary>
|
||||
<div class="tsd-accordion-details">
|
||||
<a href="#mlflow-typescript-sdk---openai"
|
||||
><span
|
||||
>M<wbr />Lflow <wbr />Typescript <wbr />SDK -<wbr /> <wbr />Open<wbr />AI</span
|
||||
></a
|
||||
>
|
||||
<ul>
|
||||
<li>
|
||||
<a href="#installation"><span>Installation</span></a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="#quickstart"><span>Quickstart</span></a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="#documentation-📘"><span>Documentation 📘</span></a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="#license"><span>License</span></a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
<div class="site-menu">
|
||||
<nav class="tsd-navigation">
|
||||
<a href="modules.html">mlflow-openai</a>
|
||||
<ul class="tsd-small-nested-navigation" id="tsd-nav-container">
|
||||
<li>Loading...</li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
<p class="tsd-generator">
|
||||
Generated using <a href="https://typedoc.org/" target="_blank">TypeDoc</a>
|
||||
</p>
|
||||
</footer>
|
||||
<div class="overlay"></div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,243 @@
|
||||
<!doctype html>
|
||||
<html class="default" lang="en" data-base="./">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta http-equiv="x-ua-compatible" content="IE=edge" />
|
||||
<title>mlflow-openai</title>
|
||||
<meta name="description" content="Documentation for mlflow-openai" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<link rel="stylesheet" href="assets/style.css" />
|
||||
<link rel="stylesheet" href="assets/highlight.css" />
|
||||
<script defer src="assets/main.js"></script>
|
||||
<script async src="assets/icons.js" id="tsd-icons-script"></script>
|
||||
<script async src="assets/search.js" id="tsd-search-script"></script>
|
||||
<script async src="assets/navigation.js" id="tsd-nav-script"></script>
|
||||
</head>
|
||||
<body>
|
||||
<script>
|
||||
document.documentElement.dataset.theme = localStorage.getItem('tsd-theme') || 'os';
|
||||
document.body.style.display = 'none';
|
||||
setTimeout(
|
||||
() => (window.app ? app.showPage() : document.body.style.removeProperty('display')),
|
||||
500,
|
||||
);
|
||||
</script>
|
||||
<header class="tsd-page-toolbar">
|
||||
<div class="tsd-toolbar-contents container">
|
||||
<a href="index.html" class="title">mlflow-openai</a>
|
||||
<div id="tsd-toolbar-links"></div>
|
||||
<button id="tsd-search-trigger" class="tsd-widget" aria-label="Search">
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-search"></use>
|
||||
</svg>
|
||||
</button>
|
||||
<dialog id="tsd-search" aria-label="Search">
|
||||
<input
|
||||
role="combobox"
|
||||
id="tsd-search-input"
|
||||
aria-controls="tsd-search-results"
|
||||
aria-autocomplete="list"
|
||||
aria-expanded="true"
|
||||
autocapitalize="off"
|
||||
autocomplete="off"
|
||||
placeholder="Search the docs"
|
||||
maxlength="100"
|
||||
/>
|
||||
<ul role="listbox" id="tsd-search-results"></ul>
|
||||
<div id="tsd-search-status" aria-live="polite" aria-atomic="true">
|
||||
<div>Preparing search index...</div>
|
||||
</div>
|
||||
</dialog>
|
||||
<a
|
||||
href="#"
|
||||
class="tsd-widget menu"
|
||||
id="tsd-toolbar-menu-trigger"
|
||||
data-toggle="menu"
|
||||
aria-label="Menu"
|
||||
><svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-menu"></use></svg
|
||||
></a>
|
||||
</div>
|
||||
</header>
|
||||
<div class="container container-main">
|
||||
<div class="col-content">
|
||||
<div class="tsd-page-title">
|
||||
<ul class="tsd-breadcrumb" aria-label="Breadcrumb"></ul>
|
||||
<h1>mlflow-openai</h1>
|
||||
</div>
|
||||
<details class="tsd-panel-group tsd-member-group tsd-accordion" open>
|
||||
<summary class="tsd-accordion-summary" data-key="section-Functions">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-chevronDown"></use>
|
||||
</svg>
|
||||
<h2>Functions</h2>
|
||||
</summary>
|
||||
<dl class="tsd-member-summaries">
|
||||
<dt class="tsd-member-summary" id="tracedopenai">
|
||||
<span class="tsd-member-summary-name"
|
||||
><svg class="tsd-kind-icon" viewBox="0 0 24 24" aria-label="Function">
|
||||
<use href="assets/icons.svg#icon-64"></use></svg
|
||||
><a href="functions/tracedOpenAI.html">tracedOpenAI</a
|
||||
><a href="#tracedopenai" aria-label="Permalink" class="tsd-anchor-icon"
|
||||
><svg viewBox="0 0 24 24" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-anchor"></use></svg></a
|
||||
></span>
|
||||
</dt>
|
||||
<dd class="tsd-member-summary"></dd>
|
||||
</dl>
|
||||
</details>
|
||||
</div>
|
||||
<div class="col-sidebar">
|
||||
<div class="page-menu">
|
||||
<div class="tsd-navigation settings">
|
||||
<details class="tsd-accordion">
|
||||
<summary class="tsd-accordion-summary">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-chevronDown"></use>
|
||||
</svg>
|
||||
<h3>Settings</h3>
|
||||
</summary>
|
||||
<div class="tsd-accordion-details">
|
||||
<div class="tsd-filter-visibility">
|
||||
<span class="settings-label">Member Visibility</span>
|
||||
<ul id="tsd-filter-options">
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input type="checkbox" id="tsd-filter-protected" name="protected" /><svg
|
||||
width="32"
|
||||
height="32"
|
||||
viewBox="0 0 32 32"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>Protected</span></label
|
||||
>
|
||||
</li>
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input
|
||||
type="checkbox"
|
||||
id="tsd-filter-inherited"
|
||||
name="inherited"
|
||||
checked
|
||||
/><svg width="32" height="32" viewBox="0 0 32 32" aria-hidden="true">
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>Inherited</span></label
|
||||
>
|
||||
</li>
|
||||
<li class="tsd-filter-item">
|
||||
<label class="tsd-filter-input"
|
||||
><input type="checkbox" id="tsd-filter-external" name="external" /><svg
|
||||
width="32"
|
||||
height="32"
|
||||
viewBox="0 0 32 32"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect
|
||||
class="tsd-checkbox-background"
|
||||
width="30"
|
||||
height="30"
|
||||
x="1"
|
||||
y="1"
|
||||
rx="6"
|
||||
fill="none"
|
||||
></rect>
|
||||
<path
|
||||
class="tsd-checkbox-checkmark"
|
||||
d="M8.35422 16.8214L13.2143 21.75L24.6458 10.25"
|
||||
stroke="none"
|
||||
stroke-width="3.5"
|
||||
stroke-linejoin="round"
|
||||
fill="none"
|
||||
></path></svg
|
||||
><span>External</span></label
|
||||
>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="tsd-theme-toggle">
|
||||
<label class="settings-label" for="tsd-theme">Theme</label
|
||||
><select id="tsd-theme">
|
||||
<option value="os">OS</option>
|
||||
<option value="light">Light</option>
|
||||
<option value="dark">Dark</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
<details open class="tsd-accordion tsd-page-navigation">
|
||||
<summary class="tsd-accordion-summary">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-chevronDown"></use>
|
||||
</svg>
|
||||
<h3>On This Page</h3>
|
||||
</summary>
|
||||
<div class="tsd-accordion-details">
|
||||
<details open class="tsd-accordion tsd-page-navigation-section">
|
||||
<summary class="tsd-accordion-summary" data-key="section-Functions">
|
||||
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" aria-hidden="true">
|
||||
<use href="assets/icons.svg#icon-chevronDown"></use></svg
|
||||
>Functions
|
||||
</summary>
|
||||
<div>
|
||||
<a href="#tracedopenai"
|
||||
><svg class="tsd-kind-icon" viewBox="0 0 24 24" aria-label="Function">
|
||||
<use href="assets/icons.svg#icon-64"></use></svg
|
||||
><span>traced<wbr />Open<wbr />AI</span></a
|
||||
>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
<div class="site-menu">
|
||||
<nav class="tsd-navigation">
|
||||
<a href="modules.html" class="current">mlflow-openai</a>
|
||||
<ul class="tsd-small-nested-navigation" id="tsd-nav-container">
|
||||
<li>Loading...</li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
<p class="tsd-generator">
|
||||
Generated using <a href="https://typedoc.org/" target="_blank">TypeDoc</a>
|
||||
</p>
|
||||
</footer>
|
||||
<div class="overlay"></div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,16 @@
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
roots: ['<rootDir>/tests', '<rootDir>/src'],
|
||||
testMatch: ['**/*.test.ts'],
|
||||
moduleFileExtensions: ['ts', 'js', 'json', 'node'],
|
||||
transform: {
|
||||
'^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
|
||||
},
|
||||
globalSetup: '<rootDir>/../../jest.global-server-setup.ts',
|
||||
globalTeardown: '<rootDir>/../../jest.global-server-teardown.ts',
|
||||
testTimeout: 30000,
|
||||
forceExit: true,
|
||||
detectOpenHandles: true,
|
||||
};
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"name": "@mlflow/openai",
|
||||
"version": "0.3.0",
|
||||
"description": "OpenAI integration package for MLflow Tracing",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mlflow/mlflow.git"
|
||||
},
|
||||
"homepage": "https://mlflow.org/",
|
||||
"author": {
|
||||
"name": "MLflow",
|
||||
"url": "https://mlflow.org/"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/mlflow/mlflow/issues"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"keywords": [
|
||||
"mlflow",
|
||||
"tracing",
|
||||
"observability",
|
||||
"opentelemetry",
|
||||
"llm",
|
||||
"openai",
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"prepublishOnly": "npm run build",
|
||||
"test": "jest",
|
||||
"lint": "eslint . --ext .ts --max-warnings 0",
|
||||
"lint:fix": "eslint . --ext .ts --fix",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@mlflow/core": "^0.3.0",
|
||||
"openai": ">=4.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"jest": "^29.6.2",
|
||||
"typescript": "^5.8.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"files": [
|
||||
"dist/"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* Main tracedOpenAI wrapper function for MLflow tracing integration
|
||||
*/
|
||||
|
||||
import { CompletionUsage } from 'openai/resources/index';
|
||||
import { ResponseUsage } from 'openai/resources/responses/responses';
|
||||
import { withSpan, LiveSpan, SpanAttributeKey, SpanType, TokenUsage } from '@mlflow/core';
|
||||
|
||||
// NB: 'Completions' represents chat.completions
|
||||
const SUPPORTED_MODULES = ['Completions', 'Responses', 'Embeddings'];
|
||||
const SUPPORTED_METHODS = ['create']; // chat.completions.create, embeddings.create, responses.create
|
||||
|
||||
type OpenAIUsage = CompletionUsage | ResponseUsage;
|
||||
|
||||
/**
|
||||
* Create a traced version of OpenAI client with MLflow tracing
|
||||
* @param openaiClient - The OpenAI client instance to trace
|
||||
* @param config - Optional configuration for tracing
|
||||
* @returns Traced OpenAI client with tracing capabilities
|
||||
*
|
||||
* @example
|
||||
* const openai = new OpenAI({ apiKey: 'test-key' });
|
||||
* const wrappedOpenAI = tracedOpenAI(openai);
|
||||
*
|
||||
* const response = await wrappedOpenAI.chat.completions.create({
|
||||
* messages: [{ role: 'user', content: 'Hello!' }],
|
||||
* model: 'gpt-4o-mini',
|
||||
* temperature: 0.5
|
||||
* });
|
||||
*
|
||||
* // The trace for the LLM call will be logged to MLflow
|
||||
*
|
||||
*/
|
||||
export function tracedOpenAI<T = any>(openaiClient: T): T {
|
||||
/**
|
||||
* Create a proxy to intercept method calls
|
||||
*/
|
||||
const tracedClient = new Proxy(openaiClient as any, {
|
||||
get(target, prop, receiver) {
|
||||
const original = Reflect.get(target, prop, receiver);
|
||||
const moduleName = (target as object).constructor.name;
|
||||
|
||||
if (typeof original === 'function') {
|
||||
// If reach to the end function to be traced, wrap it with tracing
|
||||
if (shouldTraceMethod(moduleName, String(prop))) {
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return wrapWithTracing(original as Function, moduleName) as T;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
return (original as Function).bind(target) as T;
|
||||
}
|
||||
|
||||
// For nested objects (like chat.completions), recursively apply tracking
|
||||
if (
|
||||
original &&
|
||||
!Array.isArray(original) &&
|
||||
!(original instanceof Date) &&
|
||||
typeof original === 'object'
|
||||
) {
|
||||
return tracedOpenAI(original) as T;
|
||||
}
|
||||
|
||||
return original as T;
|
||||
},
|
||||
});
|
||||
return tracedClient as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if a method should be traced based on the target object and property
|
||||
*/
|
||||
function shouldTraceMethod(module: string, methodName: string): boolean {
|
||||
return SUPPORTED_MODULES.includes(module) && SUPPORTED_METHODS.includes(methodName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a function with tracing using the full method path
|
||||
*
|
||||
* @param fn - The function to wrap
|
||||
* @param target - The target module that contains the function to wrap
|
||||
* @returns The wrapped function
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
function wrapWithTracing(fn: Function, moduleName: string): Function {
|
||||
// Use the full method path for span type determination
|
||||
const spanType = getSpanType(moduleName);
|
||||
const name = moduleName;
|
||||
|
||||
return function (this: any, ...args: any[]) {
|
||||
// If the method is not supported, return the original function
|
||||
if (!spanType) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return fn.apply(this, args);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return withSpan(
|
||||
async (span: LiveSpan) => {
|
||||
span.setInputs(args[0]);
|
||||
|
||||
const result = await fn.apply(this, args);
|
||||
|
||||
// TODO: Handle streaming responses
|
||||
span.setOutputs(result);
|
||||
|
||||
// Add token usage
|
||||
try {
|
||||
const usage = extractTokenUsage(result);
|
||||
if (usage) {
|
||||
span.setAttribute(SpanAttributeKey.TOKEN_USAGE, usage);
|
||||
}
|
||||
} catch (error) {
|
||||
console.debug('Error extracting token usage', error);
|
||||
}
|
||||
|
||||
span.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'openai');
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
|
||||
return result;
|
||||
},
|
||||
{ name, spanType },
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine span type based on the full method path
|
||||
*/
|
||||
function getSpanType(moduleName: string): SpanType | undefined {
|
||||
switch (moduleName) {
|
||||
case 'Completions':
|
||||
return SpanType.LLM;
|
||||
case 'Responses':
|
||||
return SpanType.LLM;
|
||||
case 'Embeddings':
|
||||
return SpanType.EMBEDDING;
|
||||
// TODO: Support other methods in the future.
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract token usage information from OpenAI response
|
||||
* Supports both ChatCompletion API format and Responses API format
|
||||
*/
|
||||
function extractTokenUsage(response: any): TokenUsage | undefined {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const usage = response?.usage as OpenAIUsage | undefined;
|
||||
|
||||
if (!usage) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Try Responses API format first (input_tokens, output_tokens)
|
||||
if ('input_tokens' in usage) {
|
||||
return {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
total_tokens: usage.total_tokens || usage.input_tokens + usage.output_tokens,
|
||||
};
|
||||
}
|
||||
|
||||
// Fall back to ChatCompletion API format (prompt_tokens, completion_tokens)
|
||||
if ('prompt_tokens' in usage) {
|
||||
return {
|
||||
input_tokens: usage.prompt_tokens,
|
||||
output_tokens: usage.completion_tokens ?? 0,
|
||||
total_tokens: usage.total_tokens || usage.prompt_tokens + (usage.completion_tokens ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
/**
|
||||
* Tests for MLflow OpenAI integration with MSW mock server
|
||||
*/
|
||||
|
||||
import * as mlflow from '@mlflow/core';
|
||||
import { tracedOpenAI } from '../src';
|
||||
import { OpenAI } from 'openai';
|
||||
import { http, HttpResponse } from 'msw';
|
||||
import { openAIMswServer, useMockOpenAIServer } from '../../helpers/openaiTestHelper';
|
||||
import { createAuthProvider } from '@mlflow/core/src/auth';
|
||||
|
||||
const TEST_TRACKING_URI = 'http://localhost:5000';
|
||||
|
||||
describe('tracedOpenAI', () => {
|
||||
useMockOpenAIServer();
|
||||
|
||||
let experimentId: string;
|
||||
let client: mlflow.MlflowClient;
|
||||
|
||||
beforeAll(async () => {
|
||||
// Setup MLflow client and experiment
|
||||
const authProvider = createAuthProvider({ trackingUri: TEST_TRACKING_URI });
|
||||
client = new mlflow.MlflowClient({ trackingUri: TEST_TRACKING_URI, authProvider });
|
||||
|
||||
// Create a new experiment
|
||||
const experimentName = `test-experiment-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
|
||||
experimentId = await client.createExperiment(experimentName);
|
||||
mlflow.init({
|
||||
trackingUri: TEST_TRACKING_URI,
|
||||
experimentId: experimentId,
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await client.deleteExperiment(experimentId);
|
||||
});
|
||||
|
||||
const getLastActiveTrace = async (): Promise<mlflow.Trace> => {
|
||||
await mlflow.flushTraces();
|
||||
const traceId = mlflow.getLastActiveTraceId();
|
||||
const trace = await client.getTrace(traceId!);
|
||||
return trace;
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('Chat Completions', () => {
|
||||
it('should trace chat.completions.create()', async () => {
|
||||
const openai = new OpenAI({ apiKey: 'test-key' });
|
||||
const wrappedOpenAI = tracedOpenAI(openai);
|
||||
|
||||
const result = await wrappedOpenAI.chat.completions.create({
|
||||
model: 'gpt-4',
|
||||
messages: [{ role: 'user', content: 'Hello!' }],
|
||||
});
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
|
||||
const tokenUsage = trace.info.tokenUsage;
|
||||
expect(tokenUsage).toBeDefined();
|
||||
expect(typeof tokenUsage?.input_tokens).toBe('number');
|
||||
expect(typeof tokenUsage?.output_tokens).toBe('number');
|
||||
expect(typeof tokenUsage?.total_tokens).toBe('number');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.name).toBe('Completions');
|
||||
expect(span.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(span.logLevel).toBe(mlflow.SpanLogLevel.INFO);
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'gpt-4',
|
||||
messages: [{ role: 'user', content: 'Hello!' }],
|
||||
});
|
||||
expect(span.outputs).toEqual(result);
|
||||
expect(span.startTime).toBeDefined();
|
||||
expect(span.endTime).toBeDefined();
|
||||
|
||||
// Check that token usage is stored at span level
|
||||
const spanTokenUsage = span.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
|
||||
expect(spanTokenUsage).toBeDefined();
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe('number');
|
||||
expect(typeof spanTokenUsage[mlflow.TokenUsageKey.TOTAL_TOKENS]).toBe('number');
|
||||
});
|
||||
|
||||
it('should handle chat completion errors properly', async () => {
|
||||
// Configure MSW to return rate limit error
|
||||
openAIMswServer.use(
|
||||
http.post('https://api.openai.com/v1/chat/completions', () => {
|
||||
return HttpResponse.json(
|
||||
{
|
||||
error: {
|
||||
type: 'requests',
|
||||
message: 'Rate limit exceeded',
|
||||
},
|
||||
},
|
||||
{ status: 429 },
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
const openai = new OpenAI({ apiKey: 'test-key' });
|
||||
const wrappedOpenAI = tracedOpenAI(openai);
|
||||
|
||||
await expect(
|
||||
wrappedOpenAI.chat.completions.create({
|
||||
model: 'gpt-4',
|
||||
messages: [{ role: 'user', content: 'This should fail' }],
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('ERROR');
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.ERROR);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'gpt-4',
|
||||
messages: [{ role: 'user', content: 'This should fail' }],
|
||||
});
|
||||
expect(span.outputs).toBeUndefined();
|
||||
expect(span.startTime).toBeDefined();
|
||||
expect(span.endTime).toBeDefined();
|
||||
});
|
||||
|
||||
it('should trace OpenAI request wrapped in a parent span', async () => {
|
||||
const openai = new OpenAI({ apiKey: 'test-key' });
|
||||
const wrappedOpenAI = tracedOpenAI(openai);
|
||||
|
||||
const result = await mlflow.withSpan(
|
||||
async (_span) => {
|
||||
const response = await wrappedOpenAI.chat.completions.create({
|
||||
model: 'gpt-4',
|
||||
messages: [{ role: 'user', content: 'Hello!' }],
|
||||
});
|
||||
return response.choices[0].message.content;
|
||||
},
|
||||
{
|
||||
name: 'predict',
|
||||
spanType: mlflow.SpanType.CHAIN,
|
||||
inputs: 'Hello!',
|
||||
},
|
||||
);
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
expect(trace.data.spans.length).toBe(2);
|
||||
|
||||
const parentSpan = trace.data.spans[0];
|
||||
expect(parentSpan.name).toBe('predict');
|
||||
expect(parentSpan.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(parentSpan.spanType).toBe(mlflow.SpanType.CHAIN);
|
||||
// CHAIN spans default to DEBUG; LLM children to INFO (asserted below).
|
||||
expect(parentSpan.logLevel).toBe(mlflow.SpanLogLevel.DEBUG);
|
||||
expect(parentSpan.inputs).toEqual('Hello!');
|
||||
expect(parentSpan.outputs).toEqual(result);
|
||||
expect(parentSpan.startTime).toBeDefined();
|
||||
expect(parentSpan.endTime).toBeDefined();
|
||||
|
||||
const childSpan = trace.data.spans[1];
|
||||
expect(childSpan.name).toBe('Completions');
|
||||
expect(childSpan.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(childSpan.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(childSpan.logLevel).toBe(mlflow.SpanLogLevel.INFO);
|
||||
expect(childSpan.inputs).toEqual({
|
||||
model: 'gpt-4',
|
||||
messages: [{ role: 'user', content: 'Hello!' }],
|
||||
});
|
||||
expect(childSpan.outputs).toBeDefined();
|
||||
expect(childSpan.startTime).toBeDefined();
|
||||
expect(childSpan.endTime).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Responses API', () => {
|
||||
it('should trace responses.create()', async () => {
|
||||
const openai = new OpenAI({ apiKey: 'test-key' });
|
||||
const wrappedOpenAI = tracedOpenAI(openai);
|
||||
|
||||
const response = await wrappedOpenAI.responses.create({
|
||||
input: 'Hello!',
|
||||
model: 'gpt-4o',
|
||||
temperature: 0,
|
||||
});
|
||||
|
||||
// Verify response
|
||||
expect((response as any).id).toBe('responses-123');
|
||||
|
||||
// Get and verify the trace
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
expect(trace.info.tokenUsage?.input_tokens).toBe(response.usage?.input_tokens);
|
||||
expect(trace.info.tokenUsage?.output_tokens).toBe(response.usage?.output_tokens);
|
||||
expect(trace.info.tokenUsage?.total_tokens).toBe(response.usage?.total_tokens);
|
||||
expect(trace.data.spans.length).toBe(1);
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.spanType).toBe(mlflow.SpanType.LLM);
|
||||
expect(span.inputs).toEqual({
|
||||
input: 'Hello!',
|
||||
model: 'gpt-4o',
|
||||
temperature: 0,
|
||||
});
|
||||
expect(span.outputs).toEqual(response);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Embeddings API', () => {
|
||||
it('should trace embeddings.create() with input: %p', async () => {
|
||||
const openai = new OpenAI({ apiKey: 'test-key' });
|
||||
const wrappedOpenAI = tracedOpenAI(openai);
|
||||
|
||||
const response = await wrappedOpenAI.embeddings.create({
|
||||
model: 'text-embedding-3-small',
|
||||
input: ['Hello', 'world'],
|
||||
});
|
||||
|
||||
expect(response.object).toBe('list');
|
||||
expect(response.data.length).toBe(2);
|
||||
expect(response.data[0].object).toBe('embedding');
|
||||
expect(response.data[0].embedding.length).toBeGreaterThan(0);
|
||||
expect(response.model).toBe('text-embedding-3-small');
|
||||
|
||||
const trace = await getLastActiveTrace();
|
||||
expect(trace.info.state).toBe('OK');
|
||||
expect(trace.data.spans.length).toBe(1);
|
||||
|
||||
const tokenUsage = trace.info.tokenUsage;
|
||||
expect(tokenUsage).toBeDefined();
|
||||
expect(tokenUsage?.input_tokens).toBe(response.usage.prompt_tokens);
|
||||
expect(tokenUsage?.output_tokens).toBe(0);
|
||||
expect(tokenUsage?.total_tokens).toBe(response.usage.total_tokens);
|
||||
|
||||
const span = trace.data.spans[0];
|
||||
expect(span.name).toBe('Embeddings');
|
||||
expect(span.spanType).toBe(mlflow.SpanType.EMBEDDING);
|
||||
expect(span.status.statusCode).toBe(mlflow.SpanStatusCode.OK);
|
||||
expect(span.inputs).toEqual({
|
||||
model: 'text-embedding-3-small',
|
||||
input: ['Hello', 'world'],
|
||||
});
|
||||
expect(span.outputs).toEqual(response);
|
||||
expect(span.startTime).toBeDefined();
|
||||
expect(span.endTime).toBeDefined();
|
||||
|
||||
const spanTokenUsage = span.attributes[mlflow.SpanAttributeKey.TOKEN_USAGE];
|
||||
expect(spanTokenUsage).toBeDefined();
|
||||
expect(spanTokenUsage[mlflow.TokenUsageKey.INPUT_TOKENS]).toBe(response.usage.prompt_tokens);
|
||||
expect(spanTokenUsage[mlflow.TokenUsageKey.OUTPUT_TOKENS]).toBe(0);
|
||||
expect(spanTokenUsage[mlflow.TokenUsageKey.TOTAL_TOKENS]).toBe(response.usage.total_tokens);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"include": ["src/**/*"],
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
<h1 align="center" style="border-bottom: none">
|
||||
<a href="https://mlflow.org/">
|
||||
<img alt="MLflow logo" src="https://raw.githubusercontent.com/mlflow/mlflow/refs/heads/master/assets/logo.svg" width="200" />
|
||||
</a>
|
||||
</h1>
|
||||
<h2 align="center" style="border-bottom: none">🦞 OpenClaw MLflow Observability Plugin</h2>
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](https://www.npmjs.com/package/@mlflow/mlflow-openclaw)
|
||||
[](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt)
|
||||
<a href="https://twitter.com/intent/follow?screen_name=mlflow" target="_blank">
|
||||
<img src="https://img.shields.io/twitter/follow/mlflow?logo=X&color=%20%23f5f5f5"
|
||||
alt="follow on X(Twitter)"></a>
|
||||
<a href="https://www.linkedin.com/company/mlflow-org/" target="_blank">
|
||||
<img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
|
||||
alt="follow on LinkedIn"></a>
|
||||
|
||||
</div>
|
||||
|
||||
MLflow integration for [OpenClaw](https://github.com/openclaw/openclaw) for [observability](https://mlflow.org/docs/latest/genai/tracing/), [evaluation](https://mlflow.org/docs/latest/genai/eval-monitor/), and [monitoring](https://mlflow.org/docs/latest/genai/governance/ai-gateway/). This plugin automatically traces OpenClaw agent executions in MLflow, capturing LLM calls, tool invocations, and sub-agent spans in a hierarchical trace structure.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/mlflow/mlflow/master/libs/typescript/integrations/openclaw/dashboard-screenshot.png" alt="OpenClaw MLflow Integration" width="700" style="border-radius: 10px; box-shadow: 0 4px 16px rgba(0,0,0,0.18);" />
|
||||
</p>
|
||||
|
||||
## Key Benefits
|
||||
|
||||
- 🌐 **Open Source**: MLflow is 100% open source and governed by the Linux Foundation, rooted in the same philosophy as OpenClaw.
|
||||
- 🛡️ **You Own Your Data**: MLflow is self-hosted. Trace data from OpenClaw stays on your infrastructure and never leaves it.
|
||||
- 🔀 **Vendor Neutral**: MLflow works with any LLM provider or agent framework, with no vendor lock-in.
|
||||
|
||||
## Setup
|
||||
|
||||
### 1. Install the Plugin
|
||||
|
||||
```bash
|
||||
openclaw plugins install @mlflow/mlflow-openclaw
|
||||
```
|
||||
|
||||
### 2. Start the MLflow Server
|
||||
|
||||
Start the MLflow server (self-hosting) following the [instructions](https://mlflow.org/docs/latest/genai/getting-started/connect-environment/). Alternatively, use a managed MLflow service if you prefer not to self-host.
|
||||
|
||||
### 3. Configure the Plugin
|
||||
|
||||
```
|
||||
openclaw mlflow configure
|
||||
```
|
||||
|
||||
The plugin will prompt you for the MLflow tracking URI and experiment ID. You can [create an experiment](https://mlflow.org/docs/latest/genai/tracing/quickstart/#create-a-mlflow-experiment) from the MLflow UI.
|
||||
|
||||
```
|
||||
~$ openclaw mlflow configure
|
||||
|
||||
🦞 OpenClaw 2026.3.13 (61d171a) — Automation with claws: minimal fuss, maximal pinch.
|
||||
|
||||
┌ MLflow Tracing configuration
|
||||
│
|
||||
◆ MLflow Tracking URI
|
||||
│ http://localhost:5000
|
||||
└
|
||||
◇ Experiment ID
|
||||
│ 2
|
||||
```
|
||||
|
||||
### 4. Check the Status
|
||||
|
||||
Verify the configuration by running the following command:
|
||||
|
||||
```bash
|
||||
openclaw mlflow status
|
||||
```
|
||||
|
||||
If the configuration is successful, you should see the effective configuration in the output.
|
||||
|
||||
### 5. Talk to OpenClaw
|
||||
|
||||
Run or restart the OpenClaw gateway to apply the configuration.
|
||||
|
||||
```bash
|
||||
openclaw gateway run # or openclaw gateway restart
|
||||
openclaw message send "Hello, Lobster!"
|
||||
```
|
||||
|
||||
Visit the MLflow UI (e.g. http://localhost:5000) to see the trace.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/mlflow/mlflow/master/libs/typescript/integrations/openclaw/trace-screenshot.png" alt="MLflow UI" width="700" style="border-radius: 10px; box-shadow: 0 4px 16px rgba(0,0,0,0.18);" />
|
||||
</p>
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Tracking URI and experiment ID can also be set through environment variables:
|
||||
|
||||
```bash
|
||||
export MLFLOW_TRACKING_URI=http://localhost:5000
|
||||
export MLFLOW_EXPERIMENT_ID=<your-experiment-id>
|
||||
```
|
||||
|
||||
### Plugin Allowlist
|
||||
|
||||
OpenClaw shows a warning when a community plugin is installed but not declared in the [plugin allowlist](https://docs.openclaw.ai/tools/plugin#config). Add `mlflow-openclaw` to the plugin allowlist in your `openclaw.json` file to suppress the warning.
|
||||
|
||||
```
|
||||
{
|
||||
"plugins": {
|
||||
"allow": ["mlflow-openclaw"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## What Gets Traced
|
||||
|
||||
The plugin creates a span hierarchy for each agent session:
|
||||
|
||||
```
|
||||
AGENT (openclaw_agent) ← root span
|
||||
├── LLM (llm_call) ← each LLM interaction
|
||||
├── TOOL (tool_<name>) ← each tool invocation
|
||||
├── AGENT (subagent_<label>) ← sub-agent executions
|
||||
└── ...
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
# Type-check
|
||||
npm run typecheck
|
||||
|
||||
# Test
|
||||
npm test
|
||||
|
||||
# Format
|
||||
npm run format
|
||||
|
||||
# Lint
|
||||
npm run lint
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](https://github.com/mlflow/mlflow/blob/master/LICENSE.txt) file for details.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 160 KiB |
@@ -0,0 +1,31 @@
|
||||
const path = require('path');
|
||||
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
roots: ['<rootDir>/tests'],
|
||||
testMatch: ['**/*.test.ts'],
|
||||
modulePaths: [path.resolve(__dirname, '../../node_modules')],
|
||||
transform: {
|
||||
'^.+\\.tsx?$': [
|
||||
'ts-jest',
|
||||
{
|
||||
tsconfig: {
|
||||
target: 'ES2022',
|
||||
module: 'CommonJS',
|
||||
moduleResolution: 'Node',
|
||||
esModuleInterop: true,
|
||||
strict: true,
|
||||
skipLibCheck: true,
|
||||
types: ['jest', 'node'],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
// openclaw/plugin-sdk/* doesn't exist outside the gateway runtime —
|
||||
// map to a trivial empty module so service.ts can load.
|
||||
moduleNameMapper: {
|
||||
'^openclaw/plugin-sdk/.*$': '<rootDir>/tests/__mocks__/empty.ts',
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "mlflow-openclaw",
|
||||
"name": "MLflow Tracing",
|
||||
"description": "Export OpenClaw LLM traces to MLflow",
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": { "type": "boolean" },
|
||||
"trackingUri": { "type": "string" },
|
||||
"experimentId": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"uiHints": {
|
||||
"enabled": {
|
||||
"label": "Tracing Enabled",
|
||||
"help": "Enable MLflow trace export while the plugin is loaded."
|
||||
},
|
||||
"trackingUri": {
|
||||
"label": "MLflow Tracking URI",
|
||||
"placeholder": "http://localhost:5000",
|
||||
"help": "Defaults to MLFLOW_TRACKING_URI when not set."
|
||||
},
|
||||
"experimentId": {
|
||||
"label": "Experiment ID",
|
||||
"placeholder": "0",
|
||||
"help": "Defaults to MLFLOW_EXPERIMENT_ID when not set."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"name": "@mlflow/mlflow-openclaw",
|
||||
"version": "0.2.0",
|
||||
"description": "OpenClaw integration package for MLflow Tracing",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mlflow/mlflow.git"
|
||||
},
|
||||
"homepage": "https://mlflow.org/",
|
||||
"author": {
|
||||
"name": "MLflow",
|
||||
"url": "https://mlflow.org/"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/mlflow/mlflow/issues"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"keywords": [
|
||||
"mlflow",
|
||||
"tracing",
|
||||
"observability",
|
||||
"openclaw",
|
||||
"llm",
|
||||
"ai-agent",
|
||||
"javascript",
|
||||
"typescript"
|
||||
],
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"lint": "tsc --noEmit",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "jest",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check .",
|
||||
"prepublishOnly": "npm run build"
|
||||
},
|
||||
"dependencies": {
|
||||
"@clack/prompts": "^1.1.0",
|
||||
"@mlflow/core": ">=0.2.0 || >=0.3.0-rc.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"openclaw": ">=2026.4.9"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"openclaw": {
|
||||
"optional": true
|
||||
}
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.3",
|
||||
"jest": "^29.6.2",
|
||||
"ts-jest": "^29.1.1",
|
||||
"typescript": "^5.8.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./dist/index.js"
|
||||
]
|
||||
},
|
||||
"files": [
|
||||
"dist/",
|
||||
"openclaw.plugin.json",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
/**
|
||||
* CLI registration for `openclaw mlflow configure` and `openclaw mlflow status`.
|
||||
*/
|
||||
|
||||
import type { OpenClawConfig } from 'openclaw/plugin-sdk/plugin-entry';
|
||||
|
||||
const PLUGIN_ID = 'mlflow-openclaw';
|
||||
|
||||
type ConfigDeps = {
|
||||
loadConfig: () => OpenClawConfig;
|
||||
writeConfigFile: (cfg: OpenClawConfig) => Promise<void>;
|
||||
};
|
||||
|
||||
// Minimal structural type for the Commander.js `Command` object OpenClaw
|
||||
// hands to `api.registerCli`. Only covers the subset we actually use.
|
||||
export type CommanderLike = {
|
||||
command: (name: string) => CommandLike;
|
||||
};
|
||||
|
||||
type CommandLike = {
|
||||
description: (d: string) => CommandLike;
|
||||
command: (name: string) => CommandLike;
|
||||
action: (fn: () => void) => CommandLike;
|
||||
};
|
||||
|
||||
type RegisterCliParams = {
|
||||
program: CommanderLike;
|
||||
} & ConfigDeps;
|
||||
|
||||
type ClackPrompts = {
|
||||
intro: (msg: string) => void;
|
||||
outro: (msg: string) => void;
|
||||
cancel: (msg: string) => void;
|
||||
isCancel: (value: unknown) => boolean;
|
||||
text: (options: {
|
||||
message: string;
|
||||
placeholder?: string;
|
||||
initialValue?: string;
|
||||
validate?: (value: string) => string | void;
|
||||
}) => Promise<string | symbol>;
|
||||
};
|
||||
|
||||
function asObject(value: unknown): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
return {};
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function asString(value: unknown): string {
|
||||
return typeof value === 'string' ? value : '';
|
||||
}
|
||||
|
||||
function getPluginEntry(cfg: OpenClawConfig): {
|
||||
enabled?: boolean;
|
||||
config: Record<string, unknown>;
|
||||
} {
|
||||
const root = asObject(cfg);
|
||||
const plugins = asObject(root.plugins);
|
||||
const entries = asObject(plugins.entries);
|
||||
const entry = asObject(entries[PLUGIN_ID]);
|
||||
const config = asObject(entry.config);
|
||||
return {
|
||||
enabled: typeof entry.enabled === 'boolean' ? entry.enabled : undefined,
|
||||
config,
|
||||
};
|
||||
}
|
||||
|
||||
function setPluginEntry(
|
||||
cfg: OpenClawConfig,
|
||||
config: Record<string, unknown>,
|
||||
enabled = true,
|
||||
): OpenClawConfig {
|
||||
const root = asObject(cfg);
|
||||
const plugins = asObject(root.plugins);
|
||||
const entries = asObject(plugins.entries);
|
||||
entries[PLUGIN_ID] = { ...asObject(entries[PLUGIN_ID]), enabled, config };
|
||||
plugins.entries = entries;
|
||||
root.plugins = plugins;
|
||||
return root as OpenClawConfig;
|
||||
}
|
||||
|
||||
function showStatus(deps: ConfigDeps): void {
|
||||
const cfg = deps.loadConfig();
|
||||
const entry = getPluginEntry(cfg);
|
||||
const trackingUri =
|
||||
asString(entry.config.trackingUri) || process.env.MLFLOW_TRACKING_URI || '(not set)';
|
||||
const experimentId =
|
||||
asString(entry.config.experimentId) || process.env.MLFLOW_EXPERIMENT_ID || '(not set)';
|
||||
|
||||
const lines: string[] = [
|
||||
` Enabled: ${entry.enabled ?? 'not set'}`,
|
||||
` Tracking URI: ${trackingUri}`,
|
||||
` Experiment ID: ${experimentId}`,
|
||||
];
|
||||
|
||||
process.stdout.write(`MLflow status:\n\n${lines.join('\n')}\n`);
|
||||
}
|
||||
|
||||
async function runConfigure(deps: ConfigDeps): Promise<void> {
|
||||
// Dynamic import to avoid requiring @clack/prompts at module load time
|
||||
const p = (await import('@clack/prompts')) as unknown as ClackPrompts;
|
||||
|
||||
p.intro('MLflow Tracing configuration');
|
||||
|
||||
const cfg = deps.loadConfig();
|
||||
const entry = getPluginEntry(cfg);
|
||||
|
||||
const trackingUri = await p.text({
|
||||
message: 'MLflow Tracking URI',
|
||||
placeholder: 'http://localhost:5000',
|
||||
initialValue: asString(entry.config.trackingUri) || process.env.MLFLOW_TRACKING_URI || '',
|
||||
validate: (value: string) => {
|
||||
if (!value) {
|
||||
return 'Tracking URI is required';
|
||||
}
|
||||
try {
|
||||
new URL(value);
|
||||
} catch {
|
||||
return 'Invalid URL';
|
||||
}
|
||||
},
|
||||
});
|
||||
if (p.isCancel(trackingUri)) {
|
||||
p.cancel('Configuration cancelled');
|
||||
return;
|
||||
}
|
||||
|
||||
const experimentId = await p.text({
|
||||
message: 'Experiment ID',
|
||||
placeholder: '0',
|
||||
initialValue: asString(entry.config.experimentId) || process.env.MLFLOW_EXPERIMENT_ID || '',
|
||||
validate: (value: string) => {
|
||||
if (!value) {
|
||||
return 'Experiment ID is required';
|
||||
}
|
||||
},
|
||||
});
|
||||
if (p.isCancel(experimentId)) {
|
||||
p.cancel('Configuration cancelled');
|
||||
return;
|
||||
}
|
||||
|
||||
const updated = setPluginEntry(cfg, {
|
||||
...entry.config,
|
||||
trackingUri: trackingUri as string,
|
||||
experimentId: experimentId as string,
|
||||
});
|
||||
await deps.writeConfigFile(updated);
|
||||
|
||||
p.outro('MLflow configuration saved. Restart the gateway to apply.');
|
||||
}
|
||||
|
||||
export function registerMlflowCli(params: RegisterCliParams): void {
|
||||
const { program, loadConfig, writeConfigFile } = params;
|
||||
const deps: ConfigDeps = { loadConfig, writeConfigFile };
|
||||
|
||||
const root = program.command('mlflow').description('MLflow trace export integration');
|
||||
|
||||
root
|
||||
.command('configure')
|
||||
.description('Interactive setup for MLflow trace export')
|
||||
.action(() => {
|
||||
void runConfigure(deps);
|
||||
});
|
||||
|
||||
root
|
||||
.command('status')
|
||||
.description('Show current MLflow configuration')
|
||||
.action(() => {
|
||||
showStatus(deps);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { definePluginEntry, type OpenClawPluginApi } from 'openclaw/plugin-sdk/plugin-entry';
|
||||
import { createMLflowService } from './service.js';
|
||||
import { registerMlflowCli, type CommanderLike } from './configure.js';
|
||||
|
||||
function parsePluginConfig(raw: unknown): Record<string, unknown> {
|
||||
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
||||
return {};
|
||||
}
|
||||
return raw as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export default definePluginEntry({
|
||||
id: 'mlflow-openclaw',
|
||||
name: 'MLflow Tracing',
|
||||
description: 'Export OpenClaw LLM traces to MLflow',
|
||||
register(api: OpenClawPluginApi) {
|
||||
const pluginConfig = parsePluginConfig(api.pluginConfig);
|
||||
const service = createMLflowService(api, pluginConfig);
|
||||
service.registerHooks();
|
||||
api.registerService(service);
|
||||
api.registerCli(
|
||||
({ program }) => {
|
||||
registerMlflowCli({
|
||||
program: program as CommanderLike,
|
||||
loadConfig: api.runtime.config.loadConfig,
|
||||
writeConfigFile: api.runtime.config.writeConfigFile,
|
||||
});
|
||||
},
|
||||
{ commands: ['mlflow'] },
|
||||
);
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Type declarations for the OpenClaw plugin SDK subpath imports.
|
||||
*
|
||||
* These types represent the OpenClaw plugin-sdk interface that this integration
|
||||
* targets. Once OpenClaw publishes official TypeScript types, this file can
|
||||
* be removed in favor of the `openclaw` package's own type definitions.
|
||||
*/
|
||||
|
||||
declare module 'openclaw/plugin-sdk/plugin-entry' {
|
||||
export type OpenClawConfig = Record<string, unknown>;
|
||||
|
||||
export type OpenClawPluginConfigSchema = {
|
||||
type: string;
|
||||
properties?: Record<string, unknown>;
|
||||
additionalProperties?: boolean;
|
||||
};
|
||||
|
||||
export type OpenClawPluginService = {
|
||||
id: string;
|
||||
start: (ctx: {
|
||||
config: unknown;
|
||||
logger: {
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
};
|
||||
}) => void | Promise<void>;
|
||||
stop?: (ctx?: unknown) => void | Promise<void>;
|
||||
};
|
||||
|
||||
export type OpenClawPluginApi = {
|
||||
pluginConfig?: unknown;
|
||||
registerService: (service: OpenClawPluginService) => void;
|
||||
registerCli: (
|
||||
register: (params: { program: unknown }) => void,
|
||||
options?: { commands?: string[] },
|
||||
) => void;
|
||||
runtime: {
|
||||
config: {
|
||||
loadConfig: () => OpenClawConfig;
|
||||
writeConfigFile: (cfg: OpenClawConfig) => Promise<void>;
|
||||
};
|
||||
};
|
||||
on: (event: string, handler: (event: unknown, ctx: unknown) => void) => void;
|
||||
};
|
||||
|
||||
export function definePluginEntry(options: {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
configSchema?: OpenClawPluginConfigSchema | (() => OpenClawPluginConfigSchema);
|
||||
register: (api: OpenClawPluginApi) => void;
|
||||
}): unknown;
|
||||
|
||||
export function emptyPluginConfigSchema(): OpenClawPluginConfigSchema;
|
||||
}
|
||||
|
||||
declare module 'openclaw/plugin-sdk/diagnostics-otel' {
|
||||
export type DiagnosticEventPayload = {
|
||||
type: string;
|
||||
sessionKey?: string;
|
||||
costUsd?: number;
|
||||
context?: {
|
||||
limit?: number;
|
||||
used?: number;
|
||||
};
|
||||
model?: string;
|
||||
provider?: string;
|
||||
durationMs?: number;
|
||||
usage?: {
|
||||
input?: number;
|
||||
output?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
total?: number;
|
||||
};
|
||||
};
|
||||
|
||||
export function onDiagnosticEvent(handler: (event: DiagnosticEventPayload) => void): () => void;
|
||||
}
|
||||
@@ -0,0 +1,896 @@
|
||||
/**
|
||||
* MLflow Tracing service for OpenClaw.
|
||||
*
|
||||
* Creates an OpenClawPluginService that subscribes to agent lifecycle events
|
||||
* via api.on() and creates MLflow traces in real-time. Maps OpenClaw events
|
||||
* to a span hierarchy: root AGENT → child LLM / TOOL / sub-AGENT spans.
|
||||
*/
|
||||
|
||||
import type { OpenClawPluginApi, OpenClawPluginService } from 'openclaw/plugin-sdk/plugin-entry';
|
||||
import {
|
||||
onDiagnosticEvent,
|
||||
type DiagnosticEventPayload,
|
||||
} from 'openclaw/plugin-sdk/diagnostics-otel';
|
||||
import {
|
||||
init,
|
||||
startSpan,
|
||||
flushTraces,
|
||||
SpanStatusCode,
|
||||
type SpanType as SpanTypeEnum,
|
||||
} from '@mlflow/core';
|
||||
|
||||
// Inline constants that fail to import via OpenClaw's CJS/ESM loader
|
||||
// (they're re-exported via __exportStar which the loader can't resolve).
|
||||
// Values must match the SpanType enum members in @mlflow/core.
|
||||
const SpanType = {
|
||||
LLM: 'LLM' as SpanTypeEnum.LLM,
|
||||
TOOL: 'TOOL' as SpanTypeEnum.TOOL,
|
||||
AGENT: 'AGENT' as SpanTypeEnum.AGENT,
|
||||
};
|
||||
const SpanAttributeKey = {
|
||||
TOKEN_USAGE: 'mlflow.chat.tokenUsage',
|
||||
MESSAGE_FORMAT: 'mlflow.message.format',
|
||||
} as const;
|
||||
const TraceMetadataKey = {
|
||||
TRACE_SESSION: 'mlflow.trace.session',
|
||||
TRACE_USER: 'mlflow.trace.user',
|
||||
} as const;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MAX_ACTIVE_TRACES = 50;
|
||||
const DEFAULT_STALE_TRACE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
const DEFAULT_STALE_SWEEP_INTERVAL_MS = 60 * 1000; // 1 minute
|
||||
const DEFAULT_FLUSH_RETRY_COUNT = 2;
|
||||
const DEFAULT_FLUSH_RETRY_BASE_DELAY_MS = 250;
|
||||
const MAX_FLUSH_RETRY_DELAY_MS = 5000;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type SpanLike = ReturnType<typeof startSpan>;
|
||||
|
||||
export interface PendingChild {
|
||||
span: SpanLike;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface ActiveTrace {
|
||||
rootSpan: SpanLike;
|
||||
pendingLlm: PendingChild | null;
|
||||
pendingTools: Map<string, PendingChild>;
|
||||
pendingSubagents: Map<string, PendingChild>;
|
||||
tokenUsage: {
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
totalTokens: number;
|
||||
cost: number;
|
||||
};
|
||||
costMeta: {
|
||||
costUsd?: number;
|
||||
contextLimit?: number;
|
||||
contextUsed?: number;
|
||||
model?: string;
|
||||
provider?: string;
|
||||
};
|
||||
firstPrompt: string;
|
||||
lastResponse: string;
|
||||
agentEndData: {
|
||||
success?: boolean;
|
||||
error?: string;
|
||||
durationMs?: number;
|
||||
messages?: unknown[];
|
||||
} | null;
|
||||
channelId?: string;
|
||||
trigger?: string;
|
||||
model?: string;
|
||||
provider?: string;
|
||||
lastActivityMs: number;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function evictOldest<K, V>(map: Map<K, V>, maxSize: number): void {
|
||||
while (map.size > maxSize) {
|
||||
const { value: oldest, done } = map.keys().next();
|
||||
if (done) {
|
||||
return;
|
||||
}
|
||||
map.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
export function toolKey(toolName: string, toolCallId?: string): string {
|
||||
return toolCallId ? `${toolName}:${toolCallId}` : toolName;
|
||||
}
|
||||
|
||||
export function normalizeProvider(value: unknown): string | undefined {
|
||||
if (typeof value !== 'string' || value.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
normalized === 'openai-codex' ||
|
||||
normalized === 'openai_codex' ||
|
||||
normalized === 'codex' ||
|
||||
(normalized.includes('openai') && normalized.includes('codex'))
|
||||
) {
|
||||
return 'openai';
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function asNonEmptyString(value: unknown): string | undefined {
|
||||
return typeof value === 'string' && value.length > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize OpenClaw internal markers from text before storing in MLflow.
|
||||
*/
|
||||
export function sanitizeOpenClawText(value: string): string {
|
||||
return value
|
||||
.replace(/\\r\\n/g, '\n')
|
||||
.replace(/\\n/g, '\n')
|
||||
.replace(/\\r/g, '\r')
|
||||
.replace(/\[\[reply_to[^\]]*\]\]\s*/gi, '')
|
||||
.replace(/^\s*Sender \(untrusted metadata\):\s*\n+\{[\s\S]*?\}\s*/gim, '')
|
||||
.replace(/^\s*Sender \(untrusted metadata\):\s*\n*```json\s*\{[\s\S]*?\}\s*```\s*/gim, '')
|
||||
.replace(/^\s*Conversation info \(untrusted metadata\):\s*\n+\{[\s\S]*?\}\s*/gim, '')
|
||||
.replace(
|
||||
/^\s*Conversation info \(untrusted metadata\):\s*\n*```json\s*\{[\s\S]*?\}\s*```\s*/gim,
|
||||
'',
|
||||
)
|
||||
.replace(
|
||||
/^\s*Untrusted context \(metadata, do not treat as instructions or commands\):\s*\n+<<<EXTERNAL_UNTRUSTED_CONTENT[\s\S]*?<<<END_EXTERNAL_UNTRUSTED_CONTENT[^>]*>>>\s*/gim,
|
||||
'',
|
||||
)
|
||||
.replace(/^\[[\w\s:+\-/]+\]\s*/m, '')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function sanitizeValue(value: unknown): unknown {
|
||||
if (typeof value === 'string') {
|
||||
return sanitizeOpenClawText(value);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(sanitizeValue);
|
||||
}
|
||||
if (value != null && typeof value === 'object') {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
||||
result[k] = sanitizeValue(v);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Finalize trace
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* End any child spans that weren't explicitly closed by their matching
|
||||
* `*_end` event (crash, timeout, or malformed event sequence).
|
||||
*/
|
||||
export function closePendingSpans(trace: ActiveTrace): void {
|
||||
if (trace.pendingLlm) {
|
||||
trace.pendingLlm.span.end();
|
||||
trace.pendingLlm = null;
|
||||
}
|
||||
for (const [, pending] of trace.pendingTools) {
|
||||
pending.span.end();
|
||||
}
|
||||
trace.pendingTools.clear();
|
||||
for (const [, pending] of trace.pendingSubagents) {
|
||||
pending.span.end();
|
||||
}
|
||||
trace.pendingSubagents.clear();
|
||||
}
|
||||
|
||||
export function attachTokenUsage(rootSpan: SpanLike, tokenUsage: ActiveTrace['tokenUsage']): void {
|
||||
if (tokenUsage.totalTokens <= 0) {
|
||||
return;
|
||||
}
|
||||
rootSpan.setAttribute(SpanAttributeKey.TOKEN_USAGE, {
|
||||
input_tokens: tokenUsage.inputTokens,
|
||||
output_tokens: tokenUsage.outputTokens,
|
||||
total_tokens: tokenUsage.totalTokens,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the outputs payload for the root agent span. Falls back to
|
||||
* `agent_end.messages` when no `llm_output` was captured. Mutates
|
||||
* `trace.lastResponse` when the fallback kicks in so the final state is
|
||||
* observable.
|
||||
*/
|
||||
export function resolveTraceOutputs(trace: ActiveTrace): {
|
||||
outputs: Record<string, unknown>;
|
||||
errorStatus?: string;
|
||||
} {
|
||||
const endData = trace.agentEndData;
|
||||
if (!trace.lastResponse && endData?.messages?.length) {
|
||||
trace.lastResponse = JSON.stringify(endData.messages);
|
||||
}
|
||||
|
||||
const responseText = trace.lastResponse || 'Agent completed';
|
||||
const outputs: Record<string, unknown> = {
|
||||
messages: [{ role: 'assistant', content: responseText }],
|
||||
};
|
||||
if (endData?.error) {
|
||||
outputs.error = endData.error;
|
||||
return { outputs, errorStatus: endData.error };
|
||||
}
|
||||
return { outputs };
|
||||
}
|
||||
|
||||
export function attachTraceMetadata(rootSpan: SpanLike, trace: ActiveTrace): void {
|
||||
if (trace.agentEndData?.durationMs != null) {
|
||||
rootSpan.setAttribute('agent_duration_ms', trace.agentEndData.durationMs);
|
||||
}
|
||||
|
||||
const traceModel = trace.model ?? trace.costMeta.model;
|
||||
const traceProvider = trace.provider ?? trace.costMeta.provider;
|
||||
if (traceModel) {
|
||||
rootSpan.setAttribute('mlflow.llm.model', traceModel);
|
||||
}
|
||||
if (traceProvider) {
|
||||
rootSpan.setAttribute('mlflow.llm.provider', traceProvider);
|
||||
}
|
||||
|
||||
if (trace.costMeta.costUsd != null) {
|
||||
rootSpan.setAttribute('mlflow.llm.cost', { total_cost: trace.costMeta.costUsd });
|
||||
}
|
||||
}
|
||||
|
||||
async function finalizeTrace(
|
||||
sessionKey: string,
|
||||
trace: ActiveTrace,
|
||||
userId?: string,
|
||||
flush: () => Promise<void> = () => flushTraces(),
|
||||
): Promise<void> {
|
||||
closePendingSpans(trace);
|
||||
attachTokenUsage(trace.rootSpan, trace.tokenUsage);
|
||||
|
||||
const { outputs, errorStatus } = resolveTraceOutputs(trace);
|
||||
if (errorStatus) {
|
||||
trace.rootSpan.setStatus(SpanStatusCode.ERROR, errorStatus);
|
||||
}
|
||||
trace.rootSpan.setOutputs(outputs);
|
||||
|
||||
attachTraceMetadata(trace.rootSpan, trace);
|
||||
|
||||
trace.rootSpan.end();
|
||||
await flush();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Service factory
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function createMLflowService(
|
||||
api: OpenClawPluginApi,
|
||||
pluginConfig: Record<string, unknown> = {},
|
||||
): OpenClawPluginService & { registerHooks: () => void } {
|
||||
const activeTraces = new Map<string, ActiveTrace>();
|
||||
const sessionByAgentId = new Map<string, string>();
|
||||
let lastActiveSessionKey: string | undefined;
|
||||
let warnedMissingAfterToolSessionKey = false;
|
||||
let cleanup: (() => void) | null = null;
|
||||
let hooksRegistered = false;
|
||||
let initError: unknown = null;
|
||||
let resolvedTrackingUri: string | undefined;
|
||||
let resolvedExperimentId: string | undefined;
|
||||
let log: { info: (msg: string) => void; warn: (msg: string) => void } = {
|
||||
info: () => undefined,
|
||||
warn: () => undefined,
|
||||
};
|
||||
|
||||
// Exporter metrics
|
||||
const metrics = {
|
||||
flushSuccesses: 0,
|
||||
flushFailures: 0,
|
||||
flushRetries: 0,
|
||||
spanErrors: 0,
|
||||
};
|
||||
|
||||
function rememberSession(sessionKey: string, agentId?: unknown): void {
|
||||
lastActiveSessionKey = sessionKey;
|
||||
if (typeof agentId === 'string' && agentId.length > 0) {
|
||||
sessionByAgentId.set(agentId, sessionKey);
|
||||
}
|
||||
}
|
||||
|
||||
function forgetSession(sessionKey: string): void {
|
||||
if (lastActiveSessionKey === sessionKey) {
|
||||
lastActiveSessionKey = undefined;
|
||||
}
|
||||
for (const [agentId, mapped] of sessionByAgentId) {
|
||||
if (mapped === sessionKey) {
|
||||
sessionByAgentId.delete(agentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resolveAfterToolSessionKey(ctx: Record<string, unknown>): string | undefined {
|
||||
// Primary: from context
|
||||
const direct = asNonEmptyString(ctx.sessionKey);
|
||||
if (direct && activeTraces.has(direct)) {
|
||||
return direct;
|
||||
}
|
||||
|
||||
// Fallback 1: agentId → sessionKey map
|
||||
const agentId = asNonEmptyString(ctx.agentId);
|
||||
if (agentId) {
|
||||
const byAgent = sessionByAgentId.get(agentId);
|
||||
if (byAgent && activeTraces.has(byAgent)) {
|
||||
if (!warnedMissingAfterToolSessionKey) {
|
||||
warnedMissingAfterToolSessionKey = true;
|
||||
log.warn('mlflow: after_tool_call missing sessionKey; using agentId fallback');
|
||||
}
|
||||
return byAgent;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback 2: single active trace
|
||||
if (activeTraces.size === 1) {
|
||||
if (!warnedMissingAfterToolSessionKey) {
|
||||
warnedMissingAfterToolSessionKey = true;
|
||||
log.warn('mlflow: after_tool_call missing sessionKey; using single-active-trace fallback');
|
||||
}
|
||||
return activeTraces.keys().next().value;
|
||||
}
|
||||
|
||||
// Fallback 3: last active session
|
||||
if (lastActiveSessionKey && activeTraces.has(lastActiveSessionKey)) {
|
||||
if (!warnedMissingAfterToolSessionKey) {
|
||||
warnedMissingAfterToolSessionKey = true;
|
||||
log.warn('mlflow: after_tool_call missing sessionKey; using last-active-session fallback');
|
||||
}
|
||||
return lastActiveSessionKey;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function getOrCreateTrace(sessionKey: string, prompt: string): ActiveTrace {
|
||||
const existing = activeTraces.get(sessionKey);
|
||||
if (existing) {
|
||||
activeTraces.delete(sessionKey);
|
||||
activeTraces.set(sessionKey, existing);
|
||||
existing.lastActivityMs = Date.now();
|
||||
return existing;
|
||||
}
|
||||
|
||||
const cleanPrompt = sanitizeOpenClawText(prompt);
|
||||
|
||||
const rootSpan = startSpan({
|
||||
name: 'openclaw_agent',
|
||||
inputs: {
|
||||
messages: [{ role: 'user', content: cleanPrompt }],
|
||||
},
|
||||
spanType: SpanType.AGENT,
|
||||
});
|
||||
rootSpan.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'openai');
|
||||
rootSpan.setAttribute(TraceMetadataKey.TRACE_SESSION, sessionKey);
|
||||
rootSpan.setAttribute(TraceMetadataKey.TRACE_USER, process.env.USER || '');
|
||||
|
||||
const trace: ActiveTrace = {
|
||||
rootSpan,
|
||||
pendingLlm: null,
|
||||
pendingTools: new Map(),
|
||||
pendingSubagents: new Map(),
|
||||
tokenUsage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, cost: 0 },
|
||||
costMeta: {},
|
||||
firstPrompt: cleanPrompt,
|
||||
lastResponse: '',
|
||||
agentEndData: null,
|
||||
lastActivityMs: Date.now(),
|
||||
};
|
||||
|
||||
activeTraces.set(sessionKey, trace);
|
||||
evictOldest(activeTraces, MAX_ACTIVE_TRACES);
|
||||
|
||||
return trace;
|
||||
}
|
||||
|
||||
// Flush with exponential backoff retry
|
||||
async function flushWithRetry(reason: string): Promise<void> {
|
||||
const attempts = DEFAULT_FLUSH_RETRY_COUNT + 1;
|
||||
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
await flushTraces();
|
||||
metrics.flushSuccesses += 1;
|
||||
return;
|
||||
} catch (err) {
|
||||
metrics.flushFailures += 1;
|
||||
log.warn(`mlflow: flush failed (${reason}) attempt ${attempt}/${attempts}: ${String(err)}`);
|
||||
if (attempt >= attempts) {
|
||||
return;
|
||||
}
|
||||
metrics.flushRetries += 1;
|
||||
const delayMs = Math.min(
|
||||
DEFAULT_FLUSH_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1),
|
||||
MAX_FLUSH_RETRY_DELAY_MS,
|
||||
);
|
||||
await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =====================================================================
|
||||
// registerHooks — MUST be called during register(), not start().
|
||||
// OpenClaw only accepts api.on() subscriptions during the register phase.
|
||||
// Hooks guard on `initialized` so events before SDK init are silently skipped.
|
||||
// =====================================================================
|
||||
function registerHooks(): void {
|
||||
if (hooksRegistered) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (pluginConfig.enabled === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
const trackingUri =
|
||||
(typeof pluginConfig.trackingUri === 'string' ? pluginConfig.trackingUri : '') ||
|
||||
process.env.MLFLOW_TRACKING_URI;
|
||||
const experimentId =
|
||||
(typeof pluginConfig.experimentId === 'string' ? pluginConfig.experimentId : '') ||
|
||||
process.env.MLFLOW_EXPERIMENT_ID;
|
||||
|
||||
if (!trackingUri || !experimentId) {
|
||||
return;
|
||||
}
|
||||
|
||||
initError = null;
|
||||
try {
|
||||
init({ trackingUri, experimentId });
|
||||
} catch (err) {
|
||||
initError = err;
|
||||
log.warn(`mlflow: init failed: ${String(err)}`);
|
||||
}
|
||||
|
||||
hooksRegistered = true;
|
||||
resolvedTrackingUri = trackingUri;
|
||||
resolvedExperimentId = experimentId;
|
||||
|
||||
api.on('llm_input', (event: unknown, agentCtx: unknown) => {
|
||||
const ctx = agentCtx as Record<string, unknown>;
|
||||
const evt = event as Record<string, unknown>;
|
||||
const sessionKey = ctx.sessionKey as string | undefined;
|
||||
if (!sessionKey) {
|
||||
return;
|
||||
}
|
||||
rememberSession(sessionKey, ctx.agentId);
|
||||
|
||||
const prompt = (evt.prompt as string) ?? '';
|
||||
const historyMessages = evt.historyMessages as unknown[] | undefined;
|
||||
const trace = getOrCreateTrace(sessionKey, prompt);
|
||||
|
||||
const channelId = asNonEmptyString(ctx.channelId) ?? asNonEmptyString(ctx.messageProvider);
|
||||
if (channelId) {
|
||||
trace.channelId = channelId;
|
||||
}
|
||||
const trigger = asNonEmptyString(ctx.trigger);
|
||||
if (trigger) {
|
||||
trace.trigger = trigger;
|
||||
}
|
||||
|
||||
if (trace.pendingLlm) {
|
||||
trace.pendingLlm.span.end();
|
||||
}
|
||||
|
||||
const rawProvider = evt.provider as string | undefined;
|
||||
const provider = normalizeProvider(rawProvider) ?? rawProvider;
|
||||
const model = evt.model as string | undefined;
|
||||
const modelLabel = provider && model ? `${provider}/${model}` : model || 'unknown';
|
||||
|
||||
if (model) {
|
||||
trace.model = model;
|
||||
}
|
||||
if (provider) {
|
||||
trace.provider = provider;
|
||||
}
|
||||
|
||||
const messages: { role: string; content: string }[] = [];
|
||||
if (evt.systemPrompt) {
|
||||
messages.push({ role: 'system', content: evt.systemPrompt as string });
|
||||
}
|
||||
if (historyMessages?.length) {
|
||||
for (const msg of historyMessages) {
|
||||
const m = msg as { role?: string; content?: unknown };
|
||||
if (!m.role) {
|
||||
continue;
|
||||
}
|
||||
const role = m.role === 'toolResult' ? 'tool' : m.role;
|
||||
const content =
|
||||
typeof m.content === 'string'
|
||||
? m.content
|
||||
: Array.isArray(m.content)
|
||||
? m.content.map((p: { text?: string }) => p.text ?? '').join('\n')
|
||||
: m.content != null
|
||||
? JSON.stringify(m.content)
|
||||
: '';
|
||||
messages.push({ role, content });
|
||||
}
|
||||
}
|
||||
messages.push({ role: 'user', content: prompt });
|
||||
const llmInputs = sanitizeValue({
|
||||
messages,
|
||||
model: modelLabel,
|
||||
}) as Record<string, unknown>;
|
||||
|
||||
const llmSpan = startSpan({
|
||||
name: 'llm_call',
|
||||
parent: trace.rootSpan,
|
||||
spanType: SpanType.LLM,
|
||||
inputs: llmInputs,
|
||||
attributes: {
|
||||
...(model ? { 'mlflow.llm.model': model } : {}),
|
||||
...(provider ? { 'mlflow.llm.provider': provider } : {}),
|
||||
},
|
||||
});
|
||||
llmSpan.setAttribute(SpanAttributeKey.MESSAGE_FORMAT, 'openai');
|
||||
|
||||
trace.pendingLlm = { span: llmSpan, name: 'llm_call' };
|
||||
});
|
||||
|
||||
api.on('llm_output', (event: unknown, agentCtx: unknown) => {
|
||||
const ctx = agentCtx as Record<string, unknown>;
|
||||
const evt = event as Record<string, unknown>;
|
||||
const sessionKey = ctx.sessionKey as string | undefined;
|
||||
if (!sessionKey) {
|
||||
return;
|
||||
}
|
||||
rememberSession(sessionKey, ctx.agentId);
|
||||
|
||||
const trace = activeTraces.get(sessionKey);
|
||||
if (!trace) {
|
||||
return;
|
||||
}
|
||||
|
||||
trace.lastActivityMs = Date.now();
|
||||
const assistantTexts = (evt.assistantTexts as string[] | undefined) ?? [];
|
||||
const lastAssistant = evt.lastAssistant as Record<string, unknown> | undefined;
|
||||
const rawResponse =
|
||||
assistantTexts.length > 0 ? assistantTexts.join('\n') : (evt.response as string) || '';
|
||||
const response = sanitizeOpenClawText(rawResponse);
|
||||
trace.lastResponse = response;
|
||||
|
||||
const rawProvider = evt.provider as string | undefined;
|
||||
if (rawProvider) {
|
||||
trace.provider = normalizeProvider(rawProvider) ?? rawProvider;
|
||||
}
|
||||
if (evt.model) {
|
||||
trace.model = evt.model as string;
|
||||
}
|
||||
|
||||
if (trace.pendingLlm) {
|
||||
type UsageLike = {
|
||||
input?: number;
|
||||
output?: number;
|
||||
total?: number;
|
||||
totalTokens?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
};
|
||||
const evtUsage = evt.usage as UsageLike | undefined;
|
||||
const assistantUsage = lastAssistant?.usage as UsageLike | undefined;
|
||||
const usage = evtUsage ?? assistantUsage;
|
||||
if (usage && (usage.input || usage.output || usage.total || usage.totalTokens)) {
|
||||
const inputTokens = usage.input ?? 0;
|
||||
const outputTokens = usage.output ?? 0;
|
||||
const cacheRead = usage.cacheRead ?? 0;
|
||||
const cacheWrite = usage.cacheWrite ?? 0;
|
||||
trace.pendingLlm.span.setAttribute(SpanAttributeKey.TOKEN_USAGE, {
|
||||
input_tokens: inputTokens,
|
||||
output_tokens: outputTokens,
|
||||
total_tokens: inputTokens + outputTokens + cacheRead + cacheWrite,
|
||||
...(cacheRead ? { cache_read_input_tokens: cacheRead } : {}),
|
||||
...(cacheWrite ? { cache_creation_input_tokens: cacheWrite } : {}),
|
||||
});
|
||||
}
|
||||
trace.pendingLlm.span.setOutputs({
|
||||
choices: [{ message: { role: 'assistant', content: response } }],
|
||||
});
|
||||
trace.pendingLlm.span.end();
|
||||
trace.pendingLlm = null;
|
||||
}
|
||||
});
|
||||
|
||||
api.on('before_tool_call', (event: unknown, agentCtx: unknown) => {
|
||||
const ctx = agentCtx as Record<string, unknown>;
|
||||
const evt = event as Record<string, unknown>;
|
||||
const sessionKey = ctx.sessionKey as string | undefined;
|
||||
if (!sessionKey) {
|
||||
return;
|
||||
}
|
||||
rememberSession(sessionKey, ctx.agentId);
|
||||
|
||||
const trace = activeTraces.get(sessionKey);
|
||||
if (!trace) {
|
||||
return;
|
||||
}
|
||||
|
||||
trace.lastActivityMs = Date.now();
|
||||
const toolName = evt.toolName as string;
|
||||
const toolCallId = evt.toolCallId as string | undefined;
|
||||
const key = toolKey(toolName, toolCallId);
|
||||
|
||||
const toolSpan = startSpan({
|
||||
name: toolName,
|
||||
parent: trace.rootSpan,
|
||||
spanType: SpanType.TOOL,
|
||||
inputs: sanitizeValue((evt.params as Record<string, unknown>) || {}) as Record<
|
||||
string,
|
||||
unknown
|
||||
>,
|
||||
attributes: {
|
||||
tool_name: toolName,
|
||||
...(toolCallId ? { tool_id: toolCallId } : {}),
|
||||
},
|
||||
});
|
||||
|
||||
trace.pendingTools.set(key, { span: toolSpan, name: toolName });
|
||||
});
|
||||
|
||||
api.on('after_tool_call', (event: unknown, agentCtx: unknown) => {
|
||||
const ctx = agentCtx as Record<string, unknown>;
|
||||
const evt = event as Record<string, unknown>;
|
||||
|
||||
const sessionKey = resolveAfterToolSessionKey(ctx);
|
||||
if (!sessionKey) {
|
||||
return;
|
||||
}
|
||||
rememberSession(sessionKey, ctx.agentId);
|
||||
|
||||
const trace = activeTraces.get(sessionKey);
|
||||
if (!trace) {
|
||||
return;
|
||||
}
|
||||
|
||||
trace.lastActivityMs = Date.now();
|
||||
const toolName = evt.toolName as string;
|
||||
const toolCallId = evt.toolCallId as string | undefined;
|
||||
const key = toolKey(toolName, toolCallId);
|
||||
const pending = trace.pendingTools.get(key);
|
||||
|
||||
if (pending) {
|
||||
if (evt.error) {
|
||||
pending.span.setOutputs({ error: evt.error });
|
||||
} else {
|
||||
pending.span.setOutputs({
|
||||
result: (evt.result as string) || '',
|
||||
});
|
||||
}
|
||||
pending.span.end();
|
||||
trace.pendingTools.delete(key);
|
||||
}
|
||||
});
|
||||
|
||||
api.on('subagent_spawning', (event: unknown, agentCtx: unknown) => {
|
||||
const ctx = agentCtx as Record<string, unknown>;
|
||||
const evt = event as Record<string, unknown>;
|
||||
const sessionKey = ctx.sessionKey as string | undefined;
|
||||
if (!sessionKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
const trace = activeTraces.get(sessionKey);
|
||||
if (!trace) {
|
||||
return;
|
||||
}
|
||||
|
||||
trace.lastActivityMs = Date.now();
|
||||
const agentId = evt.agentId as string;
|
||||
const label = evt.label as string | undefined;
|
||||
|
||||
const subSpan = startSpan({
|
||||
name: `subagent_${label || agentId}`,
|
||||
parent: trace.rootSpan,
|
||||
spanType: SpanType.AGENT,
|
||||
inputs: {
|
||||
agent_id: agentId,
|
||||
...(label ? { label } : {}),
|
||||
},
|
||||
});
|
||||
|
||||
trace.pendingSubagents.set(agentId, { span: subSpan, name: agentId });
|
||||
});
|
||||
|
||||
api.on('subagent_ended', (event: unknown, agentCtx: unknown) => {
|
||||
const ctx = agentCtx as Record<string, unknown>;
|
||||
const evt = event as Record<string, unknown>;
|
||||
const sessionKey = ctx.sessionKey as string | undefined;
|
||||
if (!sessionKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
const trace = activeTraces.get(sessionKey);
|
||||
if (!trace) {
|
||||
return;
|
||||
}
|
||||
|
||||
trace.lastActivityMs = Date.now();
|
||||
const agentId = evt.agentId as string;
|
||||
const pending = trace.pendingSubagents.get(agentId);
|
||||
|
||||
if (pending) {
|
||||
if (evt.error) {
|
||||
pending.span.setOutputs({ error: evt.error });
|
||||
} else {
|
||||
pending.span.setOutputs({
|
||||
result: (evt.result as string) || '',
|
||||
});
|
||||
}
|
||||
pending.span.end();
|
||||
trace.pendingSubagents.delete(agentId);
|
||||
}
|
||||
});
|
||||
|
||||
api.on('agent_end', (event: unknown, agentCtx: unknown) => {
|
||||
const ctx = agentCtx as Record<string, unknown>;
|
||||
const evt = event as Record<string, unknown>;
|
||||
const sessionKey = ctx.sessionKey as string | undefined;
|
||||
if (!sessionKey) {
|
||||
return;
|
||||
}
|
||||
rememberSession(sessionKey, ctx.agentId);
|
||||
|
||||
const trace = activeTraces.get(sessionKey);
|
||||
if (!trace) {
|
||||
return;
|
||||
}
|
||||
|
||||
const channelId = asNonEmptyString(ctx.channelId) ?? asNonEmptyString(ctx.messageProvider);
|
||||
if (channelId && !trace.channelId) {
|
||||
trace.channelId = channelId;
|
||||
}
|
||||
const trigger = asNonEmptyString(ctx.trigger);
|
||||
if (trigger && !trace.trigger) {
|
||||
trace.trigger = trigger;
|
||||
}
|
||||
|
||||
trace.agentEndData = {
|
||||
success: evt.success as boolean | undefined,
|
||||
error: evt.error as string | undefined,
|
||||
durationMs: evt.durationMs as number | undefined,
|
||||
messages: evt.messages as unknown[] | undefined,
|
||||
};
|
||||
|
||||
const userId = ctx.userId as string | undefined;
|
||||
|
||||
queueMicrotask(() => {
|
||||
void (async () => {
|
||||
try {
|
||||
const t = activeTraces.get(sessionKey);
|
||||
if (t) {
|
||||
activeTraces.delete(sessionKey);
|
||||
forgetSession(sessionKey);
|
||||
await finalizeTrace(sessionKey, t, userId, () => flushWithRetry('agent-end'));
|
||||
}
|
||||
} catch {
|
||||
// Silently ignore finalization errors
|
||||
}
|
||||
})();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
id: 'mlflow-tracing',
|
||||
registerHooks,
|
||||
|
||||
start(ctx) {
|
||||
log = { info: ctx.logger.info.bind(ctx.logger), warn: ctx.logger.warn.bind(ctx.logger) };
|
||||
registerHooks();
|
||||
|
||||
if (!hooksRegistered) {
|
||||
if (initError) {
|
||||
ctx.logger.warn(`mlflow: tracing is disabled (init failed: ${String(initError)})`);
|
||||
} else {
|
||||
ctx.logger.warn(
|
||||
'mlflow: tracing is disabled (missing trackingUri/experimentId or explicitly disabled)',
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
ctx.logger.info(
|
||||
`mlflow: exporting traces to ${resolvedTrackingUri} (experiment=${resolvedExperimentId})`,
|
||||
);
|
||||
|
||||
const unsubDiagnostics = onDiagnosticEvent((evt: DiagnosticEventPayload) => {
|
||||
if (evt.type !== 'model.usage') {
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionKey = evt.sessionKey;
|
||||
if (!sessionKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
const trace = activeTraces.get(sessionKey);
|
||||
if (!trace) {
|
||||
return;
|
||||
}
|
||||
|
||||
trace.lastActivityMs = Date.now();
|
||||
if (evt.usage) {
|
||||
trace.tokenUsage.inputTokens += evt.usage.input || 0;
|
||||
trace.tokenUsage.outputTokens += evt.usage.output || 0;
|
||||
trace.tokenUsage.totalTokens += evt.usage.total || 0;
|
||||
}
|
||||
if (evt.costUsd != null) {
|
||||
trace.costMeta.costUsd = (trace.costMeta.costUsd ?? 0) + evt.costUsd;
|
||||
}
|
||||
if (evt.context?.limit != null) {
|
||||
trace.costMeta.contextLimit = evt.context.limit;
|
||||
}
|
||||
if (evt.context?.used != null) {
|
||||
trace.costMeta.contextUsed = evt.context.used;
|
||||
}
|
||||
if (evt.model) {
|
||||
trace.costMeta.model = evt.model;
|
||||
}
|
||||
if (evt.provider) {
|
||||
trace.costMeta.provider = normalizeProvider(evt.provider) ?? evt.provider;
|
||||
}
|
||||
});
|
||||
|
||||
const sweepInterval = setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [key, trace] of activeTraces) {
|
||||
if (now - trace.lastActivityMs > DEFAULT_STALE_TRACE_TIMEOUT_MS) {
|
||||
log.warn(`mlflow: force-closing stale trace sessionKey=${key}`);
|
||||
activeTraces.delete(key);
|
||||
forgetSession(key);
|
||||
trace.rootSpan.setStatus(SpanStatusCode.ERROR, 'Trace exceeded inactivity timeout');
|
||||
finalizeTrace(key, trace, undefined, () => flushWithRetry('stale-sweep')).catch(
|
||||
() => undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
}, DEFAULT_STALE_SWEEP_INTERVAL_MS);
|
||||
|
||||
cleanup = () => {
|
||||
unsubDiagnostics();
|
||||
clearInterval(sweepInterval);
|
||||
};
|
||||
},
|
||||
|
||||
async stop() {
|
||||
cleanup?.();
|
||||
cleanup = null;
|
||||
|
||||
for (const [sessionKey, trace] of activeTraces) {
|
||||
try {
|
||||
await finalizeTrace(sessionKey, trace, undefined, () => flushWithRetry('shutdown'));
|
||||
} catch (err) {
|
||||
log.warn(`mlflow: error finalizing trace ${sessionKey} during shutdown: ${String(err)}`);
|
||||
}
|
||||
}
|
||||
activeTraces.clear();
|
||||
sessionByAgentId.clear();
|
||||
lastActiveSessionKey = undefined;
|
||||
|
||||
log.info(
|
||||
`mlflow: exporter metrics flushSuccesses=${metrics.flushSuccesses} flushFailures=${metrics.flushFailures} flushRetries=${metrics.flushRetries} spanErrors=${metrics.spanErrors}`,
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
// Stub for openclaw/plugin-sdk/* which only exists inside the gateway runtime.
|
||||
export {};
|
||||
@@ -0,0 +1,407 @@
|
||||
import {
|
||||
sanitizeOpenClawText,
|
||||
sanitizeValue,
|
||||
normalizeProvider,
|
||||
toolKey,
|
||||
evictOldest,
|
||||
asNonEmptyString,
|
||||
closePendingSpans,
|
||||
attachTokenUsage,
|
||||
resolveTraceOutputs,
|
||||
attachTraceMetadata,
|
||||
type ActiveTrace,
|
||||
type PendingChild,
|
||||
} from '../src/service';
|
||||
|
||||
type MockSpan = {
|
||||
setAttribute: jest.Mock;
|
||||
setOutputs: jest.Mock;
|
||||
setStatus: jest.Mock;
|
||||
end: jest.Mock;
|
||||
};
|
||||
|
||||
function makeMockSpan(): MockSpan {
|
||||
return {
|
||||
setAttribute: jest.fn(),
|
||||
setOutputs: jest.fn(),
|
||||
setStatus: jest.fn(),
|
||||
end: jest.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function makePending(name = 'child'): PendingChild & { span: MockSpan } {
|
||||
return { span: makeMockSpan(), name } as unknown as PendingChild & { span: MockSpan };
|
||||
}
|
||||
|
||||
function makeActiveTrace(
|
||||
overrides: Partial<ActiveTrace> = {},
|
||||
): ActiveTrace & { rootSpan: MockSpan } {
|
||||
const rootSpan = makeMockSpan();
|
||||
const base = {
|
||||
rootSpan,
|
||||
pendingLlm: null,
|
||||
pendingTools: new Map(),
|
||||
pendingSubagents: new Map(),
|
||||
tokenUsage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, cost: 0 },
|
||||
costMeta: {},
|
||||
firstPrompt: '',
|
||||
lastResponse: '',
|
||||
agentEndData: null,
|
||||
lastActivityMs: 0,
|
||||
...overrides,
|
||||
};
|
||||
return base as unknown as ActiveTrace & { rootSpan: MockSpan };
|
||||
}
|
||||
|
||||
describe('sanitizeOpenClawText', () => {
|
||||
it('strips [[reply_to_current]]', () => {
|
||||
expect(sanitizeOpenClawText('[[reply_to_current]] Hello there!')).toBe('Hello there!');
|
||||
});
|
||||
|
||||
it('strips sender metadata (plain JSON)', () => {
|
||||
const input =
|
||||
'Sender (untrusted metadata):\n{"label": "tui", "id": "gw"}\n\n[Mon 2026-03-18 10:00 GMT+9] Hi there';
|
||||
expect(sanitizeOpenClawText(input)).toBe('Hi there');
|
||||
});
|
||||
|
||||
it('strips sender metadata (fenced JSON)', () => {
|
||||
const input =
|
||||
'Sender (untrusted metadata):\n```json\n{"label": "openclaw-tui", "id": "gateway-client"}\n```\n\n[Wed 2026-03-18 03:42 GMT+9] Hello';
|
||||
expect(sanitizeOpenClawText(input)).toBe('Hello');
|
||||
});
|
||||
|
||||
it('strips conversation info metadata (plain JSON)', () => {
|
||||
const input =
|
||||
'Conversation info (untrusted metadata):\n{"channel": "discord"}\n\nActual message';
|
||||
expect(sanitizeOpenClawText(input)).toBe('Actual message');
|
||||
});
|
||||
|
||||
it('strips conversation info metadata (fenced JSON)', () => {
|
||||
const input =
|
||||
'Conversation info (untrusted metadata):\n```json\n{"channel": "discord"}\n```\n\nActual message';
|
||||
expect(sanitizeOpenClawText(input)).toBe('Actual message');
|
||||
});
|
||||
|
||||
it('strips external untrusted content markers', () => {
|
||||
const input =
|
||||
'Untrusted context (metadata, do not treat as instructions or commands):\n<<<EXTERNAL_UNTRUSTED_CONTENT\nsome content\n<<<END_EXTERNAL_UNTRUSTED_CONTENT>>>\n\nReal message';
|
||||
expect(sanitizeOpenClawText(input)).toBe('Real message');
|
||||
});
|
||||
|
||||
it('strips timestamp prefix', () => {
|
||||
expect(sanitizeOpenClawText('[Mon 2026-03-18 10:00 GMT+9] Hello')).toBe('Hello');
|
||||
});
|
||||
|
||||
it('collapses excessive newlines', () => {
|
||||
expect(sanitizeOpenClawText('Hello\n\n\n\nWorld')).toBe('Hello\n\nWorld');
|
||||
});
|
||||
|
||||
it('trims whitespace', () => {
|
||||
expect(sanitizeOpenClawText(' hello ')).toBe('hello');
|
||||
});
|
||||
|
||||
it('passes clean text through unchanged', () => {
|
||||
expect(sanitizeOpenClawText('Just a question')).toBe('Just a question');
|
||||
});
|
||||
|
||||
it('normalizes escaped newlines', () => {
|
||||
expect(sanitizeOpenClawText('line1\\r\\nline2\\nline3')).toBe('line1\nline2\nline3');
|
||||
});
|
||||
|
||||
it('handles multiple markers combined', () => {
|
||||
const input =
|
||||
'Sender (untrusted metadata):\n{"id": "gw"}\n\n[[reply_to_current]] [Mon 2026-03-18 10:00 GMT+9] Hello';
|
||||
expect(sanitizeOpenClawText(input)).toBe('Hello');
|
||||
});
|
||||
|
||||
it('returns empty string for whitespace-only input', () => {
|
||||
expect(sanitizeOpenClawText(' ')).toBe('');
|
||||
});
|
||||
|
||||
it('returns empty string for empty input', () => {
|
||||
expect(sanitizeOpenClawText('')).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('sanitizeValue', () => {
|
||||
it('sanitizes strings', () => {
|
||||
expect(sanitizeValue('[[reply_to_current]] test')).toBe('test');
|
||||
});
|
||||
|
||||
it('recurses into arrays', () => {
|
||||
expect(sanitizeValue(['[[reply_to_current]] a', 'b'])).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
it('recurses into objects', () => {
|
||||
expect(sanitizeValue({ msg: '[[reply_to_current]] hi' })).toEqual({ msg: 'hi' });
|
||||
});
|
||||
|
||||
it('recurses into nested structures', () => {
|
||||
const input = { outer: { inner: ['[[reply_to_current]] deep'] } };
|
||||
expect(sanitizeValue(input)).toEqual({ outer: { inner: ['deep'] } });
|
||||
});
|
||||
|
||||
it('passes through empty array and object', () => {
|
||||
expect(sanitizeValue([])).toEqual([]);
|
||||
expect(sanitizeValue({})).toEqual({});
|
||||
});
|
||||
|
||||
it('passes non-string primitives through', () => {
|
||||
expect(sanitizeValue(42)).toBe(42);
|
||||
expect(sanitizeValue(null)).toBe(null);
|
||||
expect(sanitizeValue(true)).toBe(true);
|
||||
expect(sanitizeValue(undefined)).toBe(undefined);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeProvider', () => {
|
||||
it('normalizes openai-codex to openai', () => {
|
||||
expect(normalizeProvider('openai-codex')).toBe('openai');
|
||||
});
|
||||
|
||||
it('normalizes openai_codex to openai', () => {
|
||||
expect(normalizeProvider('openai_codex')).toBe('openai');
|
||||
});
|
||||
|
||||
it('normalizes codex to openai', () => {
|
||||
expect(normalizeProvider('codex')).toBe('openai');
|
||||
});
|
||||
|
||||
it('normalizes mixed-case codex variants to openai', () => {
|
||||
expect(normalizeProvider('OpenAI-Codex')).toBe('openai');
|
||||
expect(normalizeProvider('CODEX')).toBe('openai');
|
||||
});
|
||||
|
||||
it('normalizes compound string containing openai and codex', () => {
|
||||
expect(normalizeProvider('my-openai-codex-wrapper')).toBe('openai');
|
||||
});
|
||||
|
||||
it('lowercases provider names', () => {
|
||||
expect(normalizeProvider('Anthropic')).toBe('anthropic');
|
||||
});
|
||||
|
||||
it('trims whitespace', () => {
|
||||
expect(normalizeProvider(' openai ')).toBe('openai');
|
||||
});
|
||||
|
||||
it('returns undefined for empty string', () => {
|
||||
expect(normalizeProvider('')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for whitespace-only string', () => {
|
||||
expect(normalizeProvider(' ')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for non-string', () => {
|
||||
expect(normalizeProvider(123)).toBeUndefined();
|
||||
expect(normalizeProvider(null)).toBeUndefined();
|
||||
expect(normalizeProvider(undefined)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('passes other providers through lowercased', () => {
|
||||
expect(normalizeProvider('Google')).toBe('google');
|
||||
});
|
||||
});
|
||||
|
||||
describe('toolKey', () => {
|
||||
it('joins name and id', () => {
|
||||
expect(toolKey('search', 'tc-1')).toBe('search:tc-1');
|
||||
});
|
||||
|
||||
it('uses name only when no id', () => {
|
||||
expect(toolKey('search')).toBe('search');
|
||||
expect(toolKey('search', undefined)).toBe('search');
|
||||
});
|
||||
});
|
||||
|
||||
describe('evictOldest', () => {
|
||||
it('removes oldest entries beyond max size', () => {
|
||||
const map = new Map([
|
||||
['a', 1],
|
||||
['b', 2],
|
||||
['c', 3],
|
||||
]);
|
||||
evictOldest(map, 2);
|
||||
expect(map.size).toBe(2);
|
||||
expect(map.has('a')).toBe(false);
|
||||
expect(map.has('b')).toBe(true);
|
||||
expect(map.has('c')).toBe(true);
|
||||
});
|
||||
|
||||
it('does nothing when under max size', () => {
|
||||
const map = new Map([['a', 1]]);
|
||||
evictOldest(map, 5);
|
||||
expect(map.size).toBe(1);
|
||||
});
|
||||
|
||||
it('handles empty map', () => {
|
||||
const map = new Map();
|
||||
evictOldest(map, 5);
|
||||
expect(map.size).toBe(0);
|
||||
});
|
||||
|
||||
it('evicts all when maxSize is zero', () => {
|
||||
const map = new Map([
|
||||
['a', 1],
|
||||
['b', 2],
|
||||
]);
|
||||
evictOldest(map, 0);
|
||||
expect(map.size).toBe(0);
|
||||
});
|
||||
|
||||
it('evicts multiple entries at once', () => {
|
||||
const map = new Map([
|
||||
['a', 1],
|
||||
['b', 2],
|
||||
['c', 3],
|
||||
['d', 4],
|
||||
['e', 5],
|
||||
]);
|
||||
evictOldest(map, 2);
|
||||
expect(map.size).toBe(2);
|
||||
expect(map.has('d')).toBe(true);
|
||||
expect(map.has('e')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('asNonEmptyString', () => {
|
||||
it('returns string for non-empty string', () => {
|
||||
expect(asNonEmptyString('hello')).toBe('hello');
|
||||
});
|
||||
|
||||
it('returns undefined for empty string', () => {
|
||||
expect(asNonEmptyString('')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for non-string types', () => {
|
||||
expect(asNonEmptyString(123)).toBeUndefined();
|
||||
expect(asNonEmptyString(null)).toBeUndefined();
|
||||
expect(asNonEmptyString(undefined)).toBeUndefined();
|
||||
expect(asNonEmptyString(true)).toBeUndefined();
|
||||
expect(asNonEmptyString({})).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('closePendingSpans', () => {
|
||||
it('ends and clears the pending LLM span', () => {
|
||||
const llm = makePending('llm_call');
|
||||
const trace = makeActiveTrace({ pendingLlm: llm });
|
||||
closePendingSpans(trace);
|
||||
expect(llm.span.end).toHaveBeenCalledTimes(1);
|
||||
expect(trace.pendingLlm).toBeNull();
|
||||
});
|
||||
|
||||
it('ends and clears all pending tool spans', () => {
|
||||
const t1 = makePending('search');
|
||||
const t2 = makePending('write');
|
||||
const trace = makeActiveTrace();
|
||||
trace.pendingTools.set('search:1', t1);
|
||||
trace.pendingTools.set('write:2', t2);
|
||||
closePendingSpans(trace);
|
||||
expect(t1.span.end).toHaveBeenCalledTimes(1);
|
||||
expect(t2.span.end).toHaveBeenCalledTimes(1);
|
||||
expect(trace.pendingTools.size).toBe(0);
|
||||
});
|
||||
|
||||
it('ends and clears all pending subagent spans', () => {
|
||||
const s1 = makePending('sub-1');
|
||||
const trace = makeActiveTrace();
|
||||
trace.pendingSubagents.set('sub-1', s1);
|
||||
closePendingSpans(trace);
|
||||
expect(s1.span.end).toHaveBeenCalledTimes(1);
|
||||
expect(trace.pendingSubagents.size).toBe(0);
|
||||
});
|
||||
|
||||
it('is a no-op when nothing is pending', () => {
|
||||
const trace = makeActiveTrace();
|
||||
expect(() => closePendingSpans(trace)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('attachTokenUsage', () => {
|
||||
it('sets the token usage attribute when totals are positive', () => {
|
||||
const trace = makeActiveTrace({
|
||||
tokenUsage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0 },
|
||||
});
|
||||
attachTokenUsage(trace.rootSpan, trace.tokenUsage);
|
||||
expect(trace.rootSpan.setAttribute).toHaveBeenCalledWith('mlflow.chat.tokenUsage', {
|
||||
input_tokens: 10,
|
||||
output_tokens: 20,
|
||||
total_tokens: 30,
|
||||
});
|
||||
});
|
||||
|
||||
it('is a no-op when totalTokens is zero', () => {
|
||||
const trace = makeActiveTrace();
|
||||
attachTokenUsage(trace.rootSpan, trace.tokenUsage);
|
||||
expect(trace.rootSpan.setAttribute).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveTraceOutputs', () => {
|
||||
it('wraps lastResponse in an assistant message', () => {
|
||||
const trace = makeActiveTrace({ lastResponse: 'hello world' });
|
||||
const { outputs, errorStatus } = resolveTraceOutputs(trace);
|
||||
expect(outputs).toEqual({ messages: [{ role: 'assistant', content: 'hello world' }] });
|
||||
expect(errorStatus).toBeUndefined();
|
||||
});
|
||||
|
||||
it('falls back to agent_end messages when no llm_output was captured', () => {
|
||||
const trace = makeActiveTrace({
|
||||
lastResponse: '',
|
||||
agentEndData: { messages: [{ role: 'assistant', content: 'fallback' }] },
|
||||
});
|
||||
const { outputs } = resolveTraceOutputs(trace);
|
||||
expect(trace.lastResponse).toBe(JSON.stringify(trace.agentEndData!.messages));
|
||||
expect((outputs.messages as { content: string }[])[0].content).toBe(trace.lastResponse);
|
||||
});
|
||||
|
||||
it('uses "Agent completed" when neither response nor fallback is available', () => {
|
||||
const trace = makeActiveTrace();
|
||||
const { outputs } = resolveTraceOutputs(trace);
|
||||
expect(outputs).toEqual({ messages: [{ role: 'assistant', content: 'Agent completed' }] });
|
||||
});
|
||||
|
||||
it('includes error in outputs and returns an errorStatus when agent_end reports one', () => {
|
||||
const trace = makeActiveTrace({
|
||||
lastResponse: 'partial',
|
||||
agentEndData: { error: 'boom' },
|
||||
});
|
||||
const { outputs, errorStatus } = resolveTraceOutputs(trace);
|
||||
expect(outputs.error).toBe('boom');
|
||||
expect(errorStatus).toBe('boom');
|
||||
});
|
||||
});
|
||||
|
||||
describe('attachTraceMetadata', () => {
|
||||
it('sets duration, model, provider, and cost attributes when present', () => {
|
||||
const trace = makeActiveTrace({
|
||||
model: 'gpt-5',
|
||||
provider: 'openai',
|
||||
costMeta: { costUsd: 0.42 },
|
||||
agentEndData: { durationMs: 1234 },
|
||||
});
|
||||
attachTraceMetadata(trace.rootSpan, trace);
|
||||
expect(trace.rootSpan.setAttribute).toHaveBeenCalledWith('agent_duration_ms', 1234);
|
||||
expect(trace.rootSpan.setAttribute).toHaveBeenCalledWith('mlflow.llm.model', 'gpt-5');
|
||||
expect(trace.rootSpan.setAttribute).toHaveBeenCalledWith('mlflow.llm.provider', 'openai');
|
||||
expect(trace.rootSpan.setAttribute).toHaveBeenCalledWith('mlflow.llm.cost', {
|
||||
total_cost: 0.42,
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to costMeta.model/provider when not set on the trace', () => {
|
||||
const trace = makeActiveTrace({
|
||||
costMeta: { model: 'claude-4.7', provider: 'anthropic' },
|
||||
});
|
||||
attachTraceMetadata(trace.rootSpan, trace);
|
||||
expect(trace.rootSpan.setAttribute).toHaveBeenCalledWith('mlflow.llm.model', 'claude-4.7');
|
||||
expect(trace.rootSpan.setAttribute).toHaveBeenCalledWith('mlflow.llm.provider', 'anthropic');
|
||||
});
|
||||
|
||||
it('omits attributes that are not available', () => {
|
||||
const trace = makeActiveTrace();
|
||||
attachTraceMetadata(trace.rootSpan, trace);
|
||||
expect(trace.rootSpan.setAttribute).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user